From 2de99c18ff02775a60ad58c81a8cccdc82c95eee Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Tue, 25 May 2021 17:42:12 -0700 Subject: [PATCH 001/258] Take IP address or DNS names (#625) * Add "-dnsname" to perfrpc so that we can handle DNS names * Update name of DNS or IP address field in client and server structures --- imgr/imgrpkg/retry-rpc.go | 2 +- imgr/imgrpkg/utils_test.go | 2 +- jrpcfs/lease_test.go | 4 ++-- jrpcfs/retryrpc.go | 2 +- pfsagentd/request.go | 2 +- retryrpc/api.go | 25 ++++++++++++++---------- retryrpc/perfrpc/client.go | 27 ++++++++++++++++++++------ retryrpc/perfrpc/server.go | 39 ++++++++++++++++++++++++++------------ retryrpc/perfrpc/tls.go | 11 +++++++++-- retryrpc/retryrpc_test.go | 13 +++++++------ retryrpc/stress_test.go | 4 ++-- retryrpc/upcall_test.go | 4 ++-- 12 files changed, 89 insertions(+), 46 deletions(-) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index db0f6fea..a7a555c6 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -35,7 +35,7 @@ func startRetryRPCServer() (err error) { retryrpcServerConfig = &retryrpc.ServerConfig{ LongTrim: globals.config.RetryRPCTTLCompleted, ShortTrim: globals.config.RetryRPCAckTrim, - IPAddr: globals.config.PublicIPAddr, + DNSOrIPAddr: globals.config.PublicIPAddr, Port: int(globals.config.RetryRPCPort), DeadlineIO: globals.config.RetryRPCDeadlineIO, KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index f00d9047..e54e884c 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -227,7 +227,7 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { } testGlobals.retryrpcClientConfig = &retryrpc.ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testRetryRPCPort, RootCAx509CertificatePEM: testGlobals.caCertPEMBlock, Callbacks: retryrpcCallbacks, diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 83552bd2..8cd840a1 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -1505,7 +1505,7 @@ func BenchmarkRpcLeaseRemote(b *testing.B) { } retryrpcClientConfig = &retryrpc.ClientConfig{ - IPAddr: globals.publicIPAddr, + DNSOrIPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, Callbacks: testRpcLeaseClient, @@ -1909,7 +1909,7 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { } retryrpcClientConfig = &retryrpc.ClientConfig{ - IPAddr: globals.publicIPAddr, + DNSOrIPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, Callbacks: testRpcLeaseClient, diff --git a/jrpcfs/retryrpc.go b/jrpcfs/retryrpc.go index 429f473d..4021ea39 100644 --- a/jrpcfs/retryrpc.go +++ b/jrpcfs/retryrpc.go @@ -21,7 +21,7 @@ func retryRPCServerUp(jserver *Server) { retryConfig := &retryrpc.ServerConfig{ LongTrim: globals.retryRPCTTLCompleted, ShortTrim: globals.retryRPCAckTrim, - IPAddr: globals.publicIPAddr, + DNSOrIPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), DeadlineIO: globals.retryRPCDeadlineIO, KeepAlivePeriod: globals.retryRPCKeepAlivePeriod, diff --git a/pfsagentd/request.go b/pfsagentd/request.go index e79ca5a8..58eff0e5 100644 --- a/pfsagentd/request.go +++ b/pfsagentd/request.go @@ -42,7 +42,7 @@ func doMountProxyFS() { ) retryrpcConfig := &retryrpc.ClientConfig{ - IPAddr: globals.config.RetryRPCPublicIPAddr, + DNSOrIPAddr: globals.config.RetryRPCPublicIPAddr, Port: int(globals.config.RetryRPCPort), RootCAx509CertificatePEM: globals.retryRPCCACertPEM, Callbacks: &globals, diff --git a/retryrpc/api.go b/retryrpc/api.go index 509d627e..10fbb2f0 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -33,7 +33,7 @@ type Server struct { completedLongTTL time.Duration // How long a completed request stays on queue completedAckTrim time.Duration // How frequently trim requests acked by client svrMap map[string]*methodArgs // Key: Method name - ipaddr string // IP address server listens too + dnsOrIpaddr string // DNS or IP address server listens too port int // Port of server clientIDNonce uint64 // Nonce used to create a unique client ID netListener net.Listener // Accepts on this to skip TLS upgrade @@ -61,20 +61,25 @@ type Server struct { type ServerConfig struct { LongTrim time.Duration // How long the results of an RPC are stored on a Server before removed ShortTrim time.Duration // How frequently completed and ACKed RPCs results are removed from Server - IPAddr string // IP Address that Server uses to listen + DNSOrIPAddr string // DNS or IP Address that Server uses to listen Port int // Port that Server uses to listen DeadlineIO time.Duration // How long I/Os on sockets wait even if idle KeepAlivePeriod time.Duration // How frequently a KEEPALIVE is sent TLSCertificate tls.Certificate // TLS Certificate to present to Clients (or tls.Certificate{} if using TCP) - dontStartTrimmers bool // Used for testing + dontStartTrimmers bool // Used for testingD } // NewServer creates the Server object func NewServer(config *ServerConfig) *Server { - server := &Server{ipaddr: config.IPAddr, port: config.Port, completedLongTTL: config.LongTrim, - completedAckTrim: config.ShortTrim, deadlineIO: config.DeadlineIO, - keepAlivePeriod: config.KeepAlivePeriod, dontStartTrimmers: config.dontStartTrimmers, - tlsCertificate: config.TLSCertificate} + server := &Server{ + dnsOrIpaddr: config.DNSOrIPAddr, + port: config.Port, + completedLongTTL: config.LongTrim, + completedAckTrim: config.ShortTrim, + deadlineIO: config.DeadlineIO, + keepAlivePeriod: config.KeepAlivePeriod, + dontStartTrimmers: config.dontStartTrimmers, + tlsCertificate: config.TLSCertificate} server.svrMap = make(map[string]*methodArgs) server.perClientInfo = make(map[uint64]*clientInfo) server.completedTickerDone = make(chan bool) @@ -92,7 +97,7 @@ func (server *Server) Register(retrySvr interface{}) (err error) { // Start listener func (server *Server) Start() (err error) { - hostPortStr := net.JoinHostPort(server.ipaddr, fmt.Sprintf("%d", server.port)) + hostPortStr := net.JoinHostPort(server.dnsOrIpaddr, fmt.Sprintf("%d", server.port)) tlsConfig := &tls.Config{ Certificates: []tls.Certificate{server.tlsCertificate}, @@ -309,7 +314,7 @@ type ClientCallbacks interface { // ClientConfig is used to configure a retryrpc Client type ClientConfig struct { - IPAddr string // IP Address of Server + DNSOrIPAddr string // DNS name or IP Address of Server Port int // Port of Server RootCAx509CertificatePEM []byte // If TLS...Root certificate; If TCP... nil Callbacks interface{} // Structure implementing ClientCallbacks @@ -335,7 +340,7 @@ func NewClient(config *ClientConfig) (client *Client, err error) { client = &Client{ connection: &connectionTracker{ state: INITIAL, - hostPortStr: net.JoinHostPort(config.IPAddr, fmt.Sprintf("%d", config.Port)), + hostPortStr: net.JoinHostPort(config.DNSOrIPAddr, fmt.Sprintf("%d", config.Port)), }, cb: config.Callbacks, keepAlivePeriod: config.KeepAlivePeriod, diff --git a/retryrpc/perfrpc/client.go b/retryrpc/perfrpc/client.go index 5c59fddb..5c02ea43 100644 --- a/retryrpc/perfrpc/client.go +++ b/retryrpc/perfrpc/client.go @@ -79,15 +79,21 @@ func (cb *stressMyClient) Interrupt(payload []byte) { func pfsagent(agentID uint64, method string, agentWG *sync.WaitGroup, warmUpCompleteWG *sync.WaitGroup, fence chan int) { var ( clientConfig *retryrpc.ClientConfig + ipAddrOrDNS string ) - defer agentWG.Done() + if globals.cs.dnsName != "" { + ipAddrOrDNS = globals.cs.dnsName + } else { + ipAddrOrDNS = globals.cs.ipAddr + } + cb := &stressMyClient{} cb.cond = sync.NewCond(&cb.Mutex) if globals.useTLS { clientConfig = &retryrpc.ClientConfig{ - IPAddr: globals.cs.ipAddr, + DNSOrIPAddr: ipAddrOrDNS, Port: globals.cs.port, RootCAx509CertificatePEM: globals.tlsCerts.caCertPEMBlock, Callbacks: cb, @@ -96,7 +102,7 @@ func pfsagent(agentID uint64, method string, agentWG *sync.WaitGroup, warmUpComp } } else { clientConfig = &retryrpc.ClientConfig{ - IPAddr: globals.cs.ipAddr, + DNSOrIPAddr: ipAddrOrDNS, Port: globals.cs.port, RootCAx509CertificatePEM: nil, Callbacks: cb, @@ -203,6 +209,7 @@ type ClientSubcommand struct { fs *flag.FlagSet clients int // Number of clients which will be sending messages ipAddr string // IP Address of server + dnsName string // DNS Name of server warmUpCnt int // Number of messages to send to "warm up" messages int // Number of messages to send port int // Port on which the server is listening @@ -216,7 +223,8 @@ func NewClientCommand() *ClientSubcommand { } cs.fs.IntVar(&cs.clients, "clients", 0, "Number of clients which will be sending a subset of messages") - cs.fs.StringVar(&cs.ipAddr, "ipaddr", "", "IP Address of server") + cs.fs.StringVar(&cs.ipAddr, "ipaddr", "", "IP Address of server (optional)") + cs.fs.StringVar(&cs.dnsName, "dnsname", "", "DNS Name of server (optional)") cs.fs.IntVar(&cs.messages, "messages", 0, "Number of total messages to send") cs.fs.IntVar(&cs.warmUpCnt, "warmupcnt", 0, "Number of total messages to send warm up client/server") cs.fs.IntVar(&cs.port, "port", 0, "Port on which the server is listening") @@ -235,8 +243,13 @@ func (cs *ClientSubcommand) Name() string { } func (cs *ClientSubcommand) Run() (err error) { - if cs.ipAddr == "" { - err = fmt.Errorf("IP Address cannot be blank") + if cs.ipAddr == "" && cs.dnsName == "" { + err = fmt.Errorf("Must pass either IP Address or DNS name") + cs.fs.PrintDefaults() + return + } + if cs.ipAddr != "" && cs.dnsName != "" { + err = fmt.Errorf("Must pass only IP Address or DNS name") cs.fs.PrintDefaults() return } @@ -260,6 +273,8 @@ func (cs *ClientSubcommand) Run() (err error) { cs.fs.PrintDefaults() return } + + // Start debug webserver if we have a debug port if cs.dbgport != "" { hostPort := net.JoinHostPort("localhost", cs.dbgport) go http.ListenAndServe(hostPort, nil) diff --git a/retryrpc/perfrpc/server.go b/retryrpc/perfrpc/server.go index bcf98b92..5598baea 100644 --- a/retryrpc/perfrpc/server.go +++ b/retryrpc/perfrpc/server.go @@ -16,21 +16,28 @@ import ( "github.com/NVIDIA/proxyfs/retryrpc" ) -func configNewServer(ipAddr string, port int, lt time.Duration, tlsDir string, dontStartTrimmers bool, useTLS bool) (rrSvr *retryrpc.Server, err error) { +func configNewServer(ipAddr string, dnsName string, port int, lt time.Duration, tlsDir string, dontStartTrimmers bool, useTLS bool) (rrSvr *retryrpc.Server, err error) { var ( - config *retryrpc.ServerConfig + config *retryrpc.ServerConfig + ipAddrOrDNS string ) + if dnsName != "" { + ipAddrOrDNS = dnsName + } else { + ipAddrOrDNS = ipAddr + } + if useTLS { _ = os.Mkdir(tlsDir, os.ModePerm) - // Create certificate/CA that last for 30 days - tlsCert := tlsCertsAllocate(ipAddr, 30*24*time.Hour, tlsDir) + // Create certificate/CA that lasts for 30 days + tlsCert := tlsCertsAllocate(ipAddr, dnsName, 30*24*time.Hour, tlsDir) config = &retryrpc.ServerConfig{ LongTrim: lt, ShortTrim: 100 * time.Millisecond, - IPAddr: ipAddr, + DNSOrIPAddr: ipAddrOrDNS, Port: port, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, @@ -40,7 +47,7 @@ func configNewServer(ipAddr string, port int, lt time.Duration, tlsDir string, d config = &retryrpc.ServerConfig{ LongTrim: lt, ShortTrim: 100 * time.Millisecond, - IPAddr: ipAddr, + DNSOrIPAddr: ipAddrOrDNS, Port: port, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, @@ -57,12 +64,12 @@ func configNewServer(ipAddr string, port int, lt time.Duration, tlsDir string, d // TODO - should this be a goroutine with WG????? // want way to shutdown with curl/WS? how dump stats? -func becomeAServer(ipAddr string, port int, tlsDir string, useTLS bool) error { +func becomeAServer(ipAddr string, dnsName string, port int, tlsDir string, useTLS bool) error { // Create new TestPingServer - needed for calling RPCs myJrpcfs := &PerfPingServer{} - rrSvr, err := configNewServer(ipAddr, port, 10*time.Minute, tlsDir, false, useTLS) + rrSvr, err := configNewServer(ipAddr, dnsName, port, 10*time.Minute, tlsDir, false, useTLS) if err != nil { return err } @@ -99,6 +106,7 @@ func becomeAServer(ipAddr string, port int, tlsDir string, useTLS bool) error { type ServerSubcommand struct { fs *flag.FlagSet ipAddr string // IP Address server will use + dnsName string // DNS Name of server port int // Port on which to listen tlsDir string // Directory to read for TLS info dbgport string // Debug port for pprof webserver @@ -108,7 +116,8 @@ func NewServerCommand() *ServerSubcommand { s := &ServerSubcommand{ fs: flag.NewFlagSet("server", flag.ContinueOnError), } - s.fs.StringVar(&s.ipAddr, "ipaddr", "", "IP Address server will use") + s.fs.StringVar(&s.ipAddr, "ipaddr", "", "IP Address server will use (optional)") + s.fs.StringVar(&s.dnsName, "dnsname", "", "DNS Name server will use (optional)") s.fs.IntVar(&s.port, "port", 0, "Port on which to listen") s.fs.StringVar(&s.tlsDir, "tlsdir", "", "Directory containing TLS info") s.fs.StringVar(&s.dbgport, "dbgport", "", "Debug port for pprof webserver (optional)") @@ -126,8 +135,13 @@ func (s *ServerSubcommand) Name() string { // TODO - how dump bucketstats, etc???? do from webserver? func (s *ServerSubcommand) Run() (err error) { - if s.ipAddr == "" { - err = fmt.Errorf("IP Address cannot be blank") + if s.ipAddr == "" && s.dnsName == "" { + err = fmt.Errorf("Must pass either IP Address or DNS name") + s.fs.PrintDefaults() + return + } + if s.ipAddr != "" && s.dnsName != "" { + err = fmt.Errorf("Must pass only IP Address or DNS name") s.fs.PrintDefaults() return } @@ -142,11 +156,12 @@ func (s *ServerSubcommand) Run() (err error) { return } + // Start debug webserver if we have a debug port if s.dbgport != "" { hostPort := net.JoinHostPort("localhost", s.dbgport) go http.ListenAndServe(hostPort, nil) } - err = becomeAServer(s.ipAddr, s.port, s.tlsDir, true) + err = becomeAServer(s.ipAddr, s.dnsName, s.port, s.tlsDir, true) return err } diff --git a/retryrpc/perfrpc/tls.go b/retryrpc/perfrpc/tls.go index 2c0d4186..d2007d87 100644 --- a/retryrpc/perfrpc/tls.go +++ b/retryrpc/perfrpc/tls.go @@ -36,7 +36,7 @@ func tlsSetFileNames(tlsCerts *tlsCertsStruct, tlsDir string) { } // Utility function to initialize tlsCerts -func tlsCertsAllocate(ipAddr string, ttl time.Duration, tlsDir string) (tlsCerts *tlsCertsStruct) { +func tlsCertsAllocate(ipAddr string, dnsName string, ttl time.Duration, tlsDir string) (tlsCerts *tlsCertsStruct) { var ( err error ) @@ -63,6 +63,13 @@ func tlsCertsAllocate(ipAddr string, ttl time.Duration, tlsDir string) (tlsCerts os.Exit(1) } + dnsToIPAddr, lookupErr := net.LookupIP(dnsName) + if dnsName != "" && lookupErr != nil { + fmt.Printf("Unable to lookup DNS name: %v - err: %v\n", dnsName, lookupErr) + os.Exit(1) + } + ipAddr = dnsToIPAddr[0].String() + tlsCerts.endpointCertPEMBlock, tlsCerts.endpointKeyPEMBlock, err = icertpkg.GenEndpointCert( icertpkg.GenerateKeyAlgorithmEd25519, pkix.Name{ @@ -73,7 +80,7 @@ func tlsCertsAllocate(ipAddr string, ttl time.Duration, tlsDir string) (tlsCerts StreetAddress: []string{}, PostalCode: []string{}, }, - []string{}, + []string{dnsName}, []net.IP{net.ParseIP(ipAddr)}, ttl, tlsCerts.caCertPEMBlock, diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index 6a3ed0f4..56c11b71 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -75,6 +75,7 @@ func testTLSCertsAllocate(t *testing.T) { testTLSCerts.caKeyPEMBlock, "", "") + if nil != err { t.Fatalf("icertpkg.genEndpointCert() failed: %v", err) } @@ -114,7 +115,7 @@ func getNewServer(lt time.Duration, dontStartTrimmers bool, useTLS bool) (rrSvr config = &ServerConfig{ LongTrim: lt, ShortTrim: 100 * time.Millisecond, - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, @@ -125,7 +126,7 @@ func getNewServer(lt time.Duration, dontStartTrimmers bool, useTLS bool) (rrSvr config = &ServerConfig{ LongTrim: lt, ShortTrim: 100 * time.Millisecond, - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, @@ -193,7 +194,7 @@ func testServer(t *testing.T, useTLS bool) { // Now - setup a client to send requests to the server if useTLS { clientConfig = &ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, Callbacks: nil, @@ -202,7 +203,7 @@ func testServer(t *testing.T, useTLS bool) { } } else { clientConfig = &ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, RootCAx509CertificatePEM: nil, Callbacks: nil, @@ -266,7 +267,7 @@ func testBtree(t *testing.T, useTLS bool) { // Setup a client - we only will be targeting the btree if useTLS { clientConfig = &ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, Callbacks: nil, @@ -275,7 +276,7 @@ func testBtree(t *testing.T, useTLS bool) { } } else { clientConfig = &ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, RootCAx509CertificatePEM: nil, Callbacks: nil, diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index 4b3eefae..ae12bf61 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -357,7 +357,7 @@ func pfsagent(t *testing.T, rrSvr *Server, agentID uint64, method string, agentW cb.cond = sync.NewCond(&cb.Mutex) if useTLS { clientConfig = &ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, Callbacks: cb, @@ -366,7 +366,7 @@ func pfsagent(t *testing.T, rrSvr *Server, agentID uint64, method string, agentW } } else { clientConfig = &ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, RootCAx509CertificatePEM: nil, Callbacks: cb, diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index 7f091db7..25191bfd 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -70,7 +70,7 @@ func testUpCall(t *testing.T, useTLS bool) { if useTLS { clientConfig = &ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, Callbacks: cb, @@ -79,7 +79,7 @@ func testUpCall(t *testing.T, useTLS bool) { } } else { clientConfig = &ClientConfig{ - IPAddr: testIPAddr, + DNSOrIPAddr: testIPAddr, Port: testPort, RootCAx509CertificatePEM: nil, Callbacks: cb, From 693b2b6ade64320e983545cc72fe83429c77c7eb Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 11 May 2021 17:23:23 -0700 Subject: [PATCH 002/258] Added AuthTokenCheck --- imgr/imgr.conf | 6 +++-- imgr/imgrpkg/api.go | 4 ++++ imgr/imgrpkg/globals.go | 45 ++++++++++++++++++++++++++------------ imgr/imgrpkg/retry-rpc.go | 40 ++++++++++++++++++++++++++++++--- imgr/imgrpkg/utils_test.go | 2 ++ imgr/imgrpkg/volume.go | 1 + 6 files changed, 79 insertions(+), 19 deletions(-) diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 390fa139..a514e03a 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -15,10 +15,12 @@ RetryRPCKeepAlivePeriod: 60s RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used -FetchNonceRangeToReturn: 100 - CheckPointInterval: 10s +AuthTokenCheckInterval: 1m + +FetchNonceRangeToReturn: 100 + MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 3125f8e0..addb87f0 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -26,6 +26,8 @@ // // CheckPointInterval: 10s // +// AuthTokenCheckInterval: 1m +// // FetchNonceRangeToReturn: 100 // // MinLeaseDuration: 250ms @@ -219,6 +221,8 @@ func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, // FetchNonceRangeRequestStruct is the request object for FetchNonceRange. // +// Possible errors: EAuthTokenRejected EUnknownMountID +// type FetchNonceRangeRequestStruct struct { MountID string } diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index dc6c17cb..56209bf8 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -36,6 +36,8 @@ type configStruct struct { CheckPointInterval time.Duration + AuthTokenCheckInterval time.Duration + FetchNonceRangeToReturn uint64 MinLeaseDuration time.Duration @@ -98,6 +100,8 @@ type statsStruct struct { VolumeCheckPointUsecs bucketstats.BucketLog2Round + AuthTokenCheckUsecs bucketstats.BucketLog2Round + SharedLeaseRequestUsecs bucketstats.BucketLog2Round PromoteLeaseRequestUsecs bucketstats.BucketLog2Round ExclusiveLeaseRequestUsecs bucketstats.BucketLog2Round @@ -133,21 +137,27 @@ type mountStruct struct { listElement *list.Element // LRU element on either volumeStruct.{healthy|expired}MountList } +type inodeTableLayoutElementStruct struct { + objectSize uint64 // matches ilayout.InodeTableLayoutEntryV1Struct.ObjectSize + bytesReferenced uint64 // matches ilayout.InodeTableLayoutEntryV1Struct.BytesReferenced +} + type volumeStruct struct { - sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() - name string // - storageURL string // - mountMap map[string]*mountStruct // key == mountStruct.mountID - healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false - leasesExpiredMountList *list.List // list of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value - authTokenExpiredMountList *list.List // list of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) - deleting bool // - checkPoint *ilayout.CheckPointV1Struct // == nil if not currently mounted and/or checkpointing - superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing - inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing - pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber - checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() - checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG + sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() + name string // + storageURL string // + mountMap map[string]*mountStruct // key == mountStruct.mountID + healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false + leasesExpiredMountList *list.List // list of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value + authTokenExpiredMountList *list.List // list of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) + deleting bool // + checkPoint *ilayout.CheckPointV1Struct // == nil if not currently mounted and/or checkpointing + superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing + inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing + inodeTableLayout map[uint64]*inodeTableLayoutElementStruct // == nil if not currently mounted and/or checkpointing; key == objectNumber (matching ilayout.InodeTableLayoutEntryV1Struct.ObjectNumber) + pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber + checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() + checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG } type globalsStruct struct { @@ -268,6 +278,11 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { logFatal(err) } + globals.config.AuthTokenCheckInterval, err = confMap.FetchOptionValueDuration("IMGR", "AuthTokenCheckInterval") + if nil != err { + logFatal(err) + } + globals.config.FetchNonceRangeToReturn, err = confMap.FetchOptionValueUint64("IMGR", "FetchNonceRangeToReturn") if nil != err { logFatal(err) @@ -372,6 +387,8 @@ func uninitializeGlobals() (err error) { globals.config.CheckPointInterval = time.Duration(0) + globals.config.AuthTokenCheckInterval = time.Duration(0) + globals.config.FetchNonceRangeToReturn = 0 globals.config.MinLeaseDuration = time.Duration(0) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index a7a555c6..4192bb26 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -143,7 +143,7 @@ retryGenerateMountID: leasesExpired: false, authTokenExpired: false, authToken: mountRequest.AuthToken, - lastAuthTime: time.Now(), + lastAuthTime: startTime, } volume.mountMap[mountIDAsString] = mount @@ -207,6 +207,7 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch mount *mountStruct ok bool startTime time.Time + volume *volumeStruct ) startTime = time.Now() @@ -220,14 +221,19 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch mount, ok = globals.mountMap[fetchNonceRangeRequest.MountID] if !ok { globals.RUnlock() - err = fmt.Errorf("MountID not recognized") + err = fmt.Errorf("%s %s", EUnknownMountID, fetchNonceRangeRequest.MountID) return } - fmt.Printf("TODO: Perform fetchNonceRange() for mountStruct @ %p\n", mount) + volume = mount.volume + volume.Lock() globals.RUnlock() + fmt.Printf("TODO: Perform fetchNonceRange() for mountStruct @ %p\n", mount) + + volume.Unlock() + return fmt.Errorf(ETODO + " fetchNonceRange") } @@ -374,3 +380,31 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) return fmt.Errorf(ETODO + " lease") } + +func (mount *mountStruct) authTokenHasExpired() (authTokenExpired bool) { + var ( + err error + startTime time.Time + ) + + if mount.authTokenExpired { + return true + } + + startTime = time.Now() + + if startTime.Sub(mount.lastAuthTime) < globals.config.AuthTokenCheckInterval { + return false + } + + _, err = swiftObjectGet(mount.volume.storageURL, mount.authToken, ilayout.CheckPointObjectNumber) + + globals.stats.AuthTokenCheckUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + + if nil == err { + mount.lastAuthTime = startTime + return false + } else { + return true + } +} diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index e54e884c..3f688432 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -136,6 +136,8 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { "IMGR.CheckPointInterval=10s", + "IMGR.AuthTokenCheckInterval=1m", + "IMGR.FetchNonceRangeToReturn=100", "IMGR.MinLeaseDuration=250ms", diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 9830b043..0e698983 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -731,6 +731,7 @@ func putVolume(name string, storageURL string) (err error) { checkPoint: nil, superBlock: nil, inodeTable: nil, + inodeTableLayout: nil, pendingObjectDeleteSet: make(map[uint64]struct{}), checkPointControlChan: nil, } From c383debbaf274da177506d51cb778d7614d0d5e8 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 11 May 2021 18:44:50 -0700 Subject: [PATCH 003/258] Updated various icert/iclient/imgr/iswift godoc's --- icert/README.md | 4 ++-- icert/icertpkg/api.go | 10 ++++++-- icert/main.go | 51 +++++++++++++++++++++++++++++++++++++++++ iclient/main.go | 6 +++++ imgr/main.go | 6 +++++ iswift/iswiftpkg/api.go | 25 ++++++++++++++++++-- iswift/main.go | 6 +++++ 7 files changed, 102 insertions(+), 6 deletions(-) diff --git a/icert/README.md b/icert/README.md index a3440b46..be29099b 100644 --- a/icert/README.md +++ b/icert/README.md @@ -47,9 +47,9 @@ A `-ttl` must be specified. Both `-caCert` and `-caKey` must be specified. -if `-ca` is specified: +If `-ca` is specified: * neither `-cert` nor `key` may be specified -* no `-dns` or `-ip` may be specified +* neither `-dns` nor `-ip` may be specified If `-ca` is not specified: * both `-cert` and `-key` must be specified diff --git a/icert/icertpkg/api.go b/icert/icertpkg/api.go index 40617e0c..21b12395 100644 --- a/icert/icertpkg/api.go +++ b/icert/icertpkg/api.go @@ -1,8 +1,14 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 +// Package icertpkg provides functions to generate certificates compatible +// with standard TLS and HTTPS packages. Functions are provided to create +// RootCA certificates that may then subsequently be used to generate +// endpoint certificates signed by a RootCA. The functions are designed to +// work with either on-disk PEM files and/or in-memory PEM blocks (byte slices). +// // Inspired by https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ - +// package icertpkg import ( @@ -33,7 +39,7 @@ const ( // GeneratedFilePerm is the permission bits that, after the application // of umask, will specify the mode of the created cert|key files. // - GeneratedFilePerm = 0644 + GeneratedFilePerm = 0400 ) // GenCACert is called to generate a Certificate Authority using the requested diff --git a/icert/main.go b/icert/main.go index 898876e0..0806766a 100644 --- a/icert/main.go +++ b/icert/main.go @@ -1,6 +1,57 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 +// Program icert provides a command-line wrapper around package icertpkg APIs. +// +// The following can be obtained by running the "icert -h" command: +// +// Usage of icert: +// -ca +// generated CA Certicate usable for signing Endpoint Certificates +// -caCert string +// path to CA Certificate +// -caKey string +// path to CA Certificate's PrivateKey +// -cert string +// path to Endpoint Certificate +// -country value +// generated Certificate's Subject.Country +// -dns value +// generated Certificate's DNS Name +// -ed25519 +// generate key via Ed25519 +// -ip value +// generated Certificate's IP Address +// -key string +// path to Endpoint Certificate's PrivateKey +// -locality value +// generated Certificate's Subject.Locality +// -organization value +// generated Certificate's Subject.Organization +// -postalCode value +// generated Certificate's Subject.PostalCode +// -province value +// generated Certificate's Subject.Province +// -rsa +// generate key via RSA +// -streetAddress value +// generated Certificate's Subject.StreetAddress +// -ttl uint +// generated Certificate's time to live in days +// -v verbose mode +// +// Precisely one of "-ed25519" or "-rsa" must be specified. +// +// A "-ttl" must be specified. +// +// Both "-caCert" and "-caKey" must be specified. +// +// If "-ca" is specified, none of "-cert" nor "key" may be specified. +// Similarly, neither "-dns" nor "-ip" may be specified. +// +// If "-ca" is not specified, both "-cert" and "-key" must be specified. +// Similarly, at least one "-dns" and/or one "-ip" must be specified. +// package main import ( diff --git a/iclient/main.go b/iclient/main.go index 0f815e8a..fdd73cb3 100644 --- a/iclient/main.go +++ b/iclient/main.go @@ -1,6 +1,12 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 +// Program iclient provides a command-line wrapper around package iclientpkg APIs. +// +// The program requires a single argument that is a path to a package config +// formatted configuration to load. Optionally, overrides the the config may +// be passed as additional arguments in the form .=. +// package main func main() {} diff --git a/imgr/main.go b/imgr/main.go index 7c2508f8..e5301d88 100644 --- a/imgr/main.go +++ b/imgr/main.go @@ -1,6 +1,12 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 +// Program imgr provides a command-line wrapper around package imgrpkg APIs. +// +// The program requires a single argument that is a path to a package config +// formatted configuration to load. Optionally, overrides the the config may +// be passed as additional arguments in the form .=. +// package main import ( diff --git a/iswift/iswiftpkg/api.go b/iswift/iswiftpkg/api.go index 918047b7..2966de23 100644 --- a/iswift/iswiftpkg/api.go +++ b/iswift/iswiftpkg/api.go @@ -1,6 +1,24 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 +// Package iswiftpkg implements an emulation of OpenStack Swift by presenting +// a Swift Proxy responding to a minimal set of base OpenStack Swift HTTP +// methods. While there is no support for TLS, a simple Auth functionality +// is provided and its usage is enforced. +// +// To configure an iswiftpkg instance, Start() is called passing, as the sole +// argument, a package conf ConfMap. Here is a sample .conf file: +// +// [ISWIFT] +// SwiftProxyIPAddr: 127.0.0.1 +// SwiftProxyTCPPort: 8080 +// +// MaxAccountNameLength: 256 +// MaxContainerNameLength: 256 +// MaxObjectNameLength: 1024 +// AccountListingLimit: 10000 +// ContainerListingLimit: 10000 +// package iswiftpkg import ( @@ -8,20 +26,23 @@ import ( ) // Start is called to start serving the NoAuth Swift Proxy Port and, -// optionally, the Auth Swift Proxy Port +// optionally, the Auth Swift Proxy Port. // func Start(confMap conf.ConfMap) (err error) { err = start(confMap) return } -// Stop is called to stop serving +// Stop is called to stop serving. // func Stop() (err error) { err = stop() return } +// ForceReAuth is called to force a "401 Unauthorized" response to a +// client's subsequent request forcing the client to reauthenticate. +// func ForceReAuth() { forceReAuth() } diff --git a/iswift/main.go b/iswift/main.go index f26cb947..61aba731 100644 --- a/iswift/main.go +++ b/iswift/main.go @@ -1,6 +1,12 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 +// Program iswift provides a command-line wrapper around package iswiftpkg APIs. +// +// The program requires a single argument that is a path to a package config +// formatted configuration to load. Optionally, overrides the the config may +// be passed as additional arguments in the form .=. +// package main import ( From 16a25531eb3233bf9ba279668ff175a9cfbd779e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 19 May 2021 11:13:15 -0700 Subject: [PATCH 004/258] implemented imgrpkg/renewMount() --- icert/README.md | 2 +- imgr/imgrpkg/globals.go | 2 +- imgr/imgrpkg/retry-rpc.go | 39 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/icert/README.md b/icert/README.md index be29099b..b6a3b06b 100644 --- a/icert/README.md +++ b/icert/README.md @@ -48,7 +48,7 @@ A `-ttl` must be specified. Both `-caCert` and `-caKey` must be specified. If `-ca` is specified: -* neither `-cert` nor `key` may be specified +* neither `-cert` nor `-key` may be specified * neither `-dns` nor `-ip` may be specified If `-ca` is not specified: diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 56209bf8..0bc34f57 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -134,7 +134,7 @@ type mountStruct struct { authTokenExpired bool // if true, authToken has been rejected... needing a renewMount() to update authToken string // lastAuthTime time.Time // used to periodically check TTL of authToken - listElement *list.Element // LRU element on either volumeStruct.{healthy|expired}MountList + listElement *list.Element // LRU element on either volumeStruct.{healthy|leasesExpired|authTokenExpired}MountList } type inodeTableLayoutElementStruct struct { diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 4192bb26..9b6c5f94 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -176,7 +176,10 @@ retryGenerateMountID: func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse *RenewMountResponseStruct) (err error) { var ( + mount *mountStruct + ok bool startTime time.Time + volume *volumeStruct ) startTime = time.Now() @@ -185,7 +188,41 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * globals.stats.RenewMountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf(ETODO + " renewMount") + globals.RLock() + + mount, ok = globals.mountMap[renewMountRequest.MountID] + if !ok { + globals.RUnlock() + err = fmt.Errorf("%s %s", EUnknownMountID, renewMountRequest.MountID) + return + } + + volume = mount.volume + + volume.Lock() + globals.RUnlock() + + mount.authToken = renewMountRequest.AuthToken + + _, err = swiftObjectGet(mount.volume.storageURL, mount.authToken, ilayout.CheckPointObjectNumber) + if nil == err { + if mount.leasesExpired { + volume.leasesExpiredMountList.MoveToBack(mount.listElement) + } else { + if mount.authTokenExpired { + _ = volume.authTokenExpiredMountList.Remove(mount.listElement) + mount.listElement = volume.healthyMountList.PushBack(mount) + } else { + volume.healthyMountList.MoveToBack(mount.listElement) + } + } + } else { + err = fmt.Errorf("%s %s", EAuthTokenRejected, renewMountRequest.AuthToken) + } + + volume.Unlock() + + return } func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountResponseStruct) (err error) { From 0f2f574b30c985f9a305ba794fdc6a83ef650be2 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 19 May 2021 11:50:47 -0700 Subject: [PATCH 005/258] implemented imgrpkg/fetchNonceRange() --- imgr/imgrpkg/retry-rpc.go | 45 +++++++++++++++++++++++++++++----- imgr/imgrpkg/retry-rpc_test.go | 14 +++++------ 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 9b6c5f94..be15435f 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -7,6 +7,7 @@ import ( "crypto/tls" "encoding/base64" "fmt" + "strings" "time" "github.com/NVIDIA/sortedmap" @@ -241,10 +242,12 @@ func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountRespo func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeResponse *FetchNonceRangeResponseStruct) (err error) { var ( - mount *mountStruct - ok bool - startTime time.Time - volume *volumeStruct + mount *mountStruct + nonceUpdatedCheckPoint *ilayout.CheckPointV1Struct + nonceUpdatedCheckPointAsString string + ok bool + startTime time.Time + volume *volumeStruct ) startTime = time.Now() @@ -267,11 +270,41 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch volume.Lock() globals.RUnlock() - fmt.Printf("TODO: Perform fetchNonceRange() for mountStruct @ %p\n", mount) + if mount.authTokenHasExpired() { + volume.Unlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + return + } + + nonceUpdatedCheckPoint = &ilayout.CheckPointV1Struct{} + *nonceUpdatedCheckPoint = *volume.checkPoint + + nonceUpdatedCheckPoint.ReservedToNonce += globals.config.FetchNonceRangeToReturn + + fetchNonceRangeResponse.NextNonce = volume.checkPoint.ReservedToNonce + 1 + fetchNonceRangeResponse.NumNoncesFetched = globals.config.FetchNonceRangeToReturn + + nonceUpdatedCheckPointAsString, err = nonceUpdatedCheckPoint.MarshalCheckPointV1() + if nil != err { + logFatalf("nonceUpdatedCheckPoint.MarshalCheckPointV1() failed: %v", err) + } + + err = swiftObjectPut(volume.storageURL, mount.authToken, ilayout.CheckPointObjectNumber, strings.NewReader(nonceUpdatedCheckPointAsString)) + if nil == err { + if mount.leasesExpired { + volume.leasesExpiredMountList.MoveToBack(mount.listElement) + } else { + volume.healthyMountList.MoveToBack(mount.listElement) + } + + volume.checkPoint = nonceUpdatedCheckPoint + } else { + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + } volume.Unlock() - return fmt.Errorf(ETODO + " fetchNonceRange") + return } func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index de29e6c1..feb88c55 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -108,13 +108,6 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"Mount(,)\",,) failed: %v", err) } - // TODO: Remove this early exit skipping of following TODOs - - if nil == err { - t.Logf("Exiting TestRetryRPC() early to skip following TODOs") - return - } - // Perform a FetchNonceRange() fetchNonceRangeRequest = &FetchNonceRangeRequestStruct{ @@ -127,6 +120,13 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"FetchNonceRange()\",,) failed: %v", err) } + // TODO: Remove this early exit skipping of following TODOs + + if nil == err { + t.Logf("Exiting TestRetryRPC() early to skip following TODOs") + return + } + // Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ From e4a2424526c05bb0c4ceb08eaa9beca1e35db90f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 19 May 2021 11:56:59 -0700 Subject: [PATCH 006/258] Fixed lint suggestion for assigning vars in var() block instead of 1st func statement in imgr/imgrpkg --- imgr/imgrpkg/retry-rpc.go | 44 ++++++++++----------------------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index be15435f..2d59cfdb 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -82,13 +82,11 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo mountIDAsByteArray []byte mountIDAsString string ok bool - startTime time.Time + startTime time.Time = time.Now() volume *volumeStruct volumeAsValue sortedmap.Value ) - startTime = time.Now() - defer func() { globals.stats.MountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -179,12 +177,10 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * var ( mount *mountStruct ok bool - startTime time.Time + startTime time.Time = time.Now() volume *volumeStruct ) - startTime = time.Now() - defer func() { globals.stats.RenewMountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -228,11 +224,9 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountResponseStruct) (err error) { var ( - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.UnmountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -246,12 +240,10 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch nonceUpdatedCheckPoint *ilayout.CheckPointV1Struct nonceUpdatedCheckPointAsString string ok bool - startTime time.Time + startTime time.Time = time.Now() volume *volumeStruct ) - startTime = time.Now() - defer func() { globals.stats.FetchNonceRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -309,11 +301,9 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { var ( - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -323,11 +313,9 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { var ( - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.PutInodeTableEntriesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -337,11 +325,9 @@ func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesReque func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { var ( - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -351,11 +337,9 @@ func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRe func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { var ( - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -368,12 +352,10 @@ func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) checkPointResponseChan chan error mount *mountStruct ok bool - startTime time.Time + startTime time.Time = time.Now() volume *volumeStruct ) - startTime = time.Now() - defer func() { globals.stats.FlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -411,11 +393,9 @@ func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { var ( - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.LeaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -454,15 +434,13 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) func (mount *mountStruct) authTokenHasExpired() (authTokenExpired bool) { var ( err error - startTime time.Time + startTime time.Time = time.Now() ) if mount.authTokenExpired { return true } - startTime = time.Now() - if startTime.Sub(mount.lastAuthTime) < globals.config.AuthTokenCheckInterval { return false } From db3007121539a4febbd6a257706c16c1b15bca32 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 19 May 2021 18:27:20 -0700 Subject: [PATCH 007/258] Integrated Lease Management into imgr/imgrpkg Lease count limits and expired leases triggering unmounts are still required --- imgr/imgrpkg/api.go | 1 + imgr/imgrpkg/globals.go | 95 ++- imgr/imgrpkg/lease.go | 1526 ++++++++++++++++++++++++++++++++++++ imgr/imgrpkg/lease_test.go | 593 ++++++++++++++ imgr/imgrpkg/retry-rpc.go | 115 ++- imgr/imgrpkg/utils_test.go | 2 +- imgr/imgrpkg/volume.go | 2 + 7 files changed, 2314 insertions(+), 20 deletions(-) create mode 100644 imgr/imgrpkg/lease.go create mode 100644 imgr/imgrpkg/lease_test.go diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index addb87f0..4eb333cf 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -152,6 +152,7 @@ func LogInfof(format string, args ...interface{}) { // const ( EAuthTokenRejected = "EAuthTokenRejected:" + ELeaseRequestDenied = "ELeaseRequestDenied:" EVolumeBeingDeleted = "EVolumeBeingDeleted:" EUnknownMountID = "EUnknownMountID:" EUnknownVolumeName = "EUnknownVolumeName:" diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 0bc34f57..81c45779 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -126,15 +126,90 @@ const ( mountIDByteArrayLen = 15 // actual mountID will be Base64-encoded form ) +type leaseRequestOperationStruct struct { + mount *mountStruct + inodeLease *inodeLeaseStruct + LeaseRequestType + replyChan chan LeaseResponseType +} + +type leaseRequestStateType uint32 + +const ( + leaseRequestStateNone leaseRequestStateType = iota + leaseRequestStateSharedRequested + leaseRequestStateSharedGranted + leaseRequestStateSharedPromoting + leaseRequestStateSharedReleasing + leaseRequestStateExclusiveRequested + leaseRequestStateExclusiveGranted + leaseRequestStateExclusiveDemoting + leaseRequestStateExclusiveReleasing +) + +type leaseRequestStruct struct { + mount *mountStruct + inodeLease *inodeLeaseStruct + requestState leaseRequestStateType + replyChan chan LeaseResponseType // copied from leaseRequestOperationStruct.replyChan for LeaseRequestType == LeaseRequestType{Shared|Promote|Exclusive} + listElement *list.Element // used when on one of inodeList.*List's +} + +type inodeLeaseStateType uint32 + +const ( + inodeLeaseStateNone inodeLeaseStateType = iota + inodeLeaseStateSharedGrantedRecently + inodeLeaseStateSharedGrantedLongAgo + inodeLeaseStateSharedPromoting + inodeLeaseStateSharedReleasing + inodeLeaseStateSharedExpired + inodeLeaseStateExclusiveGrantedRecently + inodeLeaseStateExclusiveGrantedLongAgo + inodeLeaseStateExclusiveDemoting + inodeLeaseStateExclusiveReleasing + inodeLeaseStateExclusiveExpired +) + +type inodeLeaseStruct struct { + volume *volumeStruct + inodeNumber uint64 + lruElement *list.Element // link into volumeStruct.inodeLeaseLRU + leaseState inodeLeaseStateType + + requestChan chan *leaseRequestOperationStruct + stopChan chan struct{} // closing this chan will trigger *inodeLeaseStruct.handler() to: + // revoke/reject all leaseRequestStruct's in *Holder* & requestedList + // issue volume.leaseHandlerWG.Done() + // and exit + + sharedHoldersList *list.List // each list.Element.Value.(*leaseRequestStruct).requestState == leaseRequestStateSharedGranted + promotingHolder *leaseRequestStruct // leaseRequest.requestState == leaseRequestStateSharedPromoting + exclusiveHolder *leaseRequestStruct // leaseRequest.requestState == leaseRequestStateExclusiveGranted + demotingHolder *leaseRequestStruct // leaseRequest.requestState == leaseRequestStateExclusiveDemoting + releasingHoldersList *list.List // each list.Element.Value.(*leaseRequestStruct).requestState == leaseRequestState{Shared|Exclusive}Releasing + requestedList *list.List // each list.Element.Value.(*leaseRequestStruct).requestState == leaseRequestState{Shared|Exclusive}Requested + + lastGrantTime time.Time // records the time at which the last exclusive or shared holder was set/added-to exclusiveHolder/sharedHoldersList + lastInterruptTime time.Time // records the time at which the last Interrupt was sent + interruptsSent uint32 + + longAgoTimer *time.Timer // if .C != nil, timing when to state transition from {Shared|Exclusive}LeaseGrantedRecently to {Shared|Exclusive}LeaseGrantedLogAgo + interruptTimer *time.Timer // if .C != nil, timing when to issue next Interrupt... or expire a Lease +} + type mountStruct struct { - // reentrancy covered by volumeStruct's sync.RWMutex - volume *volumeStruct // volume.{R|}Lock() also protects each mountStruct - mountID string // - leasesExpired bool // if true, leases are being expired prior to auto-deletion of mountStruct - authTokenExpired bool // if true, authToken has been rejected... needing a renewMount() to update - authToken string // - lastAuthTime time.Time // used to periodically check TTL of authToken - listElement *list.Element // LRU element on either volumeStruct.{healthy|leasesExpired|authTokenExpired}MountList + // reentrancy covered by volumeStruct's sync.RWMutex + volume *volumeStruct // volume.{R|}Lock() also protects each mountStruct + mountID string // + retryRPCClientID uint64 // + acceptingLeaseRequests bool // + leaseRequestMap map[uint64]*leaseRequestStruct // key == leaseRequestStruct.inodeLease.inodeNumber + leasesExpired bool // if true, leases are being expired prior to auto-deletion of mountStruct + authTokenExpired bool // if true, authToken has been rejected... needing a renewMount() to update + authToken string // + lastAuthTime time.Time // used to periodically check TTL of authToken + listElement *list.Element // LRU element on either volumeStruct.{healthy|leasesExpired|authTokenExpired}MountList } type inodeTableLayoutElementStruct struct { @@ -158,6 +233,10 @@ type volumeStruct struct { pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG + inodeLeaseMap map[uint64]*inodeLeaseStruct // key == inodeLeaseStruct.inodeNumber + inodeLeaseLRU *list.List // .Front() is the LRU inodeLeaseStruct.listElement + leaseHandlerWG sync.WaitGroup // .Add(1) each inodeLease insertion into inodeLeaseMap + // .Done() each inodeLease after it is removed from inodeLeaseMap } type globalsStruct struct { diff --git a/imgr/imgrpkg/lease.go b/imgr/imgrpkg/lease.go new file mode 100644 index 00000000..54bde926 --- /dev/null +++ b/imgr/imgrpkg/lease.go @@ -0,0 +1,1526 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "container/list" + "encoding/json" + "runtime" + "sync" + "time" +) + +func (inodeLease *inodeLeaseStruct) handler() { + var ( + leaseRequestOperation *leaseRequestOperationStruct + ) + + for { + select { + case leaseRequestOperation = <-inodeLease.requestChan: + inodeLease.handleOperation(leaseRequestOperation) + case _ = <-inodeLease.longAgoTimer.C: + inodeLease.handleLongAgoTimerPop() + case _ = <-inodeLease.interruptTimer.C: + inodeLease.handleInterruptTimerPop() + case _, _ = <-inodeLease.stopChan: + inodeLease.handleStopChanClose() // will not return + } + } +} + +func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *leaseRequestOperationStruct) { + var ( + err error + leaseRequest *leaseRequestStruct + leaseRequestElement *list.Element + ok bool + rpcInterrupt *RPCInterrupt + rpcInterruptBuf []byte + sharedHolderLeaseRequest *leaseRequestStruct + sharedHolderListElement *list.Element + ) + + inodeLease.volume.Lock() + defer inodeLease.volume.Unlock() + + inodeLease.volume.inodeLeaseLRU.MoveToBack(inodeLease.lruElement) + + switch leaseRequestOperation.LeaseRequestType { + case LeaseRequestTypeShared: + _, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequest = &leaseRequestStruct{ + mount: leaseRequestOperation.mount, + inodeLease: inodeLease, + requestState: leaseRequestStateSharedRequested, + replyChan: leaseRequestOperation.replyChan, + } + leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] = leaseRequest + switch inodeLease.leaseState { + case inodeLeaseStateNone: + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + leaseRequest.replyChan <- LeaseResponseTypeShared + case inodeLeaseStateSharedGrantedRecently: + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + inodeLease.lastGrantTime = time.Time{} + inodeLease.longAgoTimer = &time.Timer{} + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + leaseRequest.replyChan <- LeaseResponseTypeShared + case inodeLeaseStateSharedGrantedLongAgo: + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + leaseRequest.replyChan <- LeaseResponseTypeShared + case inodeLeaseStateSharedPromoting: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateSharedReleasing: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveGrantedLongAgo: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + inodeLease.leaseState = inodeLeaseStateExclusiveDemoting + inodeLease.demotingHolder = inodeLease.exclusiveHolder + inodeLease.demotingHolder.requestState = leaseRequestStateExclusiveDemoting + inodeLease.exclusiveHolder = nil + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeDemote, + InodeNumber: inodeLease.inodeNumber, + } + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) + } + globals.retryrpcServer.SendCallback(inodeLease.demotingHolder.mount.retryRPCClientID, rpcInterruptBuf) + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + case inodeLeaseStateExclusiveDemoting: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveReleasing: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) + } + } + case LeaseRequestTypePromote: + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + if leaseRequestStateSharedGranted == leaseRequest.requestState { + switch inodeLease.leaseState { + case inodeLeaseStateNone: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateNone") + case inodeLeaseStateSharedGrantedRecently: + if nil == inodeLease.promotingHolder { + _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + if 0 == inodeLease.sharedHoldersList.Len() { + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequestOperation.replyChan <- LeaseResponseTypePromoted + inodeLease.exclusiveHolder = leaseRequest + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { + inodeLease.promotingHolder = leaseRequest + leaseRequest.replyChan = leaseRequestOperation.replyChan + } + } else { // nil != inodeLease.promotingHolder + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedGrantedLongAgo: + _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + if 0 == inodeLease.sharedHoldersList.Len() { + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequestOperation.replyChan <- LeaseResponseTypePromoted + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { + inodeLease.leaseState = inodeLeaseStateSharedPromoting + inodeLease.promotingHolder = leaseRequest + leaseRequest.requestState = leaseRequestStateSharedPromoting + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) + } + for nil != inodeLease.sharedHoldersList.Front() { + leaseRequestElement = inodeLease.sharedHoldersList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.sharedHoldersList.Remove(leaseRequestElement) + leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedReleasing + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + } + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } + case inodeLeaseStateSharedPromoting: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedPromoting") + case inodeLeaseStateSharedReleasing: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedReleasing") + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveGrantedRecently") + case inodeLeaseStateExclusiveGrantedLongAgo: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveGrantedLongAgo") + case inodeLeaseStateExclusiveDemoting: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveDemoting") + case inodeLeaseStateExclusiveReleasing: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveReleasing") + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) + } + } else { // leaseRequestStateSharedGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case LeaseRequestTypeExclusive: + _, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequest = &leaseRequestStruct{ + mount: leaseRequestOperation.mount, + inodeLease: inodeLease, + requestState: leaseRequestStateExclusiveRequested, + replyChan: leaseRequestOperation.replyChan, + } + leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] = leaseRequest + switch inodeLease.leaseState { + case inodeLeaseStateNone: + leaseRequest.requestState = leaseRequestStateExclusiveGranted + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = leaseRequest + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + leaseRequest.replyChan <- LeaseResponseTypeExclusive + case inodeLeaseStateSharedGrantedRecently: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateSharedGrantedLongAgo: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + inodeLease.leaseState = inodeLeaseStateSharedReleasing + for nil != inodeLease.sharedHoldersList.Front() { + sharedHolderListElement = inodeLease.sharedHoldersList.Front() + sharedHolderLeaseRequest = sharedHolderListElement.Value.(*leaseRequestStruct) + _ = inodeLease.sharedHoldersList.Remove(sharedHolderListElement) + sharedHolderLeaseRequest.requestState = leaseRequestStateSharedReleasing + sharedHolderLeaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(sharedHolderLeaseRequest) + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) + } + globals.retryrpcServer.SendCallback(sharedHolderLeaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + } + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + case inodeLeaseStateSharedPromoting: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateSharedReleasing: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveGrantedLongAgo: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + inodeLease.leaseState = inodeLeaseStateExclusiveReleasing + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveReleasing + inodeLease.exclusiveHolder.listElement = inodeLease.releasingHoldersList.PushBack(inodeLease.exclusiveHolder) + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) + } + globals.retryrpcServer.SendCallback(inodeLease.exclusiveHolder.mount.retryRPCClientID, rpcInterruptBuf) + inodeLease.exclusiveHolder = nil + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + case inodeLeaseStateExclusiveDemoting: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveReleasing: + leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) + } + } + case LeaseRequestTypeDemote: + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + switch inodeLease.leaseState { + case inodeLeaseStateNone: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedGrantedRecently: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedGrantedLongAgo: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedPromoting: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedReleasing: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + if leaseRequestStateExclusiveGranted == leaseRequest.requestState { + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.exclusiveHolder = nil + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateSharedGranted + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + leaseRequestElement = nil + } + } + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveGranted == leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveGrantedLongAgo: + if leaseRequestStateExclusiveGranted == leaseRequest.requestState { + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.exclusiveHolder = nil + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveDemoting: + if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + inodeLease.demotingHolder = nil + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateSharedGranted + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateSharedRequested != leaseRequest.requestState + leaseRequestElement = nil + } + } + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveDemoting == leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveReleasing: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) + } + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case LeaseRequestTypeRelease: + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] + if ok { + switch inodeLease.leaseState { + case inodeLeaseStateNone: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + case inodeLeaseStateSharedGrantedRecently: + if leaseRequestStateSharedGranted == leaseRequest.requestState { + _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + if 0 == inodeLease.sharedHoldersList.Len() { + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + if nil == inodeLease.promotingHolder { + leaseRequestElement = inodeLease.requestedList.Front() + if nil == leaseRequestElement { + inodeLease.leaseState = inodeLeaseStateNone + } else { // nil != leaseRequestElement + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequestElement) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + if 0 == inodeLease.requestedList.Len() { + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequest.replyChan <- LeaseResponseTypeExclusive + } else { // 0 < inodeLease.requestedList.Len() + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState { + leaseRequestElement = nil + } + } + } + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequest.replyChan <- LeaseResponseTypeExclusive + } + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } + } else { // nil != inodeLease.promotingHolder + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = inodeLease.promotingHolder + inodeLease.promotingHolder = nil + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted + inodeLease.exclusiveHolder.replyChan <- LeaseResponseTypePromoted + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } + } + } else { // leaseRequestStateSharedGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedGrantedLongAgo: + if leaseRequestStateSharedGranted == leaseRequest.requestState { + _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + if 0 == inodeLease.sharedHoldersList.Len() { + inodeLease.leaseState = inodeLeaseStateNone + } + } else { // leaseRequestStateSharedGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedPromoting: + if leaseRequestStateSharedReleasing == leaseRequest.requestState { + _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + if 0 == inodeLease.releasingHoldersList.Len() { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + inodeLease.exclusiveHolder = inodeLease.promotingHolder + inodeLease.promotingHolder = nil + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted + inodeLease.exclusiveHolder.replyChan <- LeaseResponseTypePromoted + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } + } else { // leaseRequestStateSharedReleasing != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedReleasing: + if leaseRequestStateSharedReleasing == leaseRequest.requestState { + _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + if 0 == inodeLease.releasingHoldersList.Len() { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequestElement = inodeLease.requestedList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequest.replyChan <- LeaseResponseTypeExclusive + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } + } else { // leaseRequestStateSharedReleasing != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateSharedExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState inodeLeaseStateSharedExpired") + case inodeLeaseStateExclusiveGrantedRecently: + if leaseRequestStateExclusiveGranted == leaseRequest.requestState { + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + leaseRequestElement = inodeLease.requestedList.Front() + if nil == leaseRequestElement { + inodeLease.leaseState = inodeLeaseStateNone + inodeLease.exclusiveHolder = nil + inodeLease.lastGrantTime = time.Time{} + inodeLease.longAgoTimer = &time.Timer{} + } else { // nil != leaseRequestElement + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequestElement) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + _ = inodeLease.requestedList.Remove(leaseRequestElement) + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + leaseRequestElement = nil + } + } + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequest.requestState = leaseRequestStateExclusiveGranted + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.replyChan <- LeaseResponseTypeExclusive + } + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } + } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveGrantedLongAgo: + if leaseRequestStateExclusiveGranted == leaseRequest.requestState { + inodeLease.leaseState = inodeLeaseStateNone + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.exclusiveHolder = nil + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveDemoting: + if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.demotingHolder = nil + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + leaseRequestElement = inodeLease.requestedList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + if (nil == inodeLease.requestedList.Front()) || (leaseRequestStateExclusiveRequested == inodeLease.requestedList.Front().Value.(*leaseRequestStruct).requestState) { + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequest.requestState = leaseRequestStateExclusiveGranted + leaseRequest.listElement = nil + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.replyChan <- LeaseResponseTypeExclusive + } else { + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + _ = inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.replyChan <- LeaseResponseTypeShared + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + leaseRequestElement = nil + } + } + } + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveDemoting != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveReleasing: + if leaseRequestStateExclusiveReleasing == leaseRequest.requestState { + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + inodeLease.interruptTimer = &time.Timer{} + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + leaseRequestElement = inodeLease.requestedList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + leaseRequest.requestState = leaseRequestStateExclusiveGranted + _ = inodeLease.requestedList.Remove(leaseRequestElement) + leaseRequest.listElement = nil + inodeLease.exclusiveHolder = leaseRequest + leaseRequest.replyChan <- LeaseResponseTypeExclusive + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + } else { // leaseRequestStateExclusiveReleasing != leaseRequest.requestState + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveExpired: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState inodeLeaseStateExclusiveExpired") + default: + logFatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) + } + } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.inodeNumber] returned !ok + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + default: + logFatalf("(*inodeLeaseStruct).handleOperation() found unexpected leaseRequestOperation.LeaseRequestType: %v", leaseRequestOperation.LeaseRequestType) + } +} + +func (inodeLease *inodeLeaseStruct) handleLongAgoTimerPop() { + var ( + err error + leaseRequest *leaseRequestStruct + leaseRequestElement *list.Element + rpcInterrupt *RPCInterrupt + rpcInterruptBuf []byte + ) + + inodeLease.volume.Lock() + + inodeLease.lastGrantTime = time.Time{} + inodeLease.longAgoTimer = &time.Timer{} + + switch inodeLease.leaseState { + case inodeLeaseStateSharedGrantedRecently: + inodeLease.leaseState = inodeLeaseStateSharedGrantedLongAgo + + if (nil != inodeLease.promotingHolder) || (0 != inodeLease.requestedList.Len()) { + if nil != inodeLease.promotingHolder { + inodeLease.leaseState = inodeLeaseStateSharedPromoting + inodeLease.promotingHolder.requestState = leaseRequestStateSharedPromoting + } else { + inodeLease.leaseState = inodeLeaseStateSharedReleasing + } + + leaseRequestElement = inodeLease.sharedHoldersList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + leaseRequest.requestState = leaseRequestStateSharedReleasing + _ = inodeLease.sharedHoldersList.Remove(leaseRequestElement) + leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + leaseRequestElement = inodeLease.sharedHoldersList.Front() + } + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } + case inodeLeaseStateExclusiveGrantedRecently: + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedLongAgo + + leaseRequestElement = inodeLease.requestedList.Front() + if nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + switch leaseRequest.requestState { + case leaseRequestStateSharedRequested: + inodeLease.leaseState = inodeLeaseStateExclusiveDemoting + + inodeLease.demotingHolder = inodeLease.exclusiveHolder + inodeLease.demotingHolder.requestState = leaseRequestStateExclusiveDemoting + inodeLease.exclusiveHolder = nil + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeDemote, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(inodeLease.demotingHolder.mount.retryRPCClientID, rpcInterruptBuf) + case leaseRequestStateExclusiveRequested: + inodeLease.leaseState = inodeLeaseStateExclusiveReleasing + + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveReleasing + inodeLease.exclusiveHolder.listElement = inodeLease.releasingHoldersList.PushBack(inodeLease.exclusiveHolder) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(inodeLease.exclusiveHolder.mount.retryRPCClientID, rpcInterruptBuf) + + inodeLease.exclusiveHolder = nil + default: + logFatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() found requestedList with unexpected leaseRequest.requestState: %v", leaseRequest.requestState) + } + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } + default: + logFatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() called while in wrong state (%v)", inodeLease.leaseState) + } + + inodeLease.volume.Unlock() +} + +func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { + var ( + err error + leaseRequest *leaseRequestStruct + leaseRequestElement *list.Element + rpcInterrupt *RPCInterrupt + rpcInterruptBuf []byte + ) + + inodeLease.volume.Lock() + + if globals.config.LeaseInterruptLimit <= inodeLease.interruptsSent { + switch inodeLease.leaseState { + case inodeLeaseStateSharedPromoting: + inodeLease.leaseState = inodeLeaseStateSharedExpired + + leaseRequestElement = inodeLease.releasingHoldersList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 1]") + } + + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + + leaseRequest.mount.acceptingLeaseRequests = false + + inodeLease.releasingHoldersList.Remove(leaseRequestElement) + leaseRequest.listElement = nil + + leaseRequestElement = inodeLease.releasingHoldersList.Front() + } + + inodeLease.exclusiveHolder = inodeLease.promotingHolder + inodeLease.promotingHolder = nil + + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted + + inodeLease.exclusiveHolder.replyChan <- LeaseResponseTypeExclusive + + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + case inodeLeaseStateSharedReleasing: + inodeLease.leaseState = inodeLeaseStateSharedExpired + + leaseRequestElement = inodeLease.releasingHoldersList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 2]") + } + + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + + leaseRequest.mount.acceptingLeaseRequests = false + + inodeLease.releasingHoldersList.Remove(leaseRequestElement) + leaseRequest.listElement = nil + + leaseRequestElement = inodeLease.releasingHoldersList.Front() + } + + leaseRequestElement = inodeLease.requestedList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty requestedList [case 1]") + } + + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateExclusiveRequested != leaseRequest.requestState { + logFatalf("(inodeLeaseStruct).handleInterruptTimerPop() found unexpected requestedList.Front().requestState: %v [case 1]", leaseRequest.requestState) + } + + inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + inodeLease.exclusiveHolder = leaseRequest + + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted + + inodeLease.exclusiveHolder.replyChan <- LeaseResponseTypeExclusive + + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + case inodeLeaseStateExclusiveReleasing: + inodeLease.leaseState = inodeLeaseStateExclusiveExpired + + leaseRequestElement = inodeLease.releasingHoldersList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 3]") + } + + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + + leaseRequest.mount.acceptingLeaseRequests = false + + inodeLease.releasingHoldersList.Remove(leaseRequestElement) + leaseRequest.listElement = nil + + if nil != inodeLease.releasingHoldersList.Front() { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found releasingHoldersList unexpectedly with >1 leaseRequestElements") + } + + leaseRequestElement = inodeLease.requestedList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty requestedList [case 2]") + } + + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateExclusiveRequested != leaseRequest.requestState { + logFatalf("(inodeLeaseStruct).handleInterruptTimerPop() found unexpected requestedList.Front().requestState: %v [case 2]", leaseRequest.requestState) + } + + inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + inodeLease.exclusiveHolder = leaseRequest + + inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted + + inodeLease.exclusiveHolder.replyChan <- LeaseResponseTypeExclusive + + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently + case inodeLeaseStateExclusiveDemoting: + inodeLease.leaseState = inodeLeaseStateExclusiveExpired + + if nil == inodeLease.demotingHolder { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty demotingHolder [case 1]") + } + + delete(inodeLease.demotingHolder.mount.leaseRequestMap, inodeLease.inodeNumber) + + inodeLease.demotingHolder.mount.acceptingLeaseRequests = false + + inodeLease.demotingHolder = nil + + leaseRequestElement = inodeLease.requestedList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty requestedList [case 3]") + } + + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested != leaseRequest.requestState { + logFatalf("(inodeLeaseStruct).handleInterruptTimerPop() found unexpected requestedList.Front().requestState: %v [case 3]", leaseRequest.requestState) + } + + for { + inodeLease.requestedList.Remove(leaseRequestElement) + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + + leaseRequest.requestState = leaseRequestStateSharedGranted + + leaseRequest.replyChan <- LeaseResponseTypeShared + + leaseRequestElement = inodeLease.requestedList.Front() + if nil == leaseRequestElement { + break + } + + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateExclusiveRequested == leaseRequest.requestState { + break + } + } + + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + default: + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found unexpected leaseState: %v [case 1]", inodeLease.leaseState) + } + + inodeLease.lastGrantTime = time.Now() + inodeLease.longAgoTimer = time.NewTimer(globals.config.MinLeaseDuration) + + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + + inodeLease.interruptTimer = &time.Timer{} + } else { // globals.config.LeaseInterruptLimit > inodeLease.interruptsSent + switch inodeLease.leaseState { + case inodeLeaseStateSharedPromoting: + leaseRequestElement = inodeLease.releasingHoldersList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 4]") + } + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + leaseRequestElement = leaseRequestElement.Next() + } + case inodeLeaseStateSharedReleasing: + leaseRequestElement = inodeLease.releasingHoldersList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 5]") + } + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + leaseRequestElement = leaseRequestElement.Next() + } + case inodeLeaseStateExclusiveReleasing: + leaseRequestElement = inodeLease.releasingHoldersList.Front() + if nil == leaseRequestElement { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 6]") + } + + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + case inodeLeaseStateExclusiveDemoting: + if nil == inodeLease.demotingHolder { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty demotingHolder [case 2]") + } + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeDemote, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(inodeLease.demotingHolder.mount.retryRPCClientID, rpcInterruptBuf) + default: + logFatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found unexpected leaseState: %v [case 2]", inodeLease.leaseState) + } + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent++ + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } + + inodeLease.volume.Unlock() +} + +func (inodeLease *inodeLeaseStruct) handleStopChanClose() { + var ( + err error + leaseRequest *leaseRequestStruct + leaseRequestElement *list.Element + leaseRequestOperation *leaseRequestOperationStruct + ok bool + rpcInterrupt *RPCInterrupt + rpcInterruptBuf []byte + ) + + // Deny all pending requests: + + inodeLease.volume.Lock() + + for nil != inodeLease.requestedList.Front() { + leaseRequestElement = inodeLease.requestedList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + inodeLease.requestedList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + leaseRequest.replyChan <- LeaseResponseTypeDenied + } + + // If inodeLease.leaseState is inodeLeaseStateSharedPromoting: + // Reject inodeLease.promotingHolder's LeaseRequestTypePromote + // Ensure formerly inodeLease.promotingHolder is also now releasing + + if inodeLeaseStateSharedPromoting == inodeLease.leaseState { + leaseRequest = inodeLease.promotingHolder + inodeLease.promotingHolder = nil + + leaseRequest.replyChan <- LeaseResponseTypeDenied + + leaseRequest.requestState = leaseRequestStateSharedReleasing + + leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + inodeLease.leaseState = inodeLeaseStateSharedReleasing + } + + // Ensure that inodeLease.leaseState is not inodeLeaseState{Shared|Exclusive}GrantedRecently + + switch inodeLease.leaseState { + case inodeLeaseStateSharedGrantedRecently: + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + inodeLease.lastGrantTime = time.Time{} + inodeLease.longAgoTimer = &time.Timer{} + + inodeLease.leaseState = inodeLeaseStateSharedGrantedLongAgo + case inodeLeaseStateExclusiveGrantedRecently: + if !inodeLease.longAgoTimer.Stop() { + <-inodeLease.longAgoTimer.C + } + inodeLease.lastGrantTime = time.Time{} + inodeLease.longAgoTimer = &time.Timer{} + + inodeLease.leaseState = inodeLeaseStateExclusiveGrantedLongAgo + default: + // Nothing to do here + } + + // If necessary, transition inodeLease.leaseState from inodeLeaseState{Shared|Exclusive}GrantedLongAgo + // to inodeLeaseState{Shared|Exclusive}Releasing + + switch inodeLease.leaseState { + case inodeLeaseStateSharedGrantedLongAgo: + for nil != inodeLease.sharedHoldersList.Front() { + leaseRequestElement = inodeLease.sharedHoldersList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + inodeLease.sharedHoldersList.Remove(leaseRequestElement) + + leaseRequest.requestState = leaseRequestStateSharedReleasing + + leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + } + + inodeLease.leaseState = inodeLeaseStateSharedReleasing + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + case inodeLeaseStateExclusiveGrantedLongAgo: + leaseRequest = inodeLease.exclusiveHolder + inodeLease.exclusiveHolder = nil + + leaseRequest.requestState = leaseRequestStateExclusiveReleasing + + leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + inodeLease.leaseState = inodeLeaseStateExclusiveReleasing + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + default: + // Nothing to do here + } + + // Loop until inodeLease.leaseState is inodeLeaseStateNone + + for inodeLeaseStateNone != inodeLease.leaseState { + inodeLease.volume.Unlock() + + select { + case leaseRequestOperation = <-inodeLease.requestChan: + inodeLease.volume.Lock() + + switch leaseRequestOperation.LeaseRequestType { + case LeaseRequestTypeShared: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + + case LeaseRequestTypePromote: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + + case LeaseRequestTypeExclusive: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + + case LeaseRequestTypeDemote: + if inodeLeaseStateExclusiveDemoting == inodeLease.leaseState { + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[leaseRequestOperation.inodeLease.inodeNumber] + if ok { + if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { + if leaseRequest == inodeLease.demotingHolder { + leaseRequestOperation.replyChan <- LeaseResponseTypeDemoted + + inodeLease.demotingHolder = nil + + leaseRequest.requestState = leaseRequestStateExclusiveReleasing + + leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + inodeLease.leaseState = inodeLeaseStateExclusiveReleasing + + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent = 1 + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + + case LeaseRequestTypeRelease: + leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[leaseRequestOperation.inodeLease.inodeNumber] + if ok { + switch inodeLease.leaseState { + case inodeLeaseStateSharedReleasing: + if leaseRequestStateSharedReleasing == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + + if 0 == inodeLease.releasingHoldersList.Len() { + inodeLease.leaseState = inodeLeaseStateNone + + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + + inodeLease.interruptTimer = &time.Timer{} + } + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveDemoting: + if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.demotingHolder = nil + + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + + inodeLease.leaseState = inodeLeaseStateNone + + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + + inodeLease.interruptTimer = &time.Timer{} + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + case inodeLeaseStateExclusiveReleasing: + if leaseRequestStateExclusiveReleasing == leaseRequest.requestState { + leaseRequest.requestState = leaseRequestStateNone + delete(leaseRequest.mount.leaseRequestMap, inodeLease.inodeNumber) + inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) + leaseRequest.listElement = nil + + leaseRequestOperation.replyChan <- LeaseResponseTypeReleased + + inodeLease.leaseState = inodeLeaseStateNone + + if !inodeLease.interruptTimer.Stop() { + <-inodeLease.interruptTimer.C + } + + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + + inodeLease.interruptTimer = &time.Timer{} + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + default: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + } else { + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + } + + default: + logFatalf("(*inodeLeaseStruct).handleStopChanClose() read unexected leaseRequestOperationLeaseRequestType: %v", leaseRequestOperation.LeaseRequestType) + } + + case _ = <-inodeLease.interruptTimer.C: + inodeLease.volume.Lock() + + switch inodeLease.leaseState { + case inodeLeaseStateSharedGrantedLongAgo: + logFatalf("(*inodeLeaseStruct).handleStopChanClose() hit an interruptTimer pop while unexpectedly in inodeLeaseStateSharedGrantedLongAgo") + case inodeLeaseStateSharedReleasing: + if globals.config.LeaseInterruptLimit <= inodeLease.interruptsSent { + inodeLease.leaseState = inodeLeaseStateSharedExpired + + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + + inodeLease.interruptTimer = &time.Timer{} + + for nil != inodeLease.releasingHoldersList.Front() { + leaseRequestElement = inodeLease.releasingHoldersList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + inodeLease.releasingHoldersList.Remove(leaseRequestElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + } + + inodeLease.leaseState = inodeLeaseStateNone + } else { // globals.config.LeaseInterruptLimit > inodeLease.interruptsSent { + leaseRequestElement = inodeLease.releasingHoldersList.Front() + + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 5]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + leaseRequestElement = leaseRequestElement.Next() + } + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent++ + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } + case inodeLeaseStateExclusiveGrantedLongAgo: + logFatalf("(*inodeLeaseStruct).handleStopChanClose() hit an interruptTimer pop while unexpectedly in inodeLeaseStateExclusiveGrantedLongAgo") + case inodeLeaseStateExclusiveDemoting: + if globals.config.LeaseInterruptLimit <= inodeLease.interruptsSent { + inodeLease.leaseState = inodeLeaseStateExclusiveExpired + + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + + inodeLease.interruptTimer = &time.Timer{} + + leaseRequest = inodeLease.demotingHolder + inodeLease.demotingHolder = nil + leaseRequest.requestState = leaseRequestStateNone + + inodeLease.leaseState = inodeLeaseStateNone + } else { // globals.config.LeaseInterruptLimit > inodeLease.interruptsSent { + leaseRequest = inodeLease.demotingHolder + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 6]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent++ + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } + case inodeLeaseStateExclusiveReleasing: + if globals.config.LeaseInterruptLimit <= inodeLease.interruptsSent { + inodeLease.leaseState = inodeLeaseStateExclusiveExpired + + inodeLease.lastInterruptTime = time.Time{} + inodeLease.interruptsSent = 0 + + inodeLease.interruptTimer = &time.Timer{} + + leaseRequestElement = inodeLease.releasingHoldersList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + inodeLease.releasingHoldersList.Remove(leaseRequestElement) + leaseRequest.listElement = nil + leaseRequest.requestState = leaseRequestStateNone + + inodeLease.leaseState = inodeLeaseStateNone + } else { // globals.config.LeaseInterruptLimit > inodeLease.interruptsSent { + leaseRequestElement = inodeLease.releasingHoldersList.Front() + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + + rpcInterrupt = &RPCInterrupt{ + RPCInterruptType: RPCInterruptTypeRelease, + InodeNumber: inodeLease.inodeNumber, + } + + rpcInterruptBuf, err = json.Marshal(rpcInterrupt) + if nil != err { + logFatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 7]", rpcInterrupt, err) + } + + globals.retryrpcServer.SendCallback(leaseRequest.mount.retryRPCClientID, rpcInterruptBuf) + + inodeLease.lastInterruptTime = time.Now() + inodeLease.interruptsSent++ + + inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) + } + default: + logFatalf("(*inodeLeaseStruct).handleStopChanClose() hit an interruptTimer pop while unexpectedly in unknown inodeLease.leaseState: %v", inodeLease.leaseState) + } + } + } + + // Drain requestChan before exiting + + for { + select { + case leaseRequestOperation = <-inodeLease.requestChan: + leaseRequestOperation.replyChan <- LeaseResponseTypeDenied + default: + goto RequestChanDrained + } + } + +RequestChanDrained: + + delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) + _ = inodeLease.volume.inodeLeaseLRU.Remove(inodeLease.lruElement) + + inodeLease.volume.leaseHandlerWG.Done() + + inodeLease.volume.Unlock() + + runtime.Goexit() +} + +// armReleaseOfAllLeasesWhileLocked is called to schedule releasing of all held leases +// for a specific mountStruct. It is called while inodeLease.volume is locked. The +// leaseReleaseStartWG is assumed to be a sync.WaitGroup with a count of 1 such that +// the actual releasing of each lease will occur once leaseReleaseStartWG is signaled +// by calling Done() once. The caller would presumably do this after having released +// inodeLease.volume and then await completion by calling leaseReleaseFinishedWG.Wait(). +// +func (mount *mountStruct) armReleaseOfAllLeasesWhileLocked(leaseReleaseStartWG *sync.WaitGroup, leaseReleaseFinishedWG *sync.WaitGroup) { + var ( + leaseRequest *leaseRequestStruct + leaseRequestOperation *leaseRequestOperationStruct + ) + + for _, leaseRequest = range mount.leaseRequestMap { + if nil != leaseRequest.inodeLease { + leaseRequestOperation = &leaseRequestOperationStruct{ + mount: mount, + inodeLease: leaseRequest.inodeLease, + LeaseRequestType: LeaseRequestTypeRelease, + replyChan: make(chan LeaseResponseType), + } + + leaseReleaseFinishedWG.Add(1) + + go func(leaseRequestOperation *leaseRequestOperationStruct) { + leaseReleaseStartWG.Wait() + leaseRequestOperation.inodeLease.requestChan <- leaseRequestOperation + _ = <-leaseRequestOperation.replyChan + leaseReleaseFinishedWG.Done() + }(leaseRequestOperation) + } + } +} diff --git a/imgr/imgrpkg/lease_test.go b/imgr/imgrpkg/lease_test.go new file mode 100644 index 00000000..f81a248e --- /dev/null +++ b/imgr/imgrpkg/lease_test.go @@ -0,0 +1,593 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "encoding/json" + "fmt" + "os" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/NVIDIA/proxyfs/retryrpc" +) + +const ( + testRpcLeaseDelayAfterSendingRequest = 10 * time.Millisecond + testRpcLeaseDelayBeforeSendingRequest = 10 * time.Millisecond + testRpcLeaseMultiFirstInodeNumber uint64 = 1 + testRpcLeaseMultiNumInstances uint64 = 5 + testRpcLeaseSingleInodeNumber uint64 = 1 + testRpcLeaseSingleNumInstances uint64 = 101 // Must be >= 4 + testRpcLeaseTimeFormat = "15:04:05.000" +) + +var ( + testRpcLeaseRequestLetters = [5]string{"S", "P", "E", "D", "R"} + testRpcLeaseResponseLetters = [6]string{"D", "S", "P", "E", "D", "R"} + testRpcLeaseInterruptLetters = [3]string{"U", "D", "R"} + testRpcLeaseLogVerbosely bool +) + +type testRpcLeaseClientStruct struct { + instance uint64 + inodeNumber uint64 + chIn chan LeaseRequestType // close it to terminate testRpcLeaseClient instance + chOut chan interface{} // either a LeaseResponseType or an RPCInterruptType + alreadyUnmounted bool // if true, no RpcUnmount will be issued + wg *sync.WaitGroup // signaled when testRpcLeaseClient instance exits + t *testing.T +} + +func TestRPCLease(t *testing.T) { + var ( + err error + instance uint64 + postRequestBody string + putRequestBody string + testRpcLeaseClient []*testRpcLeaseClientStruct + wg sync.WaitGroup + ) + + // Setup test environment + + testSetup(t, nil) + + // Format testVolume + + postRequestBody = fmt.Sprintf("{\"StorageURL\":\"%s\",\"AuthToken\":\"%s\"}", testGlobals.containerURL, testGlobals.authToken) + + _, _, err = testDoHTTPRequest("POST", testGlobals.httpServerURL+"/volume", nil, strings.NewReader(postRequestBody)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"POST\", testGlobals.httpServerURL+\"/volume\", nil, strings.NewReader(postRequestBody)) failed: %v", err) + } + + // Start serving testVolume + + putRequestBody = fmt.Sprintf("{\"StorageURL\":\"%s\"}", testGlobals.containerURL) + + _, _, err = testDoHTTPRequest("PUT", testGlobals.httpServerURL+"/volume/"+testVolume, nil, strings.NewReader(putRequestBody)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) failed: %v", err) + } + + // Setup Single Lease instances + + wg.Add(int(testRpcLeaseSingleNumInstances)) + + testRpcLeaseClient = make([]*testRpcLeaseClientStruct, testRpcLeaseSingleNumInstances) + + for instance = 0; instance < testRpcLeaseSingleNumInstances; instance++ { + testRpcLeaseClient[instance] = &testRpcLeaseClientStruct{ + instance: instance, + inodeNumber: testRpcLeaseSingleInodeNumber, + chIn: make(chan LeaseRequestType), + chOut: make(chan interface{}), + alreadyUnmounted: false, + wg: &wg, + t: t, + } + + go testRpcLeaseClient[instance].instanceGoroutine() + } + + // Perform Single Lease test cases + + testRpcLeaseLogTestCase("1 Shared", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("2 Shared", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("3 Shared", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("1 Exclusive", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("1 Exclusive then Demote", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeDemoted) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("1 Exclusive then 1 Shared leading to Demotion", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeDemoted, RPCInterruptTypeDemote) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("1 Shared then 1 Exclusive leading to Release", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("1 Exclusive then 1 Exclusive leading to Release", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("2 Shared then Promotion leading to Release", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypePromote) + testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypePromoted) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("1 Exclusive then 2 Shared leading to Demotion", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeDemoted, RPCInterruptTypeDemote) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("2 Shared then 1 Exclusive leading to Release", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase("2 Exclusives leading to Release that Expires", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + // TODO: testRpcLeaseClient[0].alreadyUnmounted = true + // + // The above expiration should ultimately trigger an implicit Unmount + + testRpcLeaseLogTestCase("2 Shared then 2 Promotions leading to Release", true) + + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypePromote) + testRpcLeaseClient[2].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypePromote) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeDenied, RPCInterruptTypeRelease) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypePromoted) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + testRpcLeaseLogTestCase(fmt.Sprintf("%v Shared", testRpcLeaseSingleNumInstances-1), false) + + for instance = 1; instance < testRpcLeaseSingleNumInstances; instance++ { + testRpcLeaseClient[instance].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[instance].validateChOutValueIsLeaseResponseType(LeaseResponseTypeShared) + } + for instance = 1; instance < testRpcLeaseSingleNumInstances; instance++ { + testRpcLeaseClient[instance].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[instance].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + } + + testRpcLeaseLogTestCase(fmt.Sprintf("%v Exclusives", testRpcLeaseSingleNumInstances-1), false) + + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + for instance = 2; instance < testRpcLeaseSingleNumInstances; instance++ { + testRpcLeaseClient[instance].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[(instance - 1)].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + testRpcLeaseClient[(instance - 1)].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[(instance-1)].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[instance].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + } + testRpcLeaseClient[(testRpcLeaseSingleNumInstances - 1)].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[(testRpcLeaseSingleNumInstances - 1)].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + // Shutdown Single Lease instances + + for instance = 0; instance < testRpcLeaseSingleNumInstances; instance++ { + close(testRpcLeaseClient[instance].chIn) + } + + wg.Wait() + + // Setup Multi Lease instances + + wg.Add(int(testRpcLeaseMultiNumInstances)) + + testRpcLeaseClient = make([]*testRpcLeaseClientStruct, testRpcLeaseMultiNumInstances) + + for instance = 0; instance < testRpcLeaseMultiNumInstances; instance++ { + testRpcLeaseClient[instance] = &testRpcLeaseClientStruct{ + instance: instance, + inodeNumber: testRpcLeaseMultiFirstInodeNumber + instance, + chIn: make(chan LeaseRequestType), + chOut: make(chan interface{}), + alreadyUnmounted: false, + wg: &wg, + t: t, + } + + go testRpcLeaseClient[instance].instanceGoroutine() + } + + // Perform Multi Lease test case + + testRpcLeaseLogTestCase(fmt.Sprintf("%v Unique InodeNumber Exclusives", testRpcLeaseMultiNumInstances), true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeExclusive) + + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + + testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeExclusive) + + // TODO testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + // TODO testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) + // + // Above must be re-enabled once Lease Limits are enforced + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + + testRpcLeaseClient[0].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(LeaseResponseTypeReleased, RPCInterruptTypeRelease) + + testRpcLeaseClient[3].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + + testRpcLeaseClient[4].sendLeaseRequest(LeaseRequestTypeExclusive) + + testRpcLeaseClient[4].validateChOutValueIsLeaseResponseType(LeaseResponseTypeExclusive) + + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[4].sendLeaseRequest(LeaseRequestTypeRelease) + + testRpcLeaseClient[2].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[3].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + testRpcLeaseClient[4].validateChOutValueIsLeaseResponseType(LeaseResponseTypeReleased) + + // Shutdown Multi Lease instances + + for instance = 0; instance < testRpcLeaseMultiNumInstances; instance++ { + close(testRpcLeaseClient[instance].chIn) + } + + wg.Wait() + + // Teardown test environment + + testTeardown(t) +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { + var ( + err error + leaseRequest *LeaseRequestStruct + leaseRequestType LeaseRequestType + leaseResponse *LeaseResponseStruct + mountRequest *MountRequestStruct + mountResponse *MountResponseStruct + ok bool + retryrpcClient *retryrpc.Client + retryrpcClientConfig *retryrpc.ClientConfig + unmountRequest *UnmountRequestStruct + unmountResponse *UnmountResponseStruct + ) + + retryrpcClientConfig = &retryrpc.ClientConfig{} + *retryrpcClientConfig = *testGlobals.retryrpcClientConfig + retryrpcClientConfig.Callbacks = testRpcLeaseClient + + retryrpcClient, err = retryrpc.NewClient(retryrpcClientConfig) + if nil != err { + testRpcLeaseClient.Fatalf("retryrpc.NewClient() failed: %v", err) + } + + mountRequest = &MountRequestStruct{ + VolumeName: testVolume, + AuthToken: testGlobals.authToken, + } + mountResponse = &MountResponseStruct{} + + err = retryrpcClient.Send("Mount", mountRequest, mountResponse) + if nil != err { + testRpcLeaseClient.Fatalf("retryrpcClient.Send(\"Mount\",,) failed: %v", err) + } + + for { + leaseRequestType, ok = <-testRpcLeaseClient.chIn + + if ok { + leaseRequest = &LeaseRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: testRpcLeaseClient.inodeNumber, + LeaseRequestType: leaseRequestType, + } + leaseResponse = &LeaseResponseStruct{} + + testRpcLeaseClient.logEvent(leaseRequest.LeaseRequestType) + + err = retryrpcClient.Send("Lease", leaseRequest, leaseResponse) + if nil != err { + testRpcLeaseClient.Fatalf("retryrpcClient.Send(\"Lease\",LeaseRequestType=%d) failed: %v", leaseRequestType, err) + } + + testRpcLeaseClient.logEvent(leaseResponse.LeaseResponseType) + + testRpcLeaseClient.chOut <- leaseResponse.LeaseResponseType + } else { + unmountRequest = &UnmountRequestStruct{ + MountID: mountResponse.MountID, + } + unmountResponse = &UnmountResponseStruct{} + + err = retryrpcClient.Send("Unmount", unmountRequest, unmountResponse) + if testRpcLeaseClient.alreadyUnmounted { + if nil == err { + testRpcLeaseClient.Fatalf("retryrpcClient.Send(\"Unmount\",,) should have failed") + } + } else { + if nil != err { + if !strings.HasPrefix(err.Error(), ETODO) { + testRpcLeaseClient.Fatalf("retryrpcClient.Send(\"Unmount\",,) failed: %v", err) + } + } + } + + retryrpcClient.Close() + + testRpcLeaseClient.wg.Done() + + runtime.Goexit() + } + } +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) Interrupt(rpcInterruptBuf []byte) { + var ( + err error + rpcInterrupt *RPCInterrupt + ) + + rpcInterrupt = &RPCInterrupt{} + + err = json.Unmarshal(rpcInterruptBuf, rpcInterrupt) + if nil != err { + testRpcLeaseClient.Fatalf("json.Unmarshal() failed: %v", err) + } + if rpcInterrupt.InodeNumber != testRpcLeaseClient.inodeNumber { + testRpcLeaseClient.Fatalf("Interrupt() called for InodeNumber %v... expected to be for %v", rpcInterrupt.InodeNumber, testRpcLeaseClient.inodeNumber) + } + + testRpcLeaseClient.logEvent(rpcInterrupt.RPCInterruptType) + + testRpcLeaseClient.chOut <- rpcInterrupt.RPCInterruptType +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) Fatalf(format string, args ...interface{}) { + var ( + argsForPrintf []interface{} + argsIndex int + argsValue interface{} + formatForPrintf string + ) + + formatForPrintf = "Failing testRpcLeaseClient %v: " + format + "\n" + + argsForPrintf = make([]interface{}, len(args)+1) + argsForPrintf[0] = testRpcLeaseClient.instance + for argsIndex, argsValue = range args { + argsForPrintf[argsIndex+1] = argsValue + } + + fmt.Printf(formatForPrintf, argsForPrintf...) + + os.Exit(-1) +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) sendLeaseRequest(leaseRequestType LeaseRequestType) { + time.Sleep(testRpcLeaseDelayBeforeSendingRequest) + testRpcLeaseClient.chIn <- leaseRequestType + time.Sleep(testRpcLeaseDelayAfterSendingRequest) +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) sendLeaseRequestPromptly(leaseRequestType LeaseRequestType) { + testRpcLeaseClient.chIn <- leaseRequestType +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) validateChOutValueIsLeaseResponseType(expectedLeaseResponseType LeaseResponseType) { + var ( + chOutValueAsInterface interface{} + chOutValueAsLeaseResponseType LeaseResponseType + ok bool + ) + + chOutValueAsInterface = <-testRpcLeaseClient.chOut + + chOutValueAsLeaseResponseType, ok = chOutValueAsInterface.(LeaseResponseType) + if !ok { + testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut did not return a LeaseResponseType") + } + if chOutValueAsLeaseResponseType != expectedLeaseResponseType { + testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut returned LeaseResponseType %v... expected %v", chOutValueAsLeaseResponseType, expectedLeaseResponseType) + } +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) validateChOutValueIsLeaseResponseTypeIgnoringRPCInterruptType(expectedLeaseResponseType LeaseResponseType, ignoredRPCInterruptType RPCInterruptType) { + var ( + chOutValueAsInterface interface{} + chOutValueAsRPCInterruptType RPCInterruptType + chOutValueAsLeaseResponseType LeaseResponseType + ok bool + ) + + for { + chOutValueAsInterface = <-testRpcLeaseClient.chOut + + chOutValueAsRPCInterruptType, ok = chOutValueAsInterface.(RPCInterruptType) + if ok { + if chOutValueAsRPCInterruptType != ignoredRPCInterruptType { + testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut did not return an ignored RPCInterruptType") + } + } else { + break + } + } + + chOutValueAsLeaseResponseType, ok = chOutValueAsInterface.(LeaseResponseType) + if !ok { + testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut did not return a LeaseResponseType or ignored RPCInterruptType") + } + if chOutValueAsLeaseResponseType != expectedLeaseResponseType { + testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut returned LeaseResponseType %v... expected %v", chOutValueAsLeaseResponseType, expectedLeaseResponseType) + } +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) validateChOutValueIsRPCInterruptType(expectedRPCInterruptType RPCInterruptType) { + var ( + chOutValueAsInterface interface{} + chOutValueAsRPCInterruptType RPCInterruptType + ok bool + ) + + chOutValueAsInterface = <-testRpcLeaseClient.chOut + + chOutValueAsRPCInterruptType, ok = chOutValueAsInterface.(RPCInterruptType) + if !ok { + testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut did not return a RPCInterruptType") + } + if chOutValueAsRPCInterruptType != expectedRPCInterruptType { + testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut returned RPCInterruptType %v... expected %v", chOutValueAsRPCInterruptType, expectedRPCInterruptType) + } +} + +func testRpcLeaseLogTestCase(testCase string, verbose bool) { + fmt.Printf("%v %s\n", time.Now().Format(testRpcLeaseTimeFormat), testCase) + testRpcLeaseLogVerbosely = verbose +} + +func (testRpcLeaseClient *testRpcLeaseClientStruct) logEvent(ev interface{}) { + if testRpcLeaseLogVerbosely { + switch ev.(type) { + case LeaseRequestType: + fmt.Printf("%v %s%s-> \n", time.Now().Format(testRpcLeaseTimeFormat), strings.Repeat(" ", int(testRpcLeaseClient.instance)), testRpcLeaseRequestLetters[ev.(LeaseRequestType)]) + case LeaseResponseType: + fmt.Printf("%v %s <-%s\n", time.Now().Format(testRpcLeaseTimeFormat), strings.Repeat(" ", int(testRpcLeaseClient.instance)), testRpcLeaseResponseLetters[ev.(LeaseResponseType)]) + case RPCInterruptType: + fmt.Printf("%v %s ^^%s\n", time.Now().Format(testRpcLeaseTimeFormat), strings.Repeat(" ", int(testRpcLeaseClient.instance)), testRpcLeaseInterruptLetters[ev.(RPCInterruptType)]) + } + } +} diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 2d59cfdb..f130e0a6 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -4,6 +4,7 @@ package imgrpkg import ( + "container/list" "crypto/tls" "encoding/base64" "fmt" @@ -12,7 +13,6 @@ import ( "github.com/NVIDIA/sortedmap" - "github.com/NVIDIA/proxyfs/blunder" "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/retryrpc" "github.com/NVIDIA/proxyfs/utils" @@ -137,12 +137,15 @@ retryGenerateMountID: } mount = &mountStruct{ - volume: volume, - mountID: mountIDAsString, - leasesExpired: false, - authTokenExpired: false, - authToken: mountRequest.AuthToken, - lastAuthTime: startTime, + volume: volume, + mountID: mountIDAsString, + retryRPCClientID: retryRPCClientID, + acceptingLeaseRequests: true, + leaseRequestMap: make(map[uint64]*leaseRequestStruct), + leasesExpired: false, + authTokenExpired: false, + authToken: mountRequest.AuthToken, + lastAuthTime: startTime, } volume.mountMap[mountIDAsString] = mount @@ -376,6 +379,12 @@ func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) volume.RLock() globals.RUnlock() + if mount.authTokenHasExpired() { + volume.RUnlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + return + } + if nil == volume.checkPointControlChan { volume.RUnlock() err = nil @@ -393,7 +402,12 @@ func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { var ( - startTime time.Time = time.Now() + inodeLease *inodeLeaseStruct + leaseRequestOperation *leaseRequestOperationStruct + mount *mountStruct + ok bool + startTime time.Time = time.Now() + volume *volumeStruct ) defer func() { @@ -423,12 +437,91 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) }() default: leaseResponse.LeaseResponseType = LeaseResponseTypeDenied - err = fmt.Errorf("LeaseRequestType %v not supported", leaseRequest.LeaseRequestType) - err = blunder.AddError(err, blunder.BadLeaseRequest) + err = fmt.Errorf("%s LeaseRequestType %v not supported", ELeaseRequestDenied, leaseRequest.LeaseRequestType) + return + } + + globals.RLock() + + mount, ok = globals.mountMap[leaseRequest.MountID] + if !ok { + globals.RUnlock() + err = fmt.Errorf("%s %s", EUnknownMountID, leaseRequest.MountID) return } - return fmt.Errorf(ETODO + " lease") + volume = mount.volume + + volume.Lock() + globals.RUnlock() + + if mount.authTokenHasExpired() { + volume.Unlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + return + } + + if (leaseRequest.LeaseRequestType == LeaseRequestTypeShared) || (leaseRequest.LeaseRequestType == LeaseRequestTypeExclusive) { + if !mount.acceptingLeaseRequests { + volume.Unlock() + leaseResponse.LeaseResponseType = LeaseResponseTypeDenied + err = fmt.Errorf("%s LeaseRequestType %v not currently being accepted", ELeaseRequestDenied, leaseRequest.LeaseRequestType) + return + } + inodeLease, ok = volume.inodeLeaseMap[leaseRequest.InodeNumber] + if !ok { + inodeLease = &inodeLeaseStruct{ + volume: volume, + inodeNumber: leaseRequest.InodeNumber, + leaseState: inodeLeaseStateNone, + requestChan: make(chan *leaseRequestOperationStruct), + stopChan: make(chan struct{}), + sharedHoldersList: list.New(), + promotingHolder: nil, + exclusiveHolder: nil, + releasingHoldersList: list.New(), + requestedList: list.New(), + lastGrantTime: time.Time{}, + lastInterruptTime: time.Time{}, + interruptsSent: 0, + longAgoTimer: &time.Timer{}, + interruptTimer: &time.Timer{}, + } + + volume.inodeLeaseMap[leaseRequest.InodeNumber] = inodeLease + inodeLease.lruElement = volume.inodeLeaseLRU.PushBack(inodeLease) + + volume.leaseHandlerWG.Add(1) + go inodeLease.handler() + } + } else { // in.LeaseRequestType is one of LeaseRequestType{Promote|Demote|Release} + inodeLease, ok = volume.inodeLeaseMap[leaseRequest.InodeNumber] + if !ok { + volume.Unlock() + leaseResponse.LeaseResponseType = LeaseResponseTypeDenied + err = fmt.Errorf("%s LeaseRequestType %v not allowed for non-existent Lease", ELeaseRequestDenied, leaseRequest.LeaseRequestType) + return + } + } + + // Send Lease Request Operation to *inodeLeaseStruct.handler() + // + // Note that we still hold the volumesLock, so inodeLease can't disappear out from under us + + leaseRequestOperation = &leaseRequestOperationStruct{ + mount: mount, + inodeLease: inodeLease, + LeaseRequestType: leaseRequest.LeaseRequestType, + replyChan: make(chan LeaseResponseType), + } + + inodeLease.requestChan <- leaseRequestOperation + + volume.Unlock() + + leaseResponse.LeaseResponseType = <-leaseRequestOperation.replyChan + + return } func (mount *mountStruct) authTokenHasExpired() (authTokenExpired bool) { diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index 3f688432..88906070 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -142,7 +142,7 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { "IMGR.MinLeaseDuration=250ms", "IMGR.LeaseInterruptInterval=250ms", - "IMGR.LeaseInterruptLimit=20", + "IMGR.LeaseInterruptLimit=5", "IMGR.SwiftRetryDelay=100ms", "IMGR.SwiftRetryExpBackoff=2", diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 0e698983..5c774eae 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -734,6 +734,8 @@ func putVolume(name string, storageURL string) (err error) { inodeTableLayout: nil, pendingObjectDeleteSet: make(map[uint64]struct{}), checkPointControlChan: nil, + inodeLeaseMap: make(map[uint64]*inodeLeaseStruct), + inodeLeaseLRU: list.New(), } globals.Lock() From 88303fa46bc366a61a554539cfc8aa243b1ff094 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 20 May 2021 11:29:55 -0700 Subject: [PATCH 008/258] Completed Mount RPC actually mounting the volume & implemented GetInodeTableEntry RPC --- ilayout/api.go | 12 +-- ilayout/api_test.go | 35 +++++-- ilayout/impl.go | 21 ++-- imgr/imgrpkg/api.go | 5 +- imgr/imgrpkg/globals.go | 2 +- imgr/imgrpkg/retry-rpc.go | 87 ++++++++++++++++- imgr/imgrpkg/retry-rpc_test.go | 14 +-- imgr/imgrpkg/volume.go | 174 +++++++++++++++++++++++++++++++-- 8 files changed, 302 insertions(+), 48 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 71b9a813..618eac0e 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -248,8 +248,8 @@ func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) MarshalInodeTableEnt // UnmarshalInodeTableEntryValueV1 decodes inodeTableEntryValueV1 from inodeTableEntryValueV1Buf. // -func UnmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, err error) { - inodeTableEntryValueV1, err = unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf) +func UnmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, bytesConsumed int, err error) { + inodeTableEntryValueV1, bytesConsumed, err = unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf) return } @@ -397,8 +397,8 @@ func (directoryEntryValueV1 *DirectoryEntryValueV1Struct) MarshalDirectoryEntryV // UnmarshalDirectoryEntryValueV1 decodes directoryEntryValueV1 from directoryEntryValueV1Buf. // -func UnmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryEntryValueV1 *DirectoryEntryValueV1Struct, err error) { - directoryEntryValueV1, err = unmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf) +func UnmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryEntryValueV1 *DirectoryEntryValueV1Struct, bytesConsumed int, err error) { + directoryEntryValueV1, bytesConsumed, err = unmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf) return } @@ -426,8 +426,8 @@ func (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct) MarshalExtentMapEntryV // UnmarshalExtentMapEntryValueV1 decodes directoryEntryValueV1 from directoryEntryValueV1Buf. // -func UnmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct, err error) { - extentMapEntryValueV1, err = unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf) +func UnmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct, bytesConsumed int, err error) { + extentMapEntryValueV1, bytesConsumed, err = unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf) return } diff --git a/ilayout/api_test.go b/ilayout/api_test.go index 5e2b3c5e..f315e03b 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -64,9 +64,11 @@ func TestAPI(t *testing.T) { InodeHeadLength: 3, } - marshaledInodeTableEntryValueV1 []byte - unmarshaledInodeTableEntryValueVersion uint64 - unmarshaledInodeTableEntryValueV1 *InodeTableEntryValueV1Struct + marshaledInodeTableEntryValueV1 []byte + unmarshaledInodeTableEntryValueVersion uint64 + unmarshaledInodeTableEntryValueV1 *InodeTableEntryValueV1Struct + unmarshaledInodeTableEntryValueV1BytesConsumed int + unmarshaledInodeTableEntryValueV1BytesConsumedExpected = int(8 + 8 + 8) testStartTime = time.Now().Truncate(time.Second) @@ -131,8 +133,10 @@ func TestAPI(t *testing.T) { InodeType: 2, } - marshaledDirectoryEntryValueV1 []byte - unmarshaledDirectoryEntryValueV1 *DirectoryEntryValueV1Struct + marshaledDirectoryEntryValueV1 []byte + unmarshaledDirectoryEntryValueV1 *DirectoryEntryValueV1Struct + unmarshaledDirectoryEntryValueV1BytesConsumed int + unmarshaledDirectoryEntryValueV1BytesConsumedExpected = int(8 + 1) testExtentMapEntryValueV1 = &ExtentMapEntryValueV1Struct{ FileOffset: 1, @@ -141,8 +145,10 @@ func TestAPI(t *testing.T) { ObjectOffset: 4, } - marshaledExtentMapEntryValueV1 []byte - unmarshaledExtentMapEntryValueV1 *ExtentMapEntryValueV1Struct + marshaledExtentMapEntryValueV1 []byte + unmarshaledExtentMapEntryValueV1 *ExtentMapEntryValueV1Struct + unmarshaledExtentMapEntryValueV1BytesConsumed int + unmarshaledExtentMapEntryValueV1BytesConsumedExpected = int(8 + 8 + 8 + 8) ) marshaledCheckPointV1, err = testCheckPointV1.MarshalCheckPointV1() @@ -215,7 +221,7 @@ func TestAPI(t *testing.T) { t.Fatalf("Bad unmarshaledInodeTableEntryValueVersion (%016X) - expected InodeTableEntryValueVersionV1 (%016X)", unmarshaledInodeTableEntryValueVersion, InodeTableEntryValueVersionV1) } - unmarshaledInodeTableEntryValueV1, err = UnmarshalInodeTableEntryValueV1(marshaledInodeTableEntryValueV1) + unmarshaledInodeTableEntryValueV1, unmarshaledInodeTableEntryValueV1BytesConsumed, err = UnmarshalInodeTableEntryValueV1(marshaledInodeTableEntryValueV1) if nil != err { t.Fatal(err) } @@ -223,6 +229,9 @@ func TestAPI(t *testing.T) { (testInodeTableEntryValueV1.InodeHeadLength != unmarshaledInodeTableEntryValueV1.InodeHeadLength) { t.Fatalf("Bad unmarshaledInodeTableEntryValueV1 (%+v) - expected testInodeTableEntryValueV1 (%+v)", unmarshaledInodeTableEntryValueV1, testInodeTableEntryValueV1) } + if unmarshaledInodeTableEntryValueV1BytesConsumed != unmarshaledInodeTableEntryValueV1BytesConsumedExpected { + t.Fatalf("Bad unmarshaledInodeTableEntryValueV1BytesConsumed (%v) - expected %v", unmarshaledInodeTableEntryValueV1BytesConsumed, unmarshaledInodeTableEntryValueV1BytesConsumedExpected) + } marshaledInodeHeadV1, err = testInodeHeadV1.MarshalInodeHeadV1() if nil != err { @@ -274,24 +283,30 @@ func TestAPI(t *testing.T) { t.Fatal(err) } - unmarshaledDirectoryEntryValueV1, err = UnmarshalDirectoryEntryValueV1(marshaledDirectoryEntryValueV1) + unmarshaledDirectoryEntryValueV1, unmarshaledDirectoryEntryValueV1BytesConsumed, err = UnmarshalDirectoryEntryValueV1(marshaledDirectoryEntryValueV1) if nil != err { t.Fatal(err) } if *testDirectoryEntryValueV1 != *unmarshaledDirectoryEntryValueV1 { t.Fatalf("Bad unmarshaledDirectoryEntryValueV1 (%+v) - expected testDirectoryEntryValueV1 (%+v)", unmarshaledDirectoryEntryValueV1, testDirectoryEntryValueV1) } + if unmarshaledDirectoryEntryValueV1BytesConsumed != unmarshaledDirectoryEntryValueV1BytesConsumedExpected { + t.Fatalf("Bad unmarshaledDirectoryEntryValueV1BytesConsumed (%v) - expected %v", unmarshaledDirectoryEntryValueV1BytesConsumed, unmarshaledDirectoryEntryValueV1BytesConsumedExpected) + } marshaledExtentMapEntryValueV1, err = testExtentMapEntryValueV1.MarshalExtentMapEntryValueV1() if nil != err { t.Fatal(err) } - unmarshaledExtentMapEntryValueV1, err = UnmarshalExtentMapEntryValueV1(marshaledExtentMapEntryValueV1) + unmarshaledExtentMapEntryValueV1, unmarshaledExtentMapEntryValueV1BytesConsumed, err = UnmarshalExtentMapEntryValueV1(marshaledExtentMapEntryValueV1) if nil != err { t.Fatal(err) } if *testExtentMapEntryValueV1 != *unmarshaledExtentMapEntryValueV1 { t.Fatalf("Bad unmarshaledExtentMapEntryValueV1 (%+v) - expected testExtentMapEntryValueV1 (%+v)", unmarshaledExtentMapEntryValueV1, testExtentMapEntryValueV1) } + if unmarshaledExtentMapEntryValueV1BytesConsumed != unmarshaledExtentMapEntryValueV1BytesConsumedExpected { + t.Fatalf("Bad unmarshaledExtentMapEntryValueV1BytesConsumed (%v) - expected %v", unmarshaledExtentMapEntryValueV1BytesConsumed, unmarshaledExtentMapEntryValueV1BytesConsumedExpected) + } } diff --git a/ilayout/impl.go b/ilayout/impl.go index 46449349..e80c8877 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -321,7 +321,7 @@ func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) marshalInodeTableEnt return } -func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, err error) { +func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, bytesConsumed int, err error) { var ( curPos int inodeTableEntryValueVersion uint64 @@ -350,10 +350,7 @@ func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTab return } - if curPos != len(inodeTableEntryValueV1Buf) { - err = fmt.Errorf("Incorrect size for inodeTableEntryValueV1Buf") - return - } + bytesConsumed = curPos err = nil return @@ -754,7 +751,7 @@ func (directoryEntryValueV1 *DirectoryEntryValueV1Struct) marshalDirectoryEntryV return } -func unmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryEntryValueV1 *DirectoryEntryValueV1Struct, err error) { +func unmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryEntryValueV1 *DirectoryEntryValueV1Struct, bytesConsumed int, err error) { var ( curPos int ) @@ -773,10 +770,7 @@ func unmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryE return } - if curPos != len(directoryEntryValueV1Buf) { - err = fmt.Errorf("Incorrect size for directoryEntryValueV1Buf") - return - } + bytesConsumed = curPos err = nil return @@ -815,7 +809,7 @@ func (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct) marshalExtentMapEntryV return } -func unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct, err error) { +func unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct, bytesConsumed int, err error) { var ( curPos int ) @@ -844,10 +838,7 @@ func unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapE return } - if curPos != len(extentMapEntryValueV1Buf) { - err = fmt.Errorf("Incorrect size for extentMapEntryValueV1Buf") - return - } + bytesConsumed = curPos err = nil return diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 4eb333cf..4eaca9b4 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -153,7 +153,9 @@ func LogInfof(format string, args ...interface{}) { const ( EAuthTokenRejected = "EAuthTokenRejected:" ELeaseRequestDenied = "ELeaseRequestDenied:" + EMissingLease = "EMissingLease:" EVolumeBeingDeleted = "EVolumeBeingDeleted:" + EUnknownInodeNumber = "EUnknownInodeNumber:" EUnknownMountID = "EUnknownMountID:" EUnknownVolumeName = "EUnknownVolumeName:" @@ -252,7 +254,8 @@ type GetInodeTableEntryRequestStruct struct { // GetInodeTableEntryResponseStruct is the response object for GetInodeTableEntry. // type GetInodeTableEntryResponseStruct struct { - // TODO + InodeHeadObjectNumber uint64 + InodeHeadLength uint64 } // GetInodeTableEntry requests the Inode information for the specified Inode diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 81c45779..7430b48c 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -228,7 +228,7 @@ type volumeStruct struct { deleting bool // checkPoint *ilayout.CheckPointV1Struct // == nil if not currently mounted and/or checkpointing superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing - inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing + inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing; key == inodeNumber; value == *ilayout.InodeTableEntryValueV1Struct inodeTableLayout map[uint64]*inodeTableLayoutElementStruct // == nil if not currently mounted and/or checkpointing; key == objectNumber (matching ilayout.InodeTableLayoutEntryV1Struct.ObjectNumber) pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index f130e0a6..292b3289 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -75,6 +75,8 @@ func stopRetryRPCServer() (err error) { func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { var ( alreadyInGlobalsMountMap bool + inodeTableEntryInMemory *inodeTableLayoutElementStruct + inodeTableEntryOnDisk ilayout.InodeTableLayoutEntryV1Struct lastCheckPoint *ilayout.CheckPointV1Struct lastCheckPointAsByteSlice []byte lastCheckPointAsString string @@ -83,6 +85,7 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo mountIDAsString string ok bool startTime time.Time = time.Now() + superBlockAsByteSlice []byte volume *volumeStruct volumeAsValue sortedmap.Value ) @@ -160,6 +163,32 @@ retryGenerateMountID: volume.checkPoint = lastCheckPoint + superBlockAsByteSlice, err = swiftObjectGetTail(volume.storageURL, mountRequest.AuthToken, volume.checkPoint.SuperBlockObjectNumber, volume.checkPoint.SuperBlockLength) + if nil != err { + logFatalf("swiftObjectGetTail(volume.storageURL, mountRequest.AuthToken, volume.checkPoint.SuperBlockObjectNumber, volume.checkPoint.SuperBlockLength) failed: %v", err) + } + + volume.superBlock, err = ilayout.UnmarshalSuperBlockV1(superBlockAsByteSlice) + if nil != err { + logFatalf("ilayout.UnmarshalSuperBlockV1(superBlockAsByteSlice) failed: %v", err) + } + + volume.inodeTable, err = sortedmap.OldBPlusTree(volume.superBlock.InodeTableRootObjectNumber, volume.superBlock.InodeTableRootObjectOffset, volume.superBlock.InodeTableRootObjectLength, sortedmap.CompareUint64, volume, globals.inodeTableCache) + if nil != err { + logFatalf("sortedmap.OldBPlusTree(volume.superBlock.InodeTableRootObjectNumber, volume.superBlock.InodeTableRootObjectOffset, volume.superBlock.InodeTableRootObjectLength, sortedmap.CompareUint64, volume, globals.inodeTableCache) failed: %v", err) + } + + volume.inodeTableLayout = make(map[uint64]*inodeTableLayoutElementStruct) + + for _, inodeTableEntryOnDisk = range volume.superBlock.InodeTableLayout { + inodeTableEntryInMemory = &inodeTableLayoutElementStruct{ + objectSize: inodeTableEntryOnDisk.ObjectSize, + bytesReferenced: inodeTableEntryOnDisk.BytesReferenced, + } + + volume.inodeTableLayout[inodeTableEntryOnDisk.ObjectNumber] = inodeTableEntryInMemory + } + volume.checkPointControlChan = make(chan chan error) volume.checkPointControlWG.Add(1) @@ -304,14 +333,68 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { var ( - startTime time.Time = time.Now() + inodeTableEntryValue *ilayout.InodeTableEntryValueV1Struct + inodeTableEntryValueRaw sortedmap.Value + leaseRequest *leaseRequestStruct + mount *mountStruct + ok bool + startTime time.Time = time.Now() + volume *volumeStruct ) defer func() { globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf(ETODO + " getInodeTableEntry") + globals.RLock() + + mount, ok = globals.mountMap[getInodeTableEntryRequest.MountID] + if !ok { + globals.RUnlock() + err = fmt.Errorf("%s %s", EUnknownMountID, getInodeTableEntryRequest.MountID) + return + } + + volume = mount.volume + + volume.Lock() + globals.RUnlock() + + if mount.authTokenHasExpired() { + volume.Unlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + return + } + + leaseRequest, ok = mount.leaseRequestMap[getInodeTableEntryRequest.InodeNumber] + if !ok || ((leaseRequestStateSharedGranted != leaseRequest.requestState) && (leaseRequestStateExclusiveGranted != leaseRequest.requestState)) { + volume.Unlock() + err = fmt.Errorf("%s %016X", EMissingLease, getInodeTableEntryRequest.InodeNumber) + return + } + + inodeTableEntryValueRaw, ok, err = volume.inodeTable.GetByKey(getInodeTableEntryRequest.InodeNumber) + if nil != err { + logFatalf("volume.inodeTable.GetByKey(getInodeTableEntryRequest.InodeNumber) failed: %v", err) + } + if !ok { + volume.Unlock() + err = fmt.Errorf("%s %016X", EUnknownInodeNumber, getInodeTableEntryRequest.InodeNumber) + return + } + + inodeTableEntryValue, ok = inodeTableEntryValueRaw.(*ilayout.InodeTableEntryValueV1Struct) + if !ok { + logFatalf("inodeTableEntryValueRaw.(*ilayout.InodeTableEntryValueV1Struct) returned !ok") + } + + getInodeTableEntryResponse.InodeHeadObjectNumber = inodeTableEntryValue.InodeHeadObjectNumber + getInodeTableEntryResponse.InodeHeadLength = inodeTableEntryValue.InodeHeadLength + + volume.Unlock() + + err = nil + return } func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index feb88c55..4d9014ad 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -120,13 +120,6 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"FetchNonceRange()\",,) failed: %v", err) } - // TODO: Remove this early exit skipping of following TODOs - - if nil == err { - t.Logf("Exiting TestRetryRPC() early to skip following TODOs") - return - } - // Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ @@ -140,6 +133,13 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) should have failed") } + // TODO: Remove this early exit skipping of following TODOs + + if nil != err { + t.Logf("Exiting TestRetryRPC() early to skip following TODOs") + return + } + // TODO: Force a need for a re-auth // Attempt a GetInodeTableEntry() for RootDirInode... which should fail (re-auth required) diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 5c774eae..f834a32f 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -87,14 +87,14 @@ func deleteVolume(volumeName string) (err error) { volumeAsStruct, ok = volumeAsValue.(*volumeStruct) if !ok { - logFatalf("[IMGR]globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } // The following is only temporary... // TODO: Actually gracefully unmount clients, block new mounts, and lazily remove it if 0 != len(volumeAsStruct.mountMap) { - logFatalf("[IMGR]No support for deleting actively mounted volume \"%s\"", volumeName) + logFatalf("No support for deleting actively mounted volume \"%s\"", volumeName) } ok, err = globals.volumeMap.DeleteByKey(volumeAsStruct.name) @@ -102,7 +102,7 @@ func deleteVolume(volumeName string) (err error) { logFatal(err) } if !ok { - logFatalf("[IMGR]globals.volumeMap[\"%s\"] suddenly missing", volumeAsStruct.name) + logFatalf("globals.volumeMap[\"%s\"] suddenly missing", volumeAsStruct.name) } globals.Unlock() @@ -141,7 +141,7 @@ func getVolumeAsJSON(volumeName string) (volume []byte, err error) { volumeAsStruct, ok = volumeAsValue.(*volumeStruct) if !ok { - logFatalf("[IMGR]globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } volumeAsStruct.RLock() @@ -192,12 +192,12 @@ func getVolumeListAsJSON() (volumeList []byte) { logFatal(err) } if !ok { - logFatalf("[IMGR]globals.volumeMap[] len (%d) is wrong", volumeListLen) + logFatalf("globals.volumeMap[] len (%d) is wrong", volumeListLen) } volumeAsStruct, ok = volumeAsValue.(*volumeStruct) if !ok { - logFatalf("[IMGR]globals.volumeMap[%d] was not a *volumeStruct", volumeListIndex) + logFatalf("globals.volumeMap[%d] was not a *volumeStruct", volumeListIndex) } volumeAsStruct.RLock() @@ -809,3 +809,165 @@ func (volume *volumeStruct) doCheckPoint() (err error) { return } + +func (volume *volumeStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + var ( + keyAsInodeNumber uint64 + ok bool + ) + + keyAsInodeNumber, ok = key.(uint64) + if !ok { + err = fmt.Errorf("key.(uint64) returned !ok") + return + } + + keyAsString = fmt.Sprintf("%016X", keyAsInodeNumber) + + err = nil + return +} + +func (volume *volumeStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + var ( + ok bool + valueAsInodeTableEntryValue *ilayout.InodeTableEntryValueV1Struct + ) + + valueAsInodeTableEntryValue, ok = value.(*ilayout.InodeTableEntryValueV1Struct) + if !ok { + err = fmt.Errorf("value.(*ilayout.InodeTableEntryValueV1Struct) returned !ok") + return + } + + valueAsString = fmt.Sprintf("[%016X %016X]", valueAsInodeTableEntryValue.InodeHeadObjectNumber, valueAsInodeTableEntryValue.InodeHeadLength) + + err = nil + return +} + +func (volume *volumeStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { + var ( + mount *mountStruct + mountListElement *list.Element + ok bool + ) + +NextHealthyMount: + + mountListElement = volume.healthyMountList.Front() + if nil == mountListElement { + err = fmt.Errorf("no healthy mounts available [case 1]") + return + } + + mount, ok = mountListElement.Value.(*mountStruct) + if !ok { + logFatalf("mountListElement.Value.(*mountStruct) returned !ok [case 1]") + } + + volume.healthyMountList.MoveToBack(mountListElement) + + nodeByteSlice, err = swiftObjectGetRange(volume.storageURL, mount.authToken, objectNumber, objectOffset, objectLength) + if nil == err { + volume.healthyMountList.MoveToBack(mountListElement) + + return // nil err from swiftObjectGetRange() is used + } + + // Assume that the failure was due to AuthToken expiration + + _ = volume.healthyMountList.Remove(mount.listElement) + + mount.authTokenExpired = true + + mount.listElement = volume.authTokenExpiredMountList.PushBack(mount) + + goto NextHealthyMount +} + +func (volume *volumeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + err = fmt.Errorf("%s %s", ETODO, "PutNode()") + return +} + +func (volume *volumeStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + err = fmt.Errorf("%s %s", ETODO, "DiscardNode()") + return +} + +func (volume *volumeStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + var ( + keyAsUint64 uint64 + nextPos int + ok bool + ) + + keyAsUint64, ok = key.(uint64) + if !ok { + err = fmt.Errorf("(*volumeStruct).PackKey(key:%v) called with non-uint64", key) + return + } + + packedKey = make([]byte, 8) + + nextPos, err = ilayout.PutLEUint64ToBuf(packedKey, 0, keyAsUint64) + if nil != err { + return + } + + if len(packedKey) != nextPos { + err = fmt.Errorf("(*volumeStruct).PackKey(key:%016X) logic error", keyAsUint64) + return + } + + err = nil + return +} + +func (volume *volumeStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + var ( + nextPos int + ) + + key, nextPos, err = ilayout.GetLEUint64FromBuf(payloadData, 0) + if (nil == err) && (nextPos != 8) { + err = fmt.Errorf("ilayout.GetLEUint64FromBuf(payloadData, 0) consumed %v bytes (8 expected)", nextPos) + } + + bytesConsumed = 8 + + return +} + +func (volume *volumeStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + var ( + ok bool + valueAsInodeTableEntryValueV1 ilayout.InodeTableEntryValueV1Struct + ) + + valueAsInodeTableEntryValueV1, ok = value.(ilayout.InodeTableEntryValueV1Struct) + if !ok { + err = fmt.Errorf("(*volumeStruct).PackValue(value:%v) called with non-InodeTableEntryValueV1Struct", value) + return + } + + packedValue, err = valueAsInodeTableEntryValueV1.MarshalInodeTableEntryValueV1() + + return +} + +func (volume *volumeStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + var ( + bytesConsumedAsInt int + ) + + value, bytesConsumedAsInt, err = ilayout.UnmarshalInodeTableEntryValueV1(payloadData) + if (nil == err) && (bytesConsumedAsInt != 24) { + err = fmt.Errorf("ilayout.UnmarshalInodeTableEntryValueV1(payloadData) consumed %v bytes (24 expected)", bytesConsumedAsInt) + } + + bytesConsumed = 24 + + return +} From 0709939d107f947c0704963ae994964ba7ba98d5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 24 May 2021 13:44:08 -0700 Subject: [PATCH 009/258] Reverted to a single global lock in imgrpkg - several resources were global...making volume-level locking insufficient anyway --- imgr/imgrpkg/globals.go | 7 ++-- imgr/imgrpkg/lease.go | 28 ++++++++-------- imgr/imgrpkg/retry-rpc.go | 70 +++++++++++++++------------------------ imgr/imgrpkg/volume.go | 24 +++++--------- 4 files changed, 51 insertions(+), 78 deletions(-) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 7430b48c..8fcf2d95 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -174,7 +174,7 @@ const ( type inodeLeaseStruct struct { volume *volumeStruct inodeNumber uint64 - lruElement *list.Element // link into volumeStruct.inodeLeaseLRU + lruElement *list.Element // link into globals.inodeLeaseLRU leaseState inodeLeaseStateType requestChan chan *leaseRequestOperationStruct @@ -218,7 +218,6 @@ type inodeTableLayoutElementStruct struct { } type volumeStruct struct { - sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() name string // storageURL string // mountMap map[string]*mountStruct // key == mountStruct.mountID @@ -234,16 +233,16 @@ type volumeStruct struct { checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG inodeLeaseMap map[uint64]*inodeLeaseStruct // key == inodeLeaseStruct.inodeNumber - inodeLeaseLRU *list.List // .Front() is the LRU inodeLeaseStruct.listElement leaseHandlerWG sync.WaitGroup // .Add(1) each inodeLease insertion into inodeLeaseMap // .Done() each inodeLease after it is removed from inodeLeaseMap } type globalsStruct struct { - sync.RWMutex // + sync.Mutex // config configStruct // logFile *os.File // == nil if config.LogFilePath == "" inodeTableCache sortedmap.BPlusTreeCache // + inodeLeaseLRU *list.List // .Front() is the LRU inodeLeaseStruct.listElement volumeMap sortedmap.LLRBTree // key == volumeStruct.name; value == *volumeStruct mountMap map[string]*mountStruct // key == mountStruct.mountID httpClient *http.Client // diff --git a/imgr/imgrpkg/lease.go b/imgr/imgrpkg/lease.go index 54bde926..12fe2d59 100644 --- a/imgr/imgrpkg/lease.go +++ b/imgr/imgrpkg/lease.go @@ -42,10 +42,10 @@ func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *lease sharedHolderListElement *list.Element ) - inodeLease.volume.Lock() - defer inodeLease.volume.Unlock() + globals.Lock() + defer globals.Unlock() - inodeLease.volume.inodeLeaseLRU.MoveToBack(inodeLease.lruElement) + globals.inodeLeaseLRU.MoveToBack(inodeLease.lruElement) switch leaseRequestOperation.LeaseRequestType { case LeaseRequestTypeShared: @@ -675,7 +675,8 @@ func (inodeLease *inodeLeaseStruct) handleLongAgoTimerPop() { rpcInterruptBuf []byte ) - inodeLease.volume.Lock() + globals.Lock() + defer globals.Unlock() inodeLease.lastGrantTime = time.Time{} inodeLease.longAgoTimer = &time.Timer{} @@ -775,8 +776,6 @@ func (inodeLease *inodeLeaseStruct) handleLongAgoTimerPop() { default: logFatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() called while in wrong state (%v)", inodeLease.leaseState) } - - inodeLease.volume.Unlock() } func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { @@ -788,7 +787,8 @@ func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { rpcInterruptBuf []byte ) - inodeLease.volume.Lock() + globals.Lock() + defer globals.Unlock() if globals.config.LeaseInterruptLimit <= inodeLease.interruptsSent { switch inodeLease.leaseState { @@ -1045,8 +1045,6 @@ func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { inodeLease.interruptTimer = time.NewTimer(globals.config.LeaseInterruptInterval) } - - inodeLease.volume.Unlock() } func (inodeLease *inodeLeaseStruct) handleStopChanClose() { @@ -1062,7 +1060,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { // Deny all pending requests: - inodeLease.volume.Lock() + globals.Lock() for nil != inodeLease.requestedList.Front() { leaseRequestElement = inodeLease.requestedList.Front() @@ -1192,11 +1190,11 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { // Loop until inodeLease.leaseState is inodeLeaseStateNone for inodeLeaseStateNone != inodeLease.leaseState { - inodeLease.volume.Unlock() + globals.Unlock() select { case leaseRequestOperation = <-inodeLease.requestChan: - inodeLease.volume.Lock() + globals.Lock() switch leaseRequestOperation.LeaseRequestType { case LeaseRequestTypeShared: @@ -1340,7 +1338,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { } case _ = <-inodeLease.interruptTimer.C: - inodeLease.volume.Lock() + globals.Lock() switch inodeLease.leaseState { case inodeLeaseStateSharedGrantedLongAgo: @@ -1482,11 +1480,11 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { RequestChanDrained: delete(inodeLease.volume.inodeLeaseMap, inodeLease.inodeNumber) - _ = inodeLease.volume.inodeLeaseLRU.Remove(inodeLease.lruElement) + _ = globals.inodeLeaseLRU.Remove(inodeLease.lruElement) inodeLease.volume.leaseHandlerWG.Done() - inodeLease.volume.Unlock() + globals.Unlock() runtime.Goexit() } diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 292b3289..cf5855e6 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -111,10 +111,7 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo logFatalf("volumeAsValue.(*volumeStruct) returned !ok") } - volume.Lock() - if volume.deleting { - volume.Unlock() globals.Unlock() err = fmt.Errorf("%s %s", EVolumeBeingDeleted, mountRequest.VolumeName) return @@ -122,7 +119,6 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo lastCheckPointAsByteSlice, err = swiftObjectGet(volume.storageURL, mountRequest.AuthToken, ilayout.CheckPointObjectNumber) if nil != err { - volume.Unlock() globals.Unlock() err = fmt.Errorf("%s %s", EAuthTokenRejected, mountRequest.AuthToken) return @@ -196,7 +192,6 @@ retryGenerateMountID: go volume.checkPointDaemon(volume.checkPointControlChan) } - volume.Unlock() globals.Unlock() mountResponse.MountID = mountIDAsString @@ -217,19 +212,18 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * globals.stats.RenewMountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - globals.RLock() + globals.Lock() mount, ok = globals.mountMap[renewMountRequest.MountID] if !ok { - globals.RUnlock() + globals.Unlock() err = fmt.Errorf("%s %s", EUnknownMountID, renewMountRequest.MountID) return } volume = mount.volume - volume.Lock() - globals.RUnlock() + globals.Unlock() mount.authToken = renewMountRequest.AuthToken @@ -249,7 +243,7 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * err = fmt.Errorf("%s %s", EAuthTokenRejected, renewMountRequest.AuthToken) } - volume.Unlock() + globals.Unlock() return } @@ -280,22 +274,19 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch globals.stats.FetchNonceRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - globals.RLock() + globals.Lock() mount, ok = globals.mountMap[fetchNonceRangeRequest.MountID] if !ok { - globals.RUnlock() + globals.Unlock() err = fmt.Errorf("%s %s", EUnknownMountID, fetchNonceRangeRequest.MountID) return } volume = mount.volume - volume.Lock() - globals.RUnlock() - if mount.authTokenHasExpired() { - volume.Unlock() + globals.Unlock() err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) return } @@ -326,7 +317,7 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) } - volume.Unlock() + globals.Unlock() return } @@ -346,29 +337,26 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - globals.RLock() + globals.Lock() mount, ok = globals.mountMap[getInodeTableEntryRequest.MountID] if !ok { - globals.RUnlock() + globals.Unlock() err = fmt.Errorf("%s %s", EUnknownMountID, getInodeTableEntryRequest.MountID) return } volume = mount.volume - volume.Lock() - globals.RUnlock() - if mount.authTokenHasExpired() { - volume.Unlock() + globals.Unlock() err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) return } leaseRequest, ok = mount.leaseRequestMap[getInodeTableEntryRequest.InodeNumber] if !ok || ((leaseRequestStateSharedGranted != leaseRequest.requestState) && (leaseRequestStateExclusiveGranted != leaseRequest.requestState)) { - volume.Unlock() + globals.Unlock() err = fmt.Errorf("%s %016X", EMissingLease, getInodeTableEntryRequest.InodeNumber) return } @@ -378,7 +366,7 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru logFatalf("volume.inodeTable.GetByKey(getInodeTableEntryRequest.InodeNumber) failed: %v", err) } if !ok { - volume.Unlock() + globals.Unlock() err = fmt.Errorf("%s %016X", EUnknownInodeNumber, getInodeTableEntryRequest.InodeNumber) return } @@ -391,7 +379,7 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru getInodeTableEntryResponse.InodeHeadObjectNumber = inodeTableEntryValue.InodeHeadObjectNumber getInodeTableEntryResponse.InodeHeadLength = inodeTableEntryValue.InodeHeadLength - volume.Unlock() + globals.Unlock() err = nil return @@ -448,35 +436,32 @@ func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) checkPointResponseChan = make(chan error) - globals.RLock() + globals.Lock() mount, ok = globals.mountMap[flushRequest.MountID] if !ok { - globals.RUnlock() + globals.Unlock() err = fmt.Errorf("%s %s", EUnknownMountID, flushRequest.MountID) return } volume = mount.volume - volume.RLock() - globals.RUnlock() - if mount.authTokenHasExpired() { - volume.RUnlock() + globals.Unlock() err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) return } if nil == volume.checkPointControlChan { - volume.RUnlock() + globals.Unlock() err = nil return } volume.checkPointControlChan <- checkPointResponseChan - volume.RUnlock() + globals.Unlock() err = <-checkPointResponseChan @@ -524,29 +509,26 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) return } - globals.RLock() + globals.Lock() mount, ok = globals.mountMap[leaseRequest.MountID] if !ok { - globals.RUnlock() + globals.Unlock() err = fmt.Errorf("%s %s", EUnknownMountID, leaseRequest.MountID) return } volume = mount.volume - volume.Lock() - globals.RUnlock() - if mount.authTokenHasExpired() { - volume.Unlock() + globals.Unlock() err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) return } if (leaseRequest.LeaseRequestType == LeaseRequestTypeShared) || (leaseRequest.LeaseRequestType == LeaseRequestTypeExclusive) { if !mount.acceptingLeaseRequests { - volume.Unlock() + globals.Unlock() leaseResponse.LeaseResponseType = LeaseResponseTypeDenied err = fmt.Errorf("%s LeaseRequestType %v not currently being accepted", ELeaseRequestDenied, leaseRequest.LeaseRequestType) return @@ -572,7 +554,7 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) } volume.inodeLeaseMap[leaseRequest.InodeNumber] = inodeLease - inodeLease.lruElement = volume.inodeLeaseLRU.PushBack(inodeLease) + inodeLease.lruElement = globals.inodeLeaseLRU.PushBack(inodeLease) volume.leaseHandlerWG.Add(1) go inodeLease.handler() @@ -580,7 +562,7 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) } else { // in.LeaseRequestType is one of LeaseRequestType{Promote|Demote|Release} inodeLease, ok = volume.inodeLeaseMap[leaseRequest.InodeNumber] if !ok { - volume.Unlock() + globals.Unlock() leaseResponse.LeaseResponseType = LeaseResponseTypeDenied err = fmt.Errorf("%s LeaseRequestType %v not allowed for non-existent Lease", ELeaseRequestDenied, leaseRequest.LeaseRequestType) return @@ -600,7 +582,7 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) inodeLease.requestChan <- leaseRequestOperation - volume.Unlock() + globals.Unlock() leaseResponse.LeaseResponseType = <-leaseRequestOperation.replyChan diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index f834a32f..6ef594b4 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -18,6 +18,7 @@ import ( func startVolumeManagement() (err error) { globals.inodeTableCache = sortedmap.NewBPlusTreeCache(globals.config.InodeTableCacheEvictLowLimit, globals.config.InodeTableCacheEvictLowLimit) + globals.inodeLeaseLRU = list.New() globals.volumeMap = sortedmap.NewLLRBTree(sortedmap.CompareString, &globals) globals.mountMap = make(map[string]*mountStruct) @@ -27,6 +28,7 @@ func startVolumeManagement() (err error) { func stopVolumeManagement() (err error) { globals.inodeTableCache = nil + globals.inodeLeaseLRU = nil globals.volumeMap = nil globals.mountMap = nil @@ -127,14 +129,14 @@ func getVolumeAsJSON(volumeName string) (volume []byte, err error) { volumeToReturn *volumeGETStruct ) - globals.RLock() + globals.Lock() volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) if nil != err { logFatal(err) } if !ok { - globals.RUnlock() + globals.Unlock() err = fmt.Errorf("volumeName \"%s\" does not exist", volumeName) return } @@ -144,9 +146,6 @@ func getVolumeAsJSON(volumeName string) (volume []byte, err error) { logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } - volumeAsStruct.RLock() - globals.RUnlock() - volumeToReturn = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, @@ -155,7 +154,7 @@ func getVolumeAsJSON(volumeName string) (volume []byte, err error) { AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), } - volumeAsStruct.RUnlock() + globals.Unlock() volume, err = json.Marshal(volumeToReturn) if nil != err { @@ -177,7 +176,7 @@ func getVolumeListAsJSON() (volumeList []byte) { volumeListToReturn []*volumeGETStruct ) - globals.RLock() + globals.Lock() volumeListLen, err = globals.volumeMap.Len() if nil != err { @@ -200,8 +199,6 @@ func getVolumeListAsJSON() (volumeList []byte) { logFatalf("globals.volumeMap[%d] was not a *volumeStruct", volumeListIndex) } - volumeAsStruct.RLock() - volumeListToReturn[volumeListIndex] = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, @@ -209,11 +206,9 @@ func getVolumeListAsJSON() (volumeList []byte) { LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), } - - volumeAsStruct.RUnlock() } - globals.RUnlock() + globals.Unlock() volumeList, err = json.Marshal(volumeListToReturn) if nil != err { @@ -735,7 +730,6 @@ func putVolume(name string, storageURL string) (err error) { pendingObjectDeleteSet: make(map[uint64]struct{}), checkPointControlChan: nil, inodeLeaseMap: make(map[uint64]*inodeLeaseStruct), - inodeLeaseLRU: list.New(), } globals.Lock() @@ -797,7 +791,7 @@ func (volume *volumeStruct) doCheckPoint() (err error) { startTime time.Time ) - volume.Lock() + globals.Lock() startTime = time.Now() @@ -805,7 +799,7 @@ func (volume *volumeStruct) doCheckPoint() (err error) { globals.stats.VolumeCheckPointUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) - volume.Unlock() + globals.Unlock() return } From 089f014d01d3cff2ee3d50bd7e89df83f346039d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 24 May 2021 13:47:36 -0700 Subject: [PATCH 010/258] Shortened imgr/imgrpkg::TestRPCLease()... 20 stacked exclusive leases is enough --- imgr/imgrpkg/lease_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imgr/imgrpkg/lease_test.go b/imgr/imgrpkg/lease_test.go index f81a248e..3ea6929c 100644 --- a/imgr/imgrpkg/lease_test.go +++ b/imgr/imgrpkg/lease_test.go @@ -22,7 +22,7 @@ const ( testRpcLeaseMultiFirstInodeNumber uint64 = 1 testRpcLeaseMultiNumInstances uint64 = 5 testRpcLeaseSingleInodeNumber uint64 = 1 - testRpcLeaseSingleNumInstances uint64 = 101 // Must be >= 4 + testRpcLeaseSingleNumInstances uint64 = 21 // 101 // Must be >= 4 testRpcLeaseTimeFormat = "15:04:05.000" ) From 40b64f0b26f5167d6fe5286e7a0db0daf796f8df Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 24 May 2021 13:57:50 -0700 Subject: [PATCH 011/258] Added .conf keys for imgr/imgrpkg to limit number of outstanding leases... --- imgr/imgr.conf | 2 ++ imgr/imgrpkg/api.go | 2 ++ imgr/imgrpkg/globals.go | 12 ++++++++++++ imgr/imgrpkg/utils_test.go | 2 ++ 4 files changed, 18 insertions(+) diff --git a/imgr/imgr.conf b/imgr/imgr.conf index a514e03a..f009b237 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -24,6 +24,8 @@ FetchNonceRangeToReturn: 100 MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 +LeaseEvictLowLimit: 100000 +LeaseEvictHighLimit: 100010 SwiftRetryDelay: 100ms SwiftRetryExpBackoff: 2 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 4eaca9b4..db3c3ef5 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -33,6 +33,8 @@ // MinLeaseDuration: 250ms // LeaseInterruptInterval: 250ms // LeaseInterruptLimit: 20 +// LeaseEvictLowLimit: 100000 +// LeaseEvictHighLimit: 100010 // // SwiftRetryDelay: 100ms // SwiftRetryExpBackoff: 2 diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 8fcf2d95..648a7689 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -43,6 +43,8 @@ type configStruct struct { MinLeaseDuration time.Duration LeaseInterruptInterval time.Duration LeaseInterruptLimit uint32 + LeaseEvictLowLimit uint64 + LeaseEvictHighLimit uint64 SwiftRetryDelay time.Duration SwiftRetryExpBackoff float64 @@ -378,6 +380,14 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { if nil != err { logFatal(err) } + globals.config.LeaseEvictLowLimit, err = confMap.FetchOptionValueUint64("IMGR", "LeaseEvictLowLimit") + if nil != err { + logFatal(err) + } + globals.config.LeaseEvictHighLimit, err = confMap.FetchOptionValueUint64("IMGR", "LeaseEvictHighLimit") + if nil != err { + logFatal(err) + } globals.config.SwiftRetryDelay, err = confMap.FetchOptionValueDuration("IMGR", "SwiftRetryDelay") if nil != err { @@ -472,6 +482,8 @@ func uninitializeGlobals() (err error) { globals.config.MinLeaseDuration = time.Duration(0) globals.config.LeaseInterruptInterval = time.Duration(0) globals.config.LeaseInterruptLimit = 0 + globals.config.LeaseEvictLowLimit = 0 + globals.config.LeaseEvictHighLimit = 0 globals.config.SwiftRetryDelay = time.Duration(0) globals.config.SwiftRetryExpBackoff = 0.0 diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index 88906070..8919103a 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -143,6 +143,8 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { "IMGR.MinLeaseDuration=250ms", "IMGR.LeaseInterruptInterval=250ms", "IMGR.LeaseInterruptLimit=5", + "IMGR.LeaseEvictLowLimit=100000", + "IMGR.LeaseEvictHighLimit=100010", "IMGR.SwiftRetryDelay=100ms", "IMGR.SwiftRetryExpBackoff=2", From ef7299c8eacdf3b5dd1576a676cd1c50bbd15f3a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 1 Jun 2021 10:40:52 -0700 Subject: [PATCH 012/258] Dropped unsupported stst(2) POSIX fields from InodeHeadV1Struct in package ilayout Also cleaned up "lint" complaints in package ilayout --- ilayout/api.go | 20 ++++---- ilayout/api_test.go | 12 ++--- ilayout/impl.go | 104 +++++++++++++++-------------------------- imgr/imgrpkg/volume.go | 4 +- 4 files changed, 51 insertions(+), 89 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 618eac0e..bbc943bb 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -342,17 +342,15 @@ type InodeHeadLayoutEntryV1Struct struct { // that is appended. // type InodeHeadV1Struct struct { - InodeNumber uint64 - InodeType uint8 // One of InodeType* - LinkTable []InodeLinkTableEntryStruct // List of Directory Entry references to this Inode - Size uint64 // Only applicable to File Inodes - CreationTime time.Time - ModificationTime time.Time - AccessTime time.Time - AttrChangeTime time.Time - Mode uint16 // Must be <= InodeModeMask (Note: does not include InodeType encoding) - UserID uint64 - GroupID uint64 + InodeNumber uint64 // + InodeType uint8 // One of InodeType* + LinkTable []InodeLinkTableEntryStruct // List of Directory Entry references to this Inode + Size uint64 // Only applicable to File Inodes + ModificationTime time.Time // In POSIX terms, equivalent to st_mtim: Time of last modification + StatusChangeTime time.Time // In POSIX terms, equivalent to st_ctim: Time of last status change + Mode uint16 // Must be <= InodeModeMask (Note: does not include InodeType encoding) + UserID uint64 // + GroupID uint64 // StreamTable []InodeStreamTableEntryStruct // List of Alternate Data Streams for this Inode PayloadObjectNumber uint64 // For Dir & File Inodes, identifies the Object containing the root of the Directory or ExtentMap B+Tree PayloadObjectOffset uint64 // For Dir & File Inodes, starting offset in the Object of the root of the Directory or ExtentMap B+Tree diff --git a/ilayout/api_test.go b/ilayout/api_test.go index f315e03b..986ade9f 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -86,10 +86,8 @@ func TestAPI(t *testing.T) { }, }, Size: 3, - CreationTime: testStartTime.AddDate(0, 0, -4), - ModificationTime: testStartTime.AddDate(0, 0, -3), - AccessTime: testStartTime.AddDate(0, 0, -1), - AttrChangeTime: testStartTime.AddDate(0, 0, -2), + ModificationTime: testStartTime.AddDate(0, 0, -2), + StatusChangeTime: testStartTime.AddDate(0, 0, -1), Mode: 0o456, UserID: 7, GroupID: 8, @@ -246,10 +244,8 @@ func TestAPI(t *testing.T) { (testInodeHeadV1.InodeType != unmarshaledInodeHeadV1.InodeType) || (len(testInodeHeadV1.LinkTable) != len(unmarshaledInodeHeadV1.LinkTable)) || (testInodeHeadV1.Size != unmarshaledInodeHeadV1.Size) || - (testInodeHeadV1.CreationTime != unmarshaledInodeHeadV1.CreationTime) || (testInodeHeadV1.ModificationTime != unmarshaledInodeHeadV1.ModificationTime) || - (testInodeHeadV1.AccessTime != unmarshaledInodeHeadV1.AccessTime) || - (testInodeHeadV1.AttrChangeTime != unmarshaledInodeHeadV1.AttrChangeTime) || + (testInodeHeadV1.StatusChangeTime != unmarshaledInodeHeadV1.StatusChangeTime) || (testInodeHeadV1.Mode != unmarshaledInodeHeadV1.Mode) || (testInodeHeadV1.UserID != unmarshaledInodeHeadV1.UserID) || (testInodeHeadV1.GroupID != unmarshaledInodeHeadV1.GroupID) || @@ -268,7 +264,7 @@ func TestAPI(t *testing.T) { } for streamTableIndex = range testInodeHeadV1.StreamTable { if (testInodeHeadV1.StreamTable[streamTableIndex].Name != unmarshaledInodeHeadV1.StreamTable[streamTableIndex].Name) || - (0 != bytes.Compare(testInodeHeadV1.StreamTable[streamTableIndex].Value, unmarshaledInodeHeadV1.StreamTable[streamTableIndex].Value)) { + !bytes.Equal(testInodeHeadV1.StreamTable[streamTableIndex].Value, unmarshaledInodeHeadV1.StreamTable[streamTableIndex].Value) { t.Fatalf("Bad unmarshaledInodeHeadV1 (%+v) - expected testInodeHeadV1 (%+v) [Case 3]", unmarshaledInodeHeadV1, testInodeHeadV1) } } diff --git a/ilayout/impl.go b/ilayout/impl.go index e80c8877..65bbc9df 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -27,7 +27,7 @@ func unmarshalCheckPointV1(checkPointV1String string) (checkPointV1 *CheckPointV _, err = fmt.Sscanf(checkPointV1String, "%016X %016X %016X %016X", &checkPointV1.Version, &checkPointV1.SuperBlockObjectNumber, &checkPointV1.SuperBlockLength, &checkPointV1.ReservedToNonce) if (nil == err) && (CheckPointVersionV1 != checkPointV1.Version) { - err = fmt.Errorf("Version mismatch... found %016X... expected %016X", checkPointV1.Version, CheckPointVersionV1) + err = fmt.Errorf("version mismatch... found %016X... expected %016X", checkPointV1.Version, CheckPointVersionV1) } return @@ -52,7 +52,7 @@ func (objectTrailer *ObjectTrailerStruct) marshalObjectTrailer() (objectTrailerB return } - curPos, err = putLEUint32ToBuf(objectTrailerBuf, curPos, objectTrailer.Length) + _, err = putLEUint32ToBuf(objectTrailerBuf, curPos, objectTrailer.Length) if nil != err { return } @@ -69,11 +69,11 @@ func unmarshalObjectTrailer(objectTrailerBuf []byte) (objectTrailer *ObjectTrail curPos = len(objectTrailerBuf) - (2 + 2 + 4) if curPos < 0 { - err = fmt.Errorf("No room for ObjectTrailerStruct at end of objectTrailerBuf") + err = fmt.Errorf("no room for ObjectTrailerStruct at end of objectTrailerBuf") return } if curPos > math.MaxUint32 { - err = fmt.Errorf("Cannot parse an objectTrailerBuf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) + err = fmt.Errorf("cannot parse an objectTrailerBuf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) return } @@ -97,12 +97,12 @@ func unmarshalObjectTrailer(objectTrailerBuf []byte) (objectTrailer *ObjectTrail } if curPos != len(objectTrailerBuf) { - err = fmt.Errorf("Logic error extracting ObjectTrailerStruct from end of objectTrailerBuf") + err = fmt.Errorf("logic error extracting ObjectTrailerStruct from end of objectTrailerBuf") return } if objectTrailer.Length != expectedLength { - err = fmt.Errorf("Payload of objectTrailerBuf preceeding ObjectTrailerStruct length mismatch") + err = fmt.Errorf("payload of objectTrailerBuf preceeding ObjectTrailerStruct length mismatch") return } @@ -175,7 +175,7 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ } if curPos > math.MaxUint32 { - err = fmt.Errorf("Cannot marshal an superBlockV1Buf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) + err = fmt.Errorf("cannot marshal an superBlockV1Buf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) return } @@ -190,7 +190,7 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ return } - curPos, err = putFixedByteSliceToBuf(superBlockV1Buf, curPos, objectTrailerBuf) + _, err = putFixedByteSliceToBuf(superBlockV1Buf, curPos, objectTrailerBuf) if nil != err { return } @@ -279,7 +279,7 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St } if curPos != int(objectTrailer.Length) { - err = fmt.Errorf("Incorrect size for superBlockV1Buf") + err = fmt.Errorf("incorrect size for superBlockV1Buf") return } @@ -312,7 +312,7 @@ func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) marshalInodeTableEnt return } - curPos, err = putLEUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadLength) + _, err = putLEUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadLength) if nil != err { return } @@ -334,7 +334,7 @@ func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTab return } if InodeTableEntryValueVersionV1 != inodeTableEntryValueVersion { - err = fmt.Errorf("Incorrect Version for inodeTableEntryValueV1Buf") + err = fmt.Errorf("incorrect Version for inodeTableEntryValueV1Buf") return } @@ -375,7 +375,7 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt inodeHeadV1BufLen += 8 + 8 + len(inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName) } - inodeHeadV1BufLen += 8 + 8 + 8 + 8 + 8 + 2 + 8 + 8 + inodeHeadV1BufLen += 8 + 8 + 8 + 2 + 8 + 8 inodeHeadV1BufLen += 8 @@ -427,22 +427,12 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt return } - curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.CreationTime.UnixNano())) - if nil != err { - return - } - curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.ModificationTime.UnixNano())) if nil != err { return } - curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.AccessTime.UnixNano())) - if nil != err { - return - } - - curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.AttrChangeTime.UnixNano())) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.StatusChangeTime.UnixNano())) if nil != err { return } @@ -522,7 +512,7 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt } if curPos > math.MaxUint32 { - err = fmt.Errorf("Cannot marshal an inodeHeadV1Buf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) + err = fmt.Errorf("cannot marshal an inodeHeadV1Buf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) return } @@ -537,7 +527,7 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt return } - curPos, err = putFixedByteSliceToBuf(inodeHeadV1Buf, curPos, objectTrailerBuf) + _, err = putFixedByteSliceToBuf(inodeHeadV1Buf, curPos, objectTrailerBuf) if nil != err { return } @@ -548,9 +538,6 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { var ( - accessTimeAsUnixTimeInNs uint64 - attrChangeTimeAsUnixTimeInNs uint64 - creationTimeAsUnixTimeInNs uint64 curPos int layoutIndex uint64 layoutLen uint64 @@ -558,6 +545,7 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct linkTableLen uint64 modificationTimeAsUnixTimeInNs uint64 objectTrailer *ObjectTrailerStruct + statusChangeTimeAsUnixTimeInNs uint64 streamTableIndex uint64 streamTableLen uint64 ) @@ -613,13 +601,6 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct return } - creationTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) - if nil != err { - return - } - - inodeHeadV1.CreationTime = time.Unix(0, int64(creationTimeAsUnixTimeInNs)) - modificationTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return @@ -627,19 +608,12 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct inodeHeadV1.ModificationTime = time.Unix(0, int64(modificationTimeAsUnixTimeInNs)) - accessTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) - if nil != err { - return - } - - inodeHeadV1.AccessTime = time.Unix(0, int64(accessTimeAsUnixTimeInNs)) - - attrChangeTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) + statusChangeTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.AttrChangeTime = time.Unix(0, int64(attrChangeTimeAsUnixTimeInNs)) + inodeHeadV1.StatusChangeTime = time.Unix(0, int64(statusChangeTimeAsUnixTimeInNs)) inodeHeadV1.Mode, curPos, err = getLEUint16FromBuf(inodeHeadV1Buf, curPos) if nil != err { @@ -720,7 +694,7 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct } if curPos != int(objectTrailer.Length) { - err = fmt.Errorf("Incorrect size for inodeHeadV1Buf") + err = fmt.Errorf("incorrect size for inodeHeadV1Buf") return } @@ -742,7 +716,7 @@ func (directoryEntryValueV1 *DirectoryEntryValueV1Struct) marshalDirectoryEntryV return } - curPos, err = putLEUint8ToBuf(directoryEntryValueV1Buf, curPos, directoryEntryValueV1.InodeType) + _, err = putLEUint8ToBuf(directoryEntryValueV1Buf, curPos, directoryEntryValueV1.InodeType) if nil != err { return } @@ -800,7 +774,7 @@ func (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct) marshalExtentMapEntryV return } - curPos, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.ObjectOffset) + _, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.ObjectOffset) if nil != err { return } @@ -848,7 +822,7 @@ func getLEUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error nextPos = curPos + 1 if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for uint8") + err = fmt.Errorf("insufficient space in buf[curPos:] for uint8") return } @@ -862,7 +836,7 @@ func putLEUint8ToBuf(buf []byte, curPos int, u8 uint8) (nextPos int, err error) nextPos = curPos + 1 if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for uint8") + err = fmt.Errorf("insufficient space in buf[curPos:] for uint8") return } @@ -876,7 +850,7 @@ func getLEUint16FromBuf(buf []byte, curPos int) (u16 uint16, nextPos int, err er nextPos = curPos + 2 if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for uint16") + err = fmt.Errorf("insufficient space in buf[curPos:] for uint16") return } @@ -889,7 +863,7 @@ func putLEUint16ToBuf(buf []byte, curPos int, u16 uint16) (nextPos int, err erro nextPos = curPos + 2 if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for uint16") + err = fmt.Errorf("insufficient space in buf[curPos:] for uint16") return } @@ -904,7 +878,7 @@ func getLEUint32FromBuf(buf []byte, curPos int) (u32 uint32, nextPos int, err er nextPos = curPos + 4 if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for uint32") + err = fmt.Errorf("insufficient space in buf[curPos:] for uint32") return } @@ -918,7 +892,7 @@ func putLEUint32ToBuf(buf []byte, curPos int, u32 uint32) (nextPos int, err erro nextPos = curPos + 4 if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for uint32") + err = fmt.Errorf("insufficient space in buf[curPos:] for uint32") return } @@ -935,7 +909,7 @@ func getLEUint64FromBuf(buf []byte, curPos int) (u64 uint64, nextPos int, err er nextPos = curPos + 8 if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for uint64") + err = fmt.Errorf("insufficient space in buf[curPos:] for uint64") return } @@ -949,7 +923,7 @@ func putLEUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err erro nextPos = curPos + 8 if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for uint64") + err = fmt.Errorf("insufficient space in buf[curPos:] for uint64") return } @@ -977,7 +951,7 @@ func getLEStringFromBuf(buf []byte, curPos int) (str string, nextPos int, err er } if strLen > (uint64(len(buf)) - uint64(nextPos)) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for string of reported length") + err = fmt.Errorf("insufficient space in buf[curPos:] for string of reported length") return } @@ -998,7 +972,7 @@ func putLEStringToBuf(buf []byte, curPos int, str string) (nextPos int, err erro nextPos += len(str) if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for string") + err = fmt.Errorf("insufficient space in buf[curPos:] for string") return } @@ -1019,7 +993,7 @@ func getLEByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos in } if byteSliceLen > (uint64(len(buf)) - uint64(nextPos)) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for []byte of reported length") + err = fmt.Errorf("insufficient space in buf[curPos:] for []byte of reported length") return } @@ -1041,7 +1015,7 @@ func putLEByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, nextPos += len(byteSlice) if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for []byte") + err = fmt.Errorf("insufficient space in buf[curPos:] for []byte") return } @@ -1053,13 +1027,11 @@ func putLEByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, func getFixedByteSliceFromBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { var ( - byteSliceLen int + byteSliceLen int = len(byteSlice) ) - byteSliceLen = len(byteSlice) - if byteSliceLen < (len(buf) - curPos) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for []byte of len(byteSlice) length") + err = fmt.Errorf("insufficient space in buf[curPos:] for []byte of len(byteSlice) length") return } @@ -1072,15 +1044,13 @@ func getFixedByteSliceFromBuf(buf []byte, curPos int, byteSlice []byte) (nextPos func putFixedByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { var ( - byteSliceLen int + byteSliceLen int = len(byteSlice) ) - byteSliceLen = len(byteSlice) - nextPos = curPos + byteSliceLen if nextPos > len(buf) { - err = fmt.Errorf("Insufficient space in buf[curPos:] for []byte") + err = fmt.Errorf("insufficient space in buf[curPos:] for []byte") return } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 6ef594b4..d42e00bd 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -592,10 +592,8 @@ func postVolume(storageURL string, authToken string) (err error) { }, }, Size: 0, - CreationTime: timeNow, ModificationTime: timeNow, - AccessTime: timeNow, - AttrChangeTime: timeNow, + StatusChangeTime: timeNow, Mode: ilayout.InodeModeMask, UserID: 0, GroupID: 0, From 5f0f08705b4c04d5d5615d3a0f6423872c59aa53 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 1 Jun 2021 16:40:50 -0700 Subject: [PATCH 013/258] Added test of imgrpkg's RenewMount() RPC Also fixed double-unlock in imgrpkg.renewMount() --- imgr/imgrpkg/retry-rpc.go | 2 -- imgr/imgrpkg/retry-rpc_test.go | 22 +++++++++++++++------- imgr/imgrpkg/utils_test.go | 34 ++++++++++++++++++++++------------ 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index cf5855e6..64f0c727 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -223,8 +223,6 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * volume = mount.volume - globals.Unlock() - mount.authToken = renewMountRequest.AuthToken _, err = swiftObjectGet(mount.volume.storageURL, mount.authToken, ilayout.CheckPointObjectNumber) diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index 4d9014ad..063f1cd8 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/NVIDIA/proxyfs/iswift/iswiftpkg" "github.com/NVIDIA/proxyfs/retryrpc" ) @@ -133,14 +134,9 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) should have failed") } - // TODO: Remove this early exit skipping of following TODOs - - if nil != err { - t.Logf("Exiting TestRetryRPC() early to skip following TODOs") - return - } + // Force a need for a re-auth - // TODO: Force a need for a re-auth + iswiftpkg.ForceReAuth() // Attempt a GetInodeTableEntry() for RootDirInode... which should fail (re-auth required) @@ -157,6 +153,11 @@ func TestRetryRPC(t *testing.T) { // Perform a RenewMount() + err = testDoAuth() + if nil != err { + t.Fatalf("testDoAuth() failed: %v", err) + } + renewMountRequest = &RenewMountRequestStruct{ MountID: mountResponse.MountID, AuthToken: testGlobals.authToken, @@ -168,6 +169,13 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"RenewMount(,)\",,) failed: %v", err) } + // TODO: Remove this early exit skipping of following TODOs + + if nil == err { + t.Logf("Exiting TestRetryRPC() early to skip following TODOs") + return + } + // Fetch a Shared Lease on RootDirInode leaseRequest = &LeaseRequestStruct{ diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index 8919103a..de4d313a 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -56,8 +56,6 @@ var testGlobals *testGlobalsStruct func testSetup(t *testing.T, retryrpcCallbacks interface{}) { var ( - authRequestHeaders http.Header - authResponseHeaders http.Header confStrings []string err error putAccountRequestHeaders http.Header @@ -183,19 +181,11 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { t.Fatalf("iswifpkg.Start(testGlobals.confMap) failed: %v", err) } - authRequestHeaders = make(http.Header) - - authRequestHeaders["X-Auth-User"] = []string{testSwiftAuthUser} - authRequestHeaders["X-Auth-Key"] = []string{testSwiftAuthKey} - - authResponseHeaders, _, err = testDoHTTPRequest("GET", testGlobals.authURL, authRequestHeaders, nil) + err = testDoAuth() if nil != err { - t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.authURL, authRequestHeaders) failed: %v", err) + t.Fatalf("testDoAuth() failed: %v", err) } - testGlobals.authToken = authResponseHeaders.Get("X-Auth-Token") - testGlobals.accountURL = authResponseHeaders.Get("X-Storage-Url") - testGlobals.containerURL = testGlobals.accountURL + "/" + testContainer putAccountRequestHeaders = make(http.Header) @@ -310,3 +300,23 @@ func testDoHTTPRequest(method string, url string, requestHeaders http.Header, re err = nil return } + +func testDoAuth() (err error) { + var ( + authRequestHeaders http.Header + authResponseHeaders http.Header + ) + + authRequestHeaders = make(http.Header) + + authRequestHeaders["X-Auth-User"] = []string{testSwiftAuthUser} + authRequestHeaders["X-Auth-Key"] = []string{testSwiftAuthKey} + + authResponseHeaders, _, err = testDoHTTPRequest("GET", testGlobals.authURL, authRequestHeaders, nil) + if nil == err { + testGlobals.authToken = authResponseHeaders.Get("X-Auth-Token") + testGlobals.accountURL = authResponseHeaders.Get("X-Storage-Url") + } + + return +} From f867c66b10d39d86536debe2fc5eed6bba9936eb Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 1 Jun 2021 16:44:35 -0700 Subject: [PATCH 014/258] Added test of 1st imgrpkg (Shared) LeaseRequest --- imgr/imgrpkg/retry-rpc_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index 063f1cd8..1acc1336 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -169,13 +169,6 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"RenewMount(,)\",,) failed: %v", err) } - // TODO: Remove this early exit skipping of following TODOs - - if nil == err { - t.Logf("Exiting TestRetryRPC() early to skip following TODOs") - return - } - // Fetch a Shared Lease on RootDirInode leaseRequest = &LeaseRequestStruct{ @@ -203,6 +196,13 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) failed: %v", err) } + // TODO: Remove this early exit skipping of following TODOs + + if nil == err { + t.Logf("Exiting TestRetryRPC() early to skip following TODOs") + return + } + // TODO: Attempt a PutInodeTableEntries() for RootDirInode... which should fail (only Shared Lease) // Perform a Lease Promote on RootDirInode From 2236d0a2aba4b8d6e0360c05b2e3080bfe7acea9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 2 Jun 2021 16:55:12 -0700 Subject: [PATCH 015/258] Flushed out remaining imgrpkg PutInodeTableEntriesRequestStruct content This marks the completion of the RetryRPC supported by imgrpkg :-) --- imgr/imgrpkg/api.go | 49 +++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index db3c3ef5..4785be86 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -2,10 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 // Package imgrpkg implements the server side Inode Management for ProxyFS volumes. -// The package supports to communications protocols. The balance of the package -// documentation will describe the Go-callable API. Note that func's listed under -// type RetryRPCServerStruct are the RPCs issued by the client side via package -// retryrpc connections (and, thus, not intended to be called directly). +// While the package provides a small set of Go-callable APIs, the bulk of its +// functionality is accessed via package retryrpc-exposed RPCs. While these RPCs +// reference active volumes known to an imgrpkg instance, a RESTful API is provided +// to specify those active volumes. +// +// Note that func's listed under type RetryRPCServerStruct are the RPCs issued by +// the client side via package retryrpc connections (and, thus, not intended to be +// called directly). // // To configure an imgrpkg instance, Start() is called passing, as the sole // argument, a package conf ConfMap. Here is a sample .conf file: @@ -63,9 +67,8 @@ // files, the retryrpc package will be configured to use TLS. In any event, // the RPCs will be available via :. // -// In addition to the package retryrpc-exposed RPCs, the package also includes -// an embedded HTTP Server (at URL http://:) -// responses to the following: +// The RESTful API is provided by an embedded HTTP Server +// (at URL http://:) responsing to the following: // // DELETE /volume/ // @@ -267,11 +270,33 @@ func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest return getInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) } -// PutInodeTableEntriesRequestStruct is the request object for PutInodeTableEntries. +// PutInodeTableEntryStruct is used to indicate the change to an individual +// InodeTableEntry as part of the collection of changes in a PutInodeTablesEntries +// request (which must have an active Exclusive Lease granted to the MountID). +// +type PutInodeTableEntryStruct struct { + InodeNumber uint64 + InodeHeadObjectNumber uint64 + InodeHeadLength uint64 +} + +// PutInodeTableEntriesRequestStruct is the request object for PutInodeTableEntries +// (which must have an active Exclusive Lease for every PutInodeTableEntryStruct.InodeNumber +// granted to the MountID). +// +// Note that dereferenced objects listed in the DereferencedObjectNumberArray will +// not be deleted until the next CheckPoint is performed. +// +// The SuperBlockInode{ObjectCount|ObjectSize|BytesReferenced}Adjustment fields +// are used to update the corresponding fields in the volume's SuperBlock. // type PutInodeTableEntriesRequestStruct struct { - MountID string - // TODO + MountID string + UpdatedInodeTableEntryArray []PutInodeTableEntryStruct + DereferencedObjectNumberArray []uint64 + SuperBlockInodeObjectCountAdjustment int64 + SuperBlockInodeObjectSizeAdjustment int64 + SuperBlockInodeBytesReferencedAdjustment int64 } // PutInodeTableEntriesResponseStruct is the response object for PutInodeTableEntries. @@ -294,9 +319,7 @@ type DeleteInodeTableEntryRequestStruct struct { // DeleteInodeTableEntryResponseStruct is the response object for DeleteInodeTableEntry. // -type DeleteInodeTableEntryResponseStruct struct { - // TODO -} +type DeleteInodeTableEntryResponseStruct struct{} // DeleteInodeTableEntry requests the specified Inode information be deleted. // An active Exclusive Lease must be granted to the MountID. Note that From 8add0df7aa5548f91bec21b40e83318592c25987 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Thu, 17 Jun 2021 14:48:15 -0700 Subject: [PATCH 016/258] Cherry pick Craig's benchmarks and tests for trying different encoding techniques (#628) * Log a warning if Dial failed. * Add tests and buckestats to measure JSON en/decoding CPU time. Add tests to benchmark the time required to encode and decode LeaseREquest/Reply pairs using different encoding schemes. BenchmarkRpcLeaseEncodeBinary/JSON/Gob() perform all of the work to: * encode a LeaseRequest, wrap it with a jsonRequest and also encode the ioRequest header (ordinarily done by the client) * encode a LeaseReply, wrap it with a jsonReply and also encode the ioReply header (ordinarily done by the server) They use a hand-written binary encoding, JSON encoding, and gob encoding, respectively. BenchmarkRpcLeaseDecodeBinary/JSON/Gob() perform all of the work to: * decode a LeaseRequest, wrap it with a jsonRequest and also decode the ioRequest header (ordinarily done by the server) * decode a LeaseReply, wrap it with a jsonReply and also decode the ioReply header (ordinarily done by the client) They use a hand-written binary encoding, JSON encoding, and gob encoding, respectively. Each of these measure the full CPU time required to encode request/reply or decode request/reply for the respective encodings. Also add tests to verify the encoding and decoding routines match. Add bucketstats to each encode/decode of JSON in benchmarkRpcLeaseShortcutServer (but did not add them to the corresponding encode/decode in the shortcut client). Also clean up the output to stop printing the measured bucketstats for the lease test runs. The calls to bucketstats.SprintfStats() are commented out. * Add Craig's benchmarks for encoding of retryrpc Co-authored-by: craig harmer --- jrpcfs/encoding_test.go | 1140 +++++++++++++++++++++++++++++++++ jrpcfs/lease_test.go | 45 +- jrpcfs/setup_teardown_test.go | 7 + retryrpc/client.go | 1 + 4 files changed, 1190 insertions(+), 3 deletions(-) create mode 100644 jrpcfs/encoding_test.go diff --git a/jrpcfs/encoding_test.go b/jrpcfs/encoding_test.go new file mode 100644 index 00000000..5c68fcbe --- /dev/null +++ b/jrpcfs/encoding_test.go @@ -0,0 +1,1140 @@ +//Allocate response/ Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package jrpcfs + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +// The encoding tests want to experiment with various ways to encode the same +// RPC requests sent over the wire. The retryrpc code wraps an RPC request with +// jsonRequest and then prepends ioHeader before sending it over the wire. +// Replies get the same treatment. +// +// Copy jsonRequest, ioRequest, jsonReply, ioReply, and some routines that fill +// them in from the retryrpc code so we can experiment with different ways of +// encoding & decoding them. +type requestID uint64 +type MsgType uint16 + +type jsonRequest struct { + MyUniqueID uint64 `json:"myuniqueid"` // ID of client + RequestID requestID `json:"requestid"` // ID of this request + HighestReplySeen requestID `json:"highestReplySeen"` // Used to trim completedRequests on server + Method string `json:"method"` + Params [1]interface{} `json:"params"` +} + +// jsonReply is used to marshal an RPC response in/out of JSON +type jsonReply struct { + MyUniqueID uint64 `json:"myuniqueid"` // ID of client + RequestID requestID `json:"requestid"` // ID of this request + ErrStr string `json:"errstr"` + Result interface{} `json:"result"` +} + +// ioRequest tracks fields written on wire +type ioRequestRetryRpc struct { + Hdr ioHeader + JReq []byte // JSON containing request +} + +// ioReply is the structure returned over the wire +type ioReplyRetryRpc struct { + Hdr ioHeader + JResult []byte // JSON containing response +} + +type ioHeader struct { + Len uint32 // Number of bytes following header + Protocol uint16 + Version uint16 + Type MsgType + Magic uint32 // Magic number - if invalid means have not read complete header +} + +// Encode a LeaseRequest to binary, json, or some other form. +type encodeLeaseRequestFunc func(leaseReq *LeaseRequest, jreq *jsonRequest) (hdrBytes []byte, jsonBytes []byte) + +// Decode a LeaseRequest from binary, json, or some other form. +type decodeLeaseRequestFunc func(hdrBytes []byte, jsonBytes []byte) (leaseReq *LeaseRequest, jreq *jsonRequest) + +// Encode a LeaseRequest to binary, json, or some other form. +type encodeLeaseReplyFunc func(leaseReply *LeaseReply, jreply *jsonReply) (hdrBytes []byte, jsonBytes []byte) + +// Decode a LeaseRequest from binary, json, or some other form. +type decodeLeaseReplyFunc func(hdrBytes []byte, jsonBytes []byte) (leaseReply *LeaseReply, jreply *jsonReply) + +// Test that the request and reply decode functions match the encode functions +func TestEncodeDecodeBinary(t *testing.T) { + testEncodeDecodeFunctions(t, + encodeLeaseRequestBinary, encodeLeaseReplyBinary, + decodeLeaseRequestBinary, decodeLeaseReplyBinary) +} + +// Test that the request and reply decode functions match the encode functions +func TestEncodeDecodeJSON(t *testing.T) { + testEncodeDecodeFunctions(t, + encodeLeaseRequestJson, encodeLeaseReplyJson, + decodeLeaseRequestJson, decodeLeaseReplyJson) +} + +// Test that the request and reply decode functions match the encode functions +func TestEncodeDecodeGob(t *testing.T) { + testEncodeDecodeFunctions(t, + encodeLeaseRequestGob, encodeLeaseReplyGob, + decodeLeaseRequestGob, decodeLeaseReplyGob) +} + +// BenchmarkRpcLeaseEncodeBinary emulates the Lease request/reply RPC encoding +// by building the structures and simply calling the routines that do a binary +// encoding. +func BenchmarkRpcLeaseEncodeBinary(b *testing.B) { + benchmarkRpcLeaseEncode(b, + encodeLeaseRequestBinary, encodeLeaseReplyBinary) +} + +// BenchmarkRpcLeaseEncodeJSON emulates the Lease request/reply RPC encoding +// by building the structures and simply calling the routines that do a json +// encoding. +func BenchmarkRpcLeaseEncodeJSON(b *testing.B) { + benchmarkRpcLeaseEncode(b, + encodeLeaseRequestJson, encodeLeaseReplyJson) +} + +// BenchmarkRpcLeaseEncodeGob emulates the Lease request/reply RPC encoding +// by building the structures and simply calling the routines that do a gob +// encoding. +func BenchmarkRpcLeaseEncodeGob(b *testing.B) { + benchmarkRpcLeaseEncode(b, + encodeLeaseRequestGob, encodeLeaseReplyGob) +} + +// BenchmarkRpcLeaseDecodeBinary emulates the Lease request/reply RPC encoding +// by building the structures and simply calling the routines that do a binary +// encoding. +func BenchmarkRpcLeaseDecodeBinary(b *testing.B) { + benchmarkRpcLeaseDecode(b, + encodeLeaseRequestJson, encodeLeaseReplyJson, + decodeLeaseRequestJson, decodeLeaseReplyJson) +} + +// BenchmarkRpcLeaseDecodeJSON emulates the Lease request/reply RPC encoding +// by building the structures and simply calling the routines that do a binary +// encoding. +func BenchmarkRpcLeaseDecodeJSON(b *testing.B) { + benchmarkRpcLeaseDecode(b, + encodeLeaseRequestJson, encodeLeaseReplyJson, + decodeLeaseRequestJson, decodeLeaseReplyJson) +} + +// BenchmarkRpcLeaseDecodeGob emulates the Lease request/reply RPC encoding +// by building the structures and simply calling the routines that do a binary +// encoding. +func BenchmarkRpcLeaseDecodeGob(b *testing.B) { + benchmarkRpcLeaseDecode(b, + encodeLeaseRequestGob, encodeLeaseReplyGob, + decodeLeaseRequestGob, decodeLeaseReplyGob) +} + +// testEncodeDecodeFunctions tests "generic" encoding and decoding functions for +// LeaseRequests to see if the decoded requests match what was encoded. +func testEncodeDecodeFunctions(t *testing.T, + encodeLeaseRequest encodeLeaseRequestFunc, + encodeLeaseReply encodeLeaseReplyFunc, + decodeLeaseRequest decodeLeaseRequestFunc, + decodeLeaseReply decodeLeaseReplyFunc) { + + assert := assert.New(t) + + // encode a lease request and then decode it + mountByAccountNameReply := &MountByAccountNameReply{ + MountID: "66", + } + leaseRequest := &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeExclusive, + } + jsonRequest := newJsonRequest("RpcLease") + + // this marshals both the RPCrequest and the header for the request + leaseRequestHdrBuf, leaseRequestPayloadBuf := + encodeLeaseRequest(leaseRequest, jsonRequest) + + leaseRequest2, jsonRequest2 := + decodeLeaseRequest(leaseRequestHdrBuf, leaseRequestPayloadBuf) + + assert.Equal(leaseRequest, leaseRequest2, "Decoded struct should match the original") + assert.Equal(jsonRequest, jsonRequest2, "Decoded struct should match the original") + + // try again with release lease request + leaseRequest.LeaseRequestType = LeaseRequestTypeRelease + + leaseRequestHdrBuf, leaseRequestPayloadBuf = + encodeLeaseRequest(leaseRequest, jsonRequest) + + leaseRequest2, jsonRequest2 = + decodeLeaseRequest(leaseRequestHdrBuf, leaseRequestPayloadBuf) + + assert.Equal(leaseRequest, leaseRequest2, "Decoded struct should match the original") + assert.Equal(jsonRequest, jsonRequest2, "Decoded struct should match the original") + + // encode reply and decode it + leaseReply := &LeaseReply{ + LeaseReplyType: LeaseReplyTypeExclusive, + } + jsonReply := newJsonReply() + leaseReplyHdrBuf, leaseReplyPayloadBuf := + encodeLeaseReply(leaseReply, jsonReply) + + leaseReply2, jsonReply2 := + decodeLeaseReply(leaseReplyHdrBuf, leaseReplyPayloadBuf) + + assert.Equal(leaseReply, leaseReply2, "Decoded struct should match the original") + assert.Equal(jsonReply, jsonReply2, "Decoded struct should match the original") + + // try again with release lease reply + leaseReply.LeaseReplyType = LeaseReplyTypeReleased + + leaseReplyHdrBuf, leaseReplyPayloadBuf = + encodeLeaseReply(leaseReply, jsonReply) + + leaseReply2, jsonReply2 = + decodeLeaseReply(leaseReplyHdrBuf, leaseReplyPayloadBuf) + + assert.Equal(leaseReply, leaseReply2, "Decoded struct should match the original") + assert.Equal(jsonReply, jsonReply2, "Decoded struct should match the original") +} + +// benchmarkRpcLeaseEncode is the guts to benchmark the cost of encoding the LeaseRequest +// and LeaseReply parts of a LeaseRequest RPC. +func benchmarkRpcLeaseEncode(b *testing.B, + encodeLeaseRequest encodeLeaseRequestFunc, + encodeLeaseReply encodeLeaseReplyFunc) { + + var ( + benchmarkIteration int + leaseReply *LeaseReply + mountByAccountNameReply *MountByAccountNameReply + ) + + mountByAccountNameReply = &MountByAccountNameReply{ + MountID: "66", + } + + b.ResetTimer() + + for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { + leaseRequest := &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeExclusive, + } + + // this marshals both the RPCrequest and the header for the request + hdrBuf, requestBuf := encodeLeaseRequest(leaseRequest, newJsonRequest("RpcLease")) + _ = hdrBuf + _ = requestBuf + + leaseReply = &LeaseReply{ + LeaseReplyType: LeaseReplyTypeExclusive, + } + + // marshal the RPC reply + hdrBuf, replyBuf := encodeLeaseReply(leaseReply, newJsonReply()) + _ = hdrBuf + _ = replyBuf + + // "validate" the returned value + if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReply.LeaseReplyType) + } + + // release the lock + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeRelease, + } + + hdrBuf, requestBuf = encodeLeaseRequest(leaseRequest, newJsonRequest("RpcLease")) + _ = hdrBuf + _ = requestBuf + + leaseReply = &LeaseReply{ + LeaseReplyType: LeaseReplyTypeReleased, + } + + hdrBuf, replyBuf = encodeLeaseReply(leaseReply, newJsonReply()) + _ = hdrBuf + _ = replyBuf + + // "validate" the return value + if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReply.LeaseReplyType) + } + } + + b.StopTimer() +} + +// benchmarkRpcLeaseEncode is the guts to benchmark the cost of decoding the +// LeaseRequest and LeaseReply parts of a LeaseRequest RPC. +func benchmarkRpcLeaseDecode(b *testing.B, + encodeLeaseRequest encodeLeaseRequestFunc, + encodeLeaseReply encodeLeaseReplyFunc, + decodeLeaseRequest decodeLeaseRequestFunc, + decodeLeaseReply decodeLeaseReplyFunc) { + + var ( + benchmarkIteration int + leaseReply *LeaseReply + mountByAccountNameReply *MountByAccountNameReply + ) + + mountByAccountNameReply = &MountByAccountNameReply{ + MountID: "6", + } + + // build the over-the-wire buffers that will be decoded later + // + // get lease request first + leaseRequest := &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeExclusive, + } + + // this marshals both the RPCrequest and the header for the request + getLeaseRequestHdrBuf, getLeaseRequestPayloadBuf := + encodeLeaseRequest(leaseRequest, newJsonRequest("RpcLease")) + + // release lease request + leaseRequest.LeaseRequestType = LeaseRequestTypeRelease + releaseLeaseRequestHdrBuf, releaseLeaseRequestPayloadBuf := + encodeLeaseRequest(leaseRequest, newJsonRequest("RpcLease")) + + // now build the replies + // get lease reply + leaseReply = &LeaseReply{ + LeaseReplyType: LeaseReplyTypeExclusive, + } + getLeaseReplyHdrBuf, getLeaseReplyPayloadBuf := + encodeLeaseReply(leaseReply, newJsonReply()) + + // release lease reply + leaseReply.LeaseReplyType = LeaseReplyTypeReleased + releaseLeaseReplyHdrBuf, releaseLeaseReplyPayloadBuf := + encodeLeaseReply(leaseReply, newJsonReply()) + + b.ResetTimer() + + for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { + + leaseRequest, jRequest := + decodeLeaseRequest(getLeaseRequestHdrBuf, getLeaseRequestPayloadBuf) + _ = leaseRequest + _ = jRequest + + leaseReply, jReply := + decodeLeaseReply(getLeaseReplyHdrBuf, getLeaseReplyPayloadBuf) + _ = jReply + + // "validate" the returned value + if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", + leaseReply.LeaseReplyType) + } + + // release the lock + leaseRequest, jRequest = + decodeLeaseRequest(releaseLeaseRequestHdrBuf, releaseLeaseRequestPayloadBuf) + + leaseReply, jReply = + decodeLeaseReply(releaseLeaseReplyHdrBuf, releaseLeaseReplyPayloadBuf) + + // "validate" the return value + if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", + leaseReply.LeaseReplyType) + } + } + + b.StopTimer() +} + +func newJsonRequest(method string) (jreq *jsonRequest) { + jreq = &jsonRequest{ + MyUniqueID: 77, + RequestID: 88, + HighestReplySeen: 33, + Method: method, + } + + return +} + +func newJsonReply() (jreply *jsonReply) { + jreply = &jsonReply{ + MyUniqueID: 77, + RequestID: 88, + ErrStr: "", + } + + return +} + +// Perform all of the encoding required by the client to send a LeaseRequest. +// This includes encoding the LeaseRequest, the jsonRequest "wrapper", and the +// ioRequest header (which is always a fixed binary encoding). +// +// The encoding of the jsonRequest and LeaseRequest also use a hand-written +// binary encoding. +func encodeLeaseRequestBinary(leaseReq *LeaseRequest, jreq *jsonRequest) (hdrBytes []byte, jsonBytes []byte) { + var err error + + hdrBuf := &bytes.Buffer{} + jsonBuf := &bytes.Buffer{} + + // marshal jsonRequest fields + err = binary.Write(jsonBuf, binary.LittleEndian, jreq.MyUniqueID) + if err != nil { + panic("jreq.MyUniqueID") + } + err = binary.Write(jsonBuf, binary.LittleEndian, jreq.RequestID) + if err != nil { + panic("jreq.RequestID") + } + err = binary.Write(jsonBuf, binary.LittleEndian, jreq.HighestReplySeen) + if err != nil { + panic("jreq.HighestReplySeen") + } + // use a Nul terminated string + methodWithNul := jreq.Method + string([]byte{0}) + _, err = jsonBuf.WriteString(methodWithNul) + if err != nil { + panic("jreq.Method") + } + + // marshal LeaseRequest fields + mountIDWithNul := leaseReq.MountID + MountIDAsString([]byte{0}) + _, err = jsonBuf.WriteString(string(mountIDWithNul)) + if err != nil { + panic("leaseReq.MountID") + } + err = binary.Write(jsonBuf, binary.LittleEndian, leaseReq.InodeNumber) + if err != nil { + panic("leaseReq.LeaseInodeNumber") + } + err = binary.Write(jsonBuf, binary.LittleEndian, leaseReq.LeaseRequestType) + if err != nil { + panic("leaseReq.LeaseRequestType") + } + + // now create the IoRequest header and Marshal it + ioReq := ioRequestRetryRpc{ + Hdr: ioHeader{ + Len: uint32(jsonBuf.Len()), + Protocol: uint16(1), + Version: 1, + Type: 1, + Magic: 0xCAFEFEED, + }, + } + + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Len) + if err != nil { + panic("ioReq.Hdr.Len") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Protocol) + if err != nil { + panic("ioReq.Hdr.Protocol") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Version) + if err != nil { + panic("ioReq.Hdr.Version") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Type) + if err != nil { + panic("ioReq.Hdr.Type") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Magic) + if err != nil { + panic("ioReq.Hdr.Magic") + } + + hdrBytes = hdrBuf.Bytes() + jsonBytes = jsonBuf.Bytes() + return +} + +// Perform all of the encoding required on the server to send a LeaseReply. +// This includes encoding the LeaseReply, the jsonReply "wrapper", and the +// ioReply header (which is always a fixed binary encoding). +// +// The encoding of the jsonReply and LeaseReply also use a hand-written binary +// encoding. +func encodeLeaseReplyBinary(leaseReply *LeaseReply, jreply *jsonReply) (hdrBytes []byte, jsonBytes []byte) { + var err error + + hdrBuf := &bytes.Buffer{} + jsonBuf := &bytes.Buffer{} + + // marshal jsonReply fields + err = binary.Write(jsonBuf, binary.LittleEndian, jreply.MyUniqueID) + if err != nil { + panic("jreply.MyUniqueID") + } + err = binary.Write(jsonBuf, binary.LittleEndian, jreply.RequestID) + if err != nil { + panic("jreply.RequestID") + } + // use a Nul termianted string + errStrWithNul := jreply.ErrStr + string([]byte{0}) + _, err = jsonBuf.WriteString(errStrWithNul) + if err != nil { + panic("jreply.ErrStr") + } + + // marshal LeaseReply fields + err = binary.Write(jsonBuf, binary.LittleEndian, leaseReply.LeaseReplyType) + if err != nil { + panic("leaseReply.LeaseReplyType") + } + + // now create the IoReply header and Marshal it + ioReply := ioReplyRetryRpc{ + Hdr: ioHeader{ + Len: uint32(jsonBuf.Len()), + Protocol: uint16(1), + Version: 1, + Type: 1, + Magic: 0xCAFEFEED, + }, + } + + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Len) + if err != nil { + panic("ioReply.Hdr.Len") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Protocol) + if err != nil { + panic("ioReply.Hdr.Protocol") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Version) + if err != nil { + panic("ioReply.Hdr.Version") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Type) + if err != nil { + panic("ioReply.Hdr.Type") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Magic) + if err != nil { + panic("ioReply.Hdr.Magic") + } + + hdrBytes = hdrBuf.Bytes() + jsonBytes = jsonBuf.Bytes() + return +} + +// Perform all of the decoding required on the server to recieve a LeaseRequest. +// This includes decoding the LeaseRequest, the jsonRequest "wrapper", and the +// ioRequest header (which is always a fixed binary encoding). +// +// The decoding of the jsonRequest and LeaseRequest use a hand-written binary +// decoder. +func decodeLeaseRequestBinary(hdrBytes []byte, jsonBytes []byte) (leaseReq *LeaseRequest, jreq *jsonRequest) { + var err error + + leaseReq = &LeaseRequest{} + jreq = &jsonRequest{} + ioReq := &ioRequestRetryRpc{} + + hdrBuf := bytes.NewBuffer(hdrBytes) + jsonBuf := bytes.NewBuffer(jsonBytes) + + // Unmarshal the IoRequest header + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Len) + if err != nil { + panic("ioReq.Hdr.Len") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Protocol) + if err != nil { + panic("ioReq.Hdr.Protocol") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Version) + if err != nil { + panic("ioReq.Hdr.Version") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Type) + if err != nil { + panic("ioReq.Hdr.Type") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Magic) + if err != nil { + panic("ioReq.Hdr.Magic") + } + + // now unmarshal the jsonRequest fields + err = binary.Read(jsonBuf, binary.LittleEndian, &jreq.MyUniqueID) + if err != nil { + panic("jreq.MyUniqueID") + } + err = binary.Read(jsonBuf, binary.LittleEndian, &jreq.RequestID) + if err != nil { + panic("jreq.RequestID") + } + err = binary.Read(jsonBuf, binary.LittleEndian, &jreq.HighestReplySeen) + if err != nil { + panic("jreq.HighestReplySeen") + } + // method is Nul terminated + method, err := jsonBuf.ReadString(0) + if err != nil { + panic(fmt.Sprintf("jreq.Method '%s' buf '%+v' err: %v", jreq.Method, jsonBuf, err)) + } + jreq.Method = method[0 : len(method)-1] + + // unmarshal LeaseRequest fields (MountID is Nul terminated) + mountID, err := jsonBuf.ReadString(0) + if err != nil { + panic("leaseReq.MountID") + } + leaseReq.MountID = MountIDAsString(mountID[0 : len(mountID)-1]) + err = binary.Read(jsonBuf, binary.LittleEndian, &leaseReq.InodeNumber) + if err != nil { + panic("leaseReq.LeaseInodeNumber") + } + err = binary.Read(jsonBuf, binary.LittleEndian, &leaseReq.LeaseRequestType) + if err != nil { + panic("leaseReq.LeaseRequestType") + } + + hdrBytes = hdrBuf.Bytes() + jsonBytes = jsonBuf.Bytes() + return +} + +// Perform all of the decoding required on the server to recieve a LeaseReply. +// This includes decoding the LeaseReply, the jsonReply "wrapper", and the +// ioReply header (which is always a fixed binary encoding). +// +// The decoding of the jsonReply and LeaseRequest are also a hand-written binary +// decoder. +func decodeLeaseReplyBinary(hdrBytes []byte, jsonBytes []byte) (leaseReply *LeaseReply, jreply *jsonReply) { + var err error + + leaseReply = &LeaseReply{} + jreply = &jsonReply{} + ioReply := &ioReplyRetryRpc{} + + hdrBuf := bytes.NewBuffer(hdrBytes) + jsonBuf := bytes.NewBuffer(jsonBytes) + + // first unmarshal the IoReply + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Len) + if err != nil { + panic("ioReply.Hdr.Len") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Protocol) + if err != nil { + panic("ioReply.Hdr.Protocol") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Version) + if err != nil { + panic("ioReply.Hdr.Version") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Type) + if err != nil { + panic("ioReply.Hdr.Type") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Magic) + if err != nil { + panic("ioReply.Hdr.Magic") + } + + // now unmarshal jsonReply fields + err = binary.Read(jsonBuf, binary.LittleEndian, &jreply.MyUniqueID) + if err != nil { + panic("jreply.MyUniqueID") + } + err = binary.Read(jsonBuf, binary.LittleEndian, &jreply.RequestID) + if err != nil { + panic("jreply.RequestID") + } + // ErrStr is Nul terminated + errStr, err := jsonBuf.ReadString(0) + if err != nil { + panic("jreply.ErrStr") + } + jreply.ErrStr = errStr[0 : len(errStr)-1] + + // unmarshal LeaseReply fields + err = binary.Read(jsonBuf, binary.LittleEndian, &leaseReply.LeaseReplyType) + if err != nil { + panic("leaseReply.LeaseReplyType") + } + + hdrBytes = hdrBuf.Bytes() + jsonBytes = jsonBuf.Bytes() + return +} + +// Perform all of the encoding required by the client to send a LeaseRequest. +// This includes encoding the LeaseRequest, the jsonRequest "wrapper", and the +// ioRequest header (which is always a fixed binary encoding). +// +// The encoding of the jsonRequest and LeaseRequest uses json.Marshal(). +func encodeLeaseRequestJson(leaseReq *LeaseRequest, jreq *jsonRequest) (hdrBytes []byte, jsonBytes []byte) { + var err error + + hdrBuf := &bytes.Buffer{} + + // the Lease Request is part of the JSON request + jreq.Params[0] = leaseReq + + // marshal the json request (and lease request) + jsonBytes, err = json.Marshal(jreq) + if err != nil { + panic("json.Marshal") + } + + // now create the IoRequest header and Marshal it + // (this is always binary) + ioReq := ioRequestRetryRpc{ + Hdr: ioHeader{ + Len: uint32(len(jsonBytes)), + Protocol: uint16(1), + Version: 1, + Type: 1, + Magic: 0xCAFEFEED, + }, + } + + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Len) + if err != nil { + panic("ioReq.Hdr.Len") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Protocol) + if err != nil { + panic("ioReq.Hdr.Protocol") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Version) + if err != nil { + panic("ioReq.Hdr.Version") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Type) + if err != nil { + panic("ioReq.Hdr.Type") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Magic) + if err != nil { + panic("ioReq.Hdr.Magic") + } + + hdrBytes = hdrBuf.Bytes() + return +} + +// Perform all of the decoding required on the server to recieve a LeaseRequest. +// This includes decoding the LeaseRequest, the jsonRequest "wrapper", and the +// ioRequest header (which is always a fixed binary encoding). +// +// The decoding of the jsonRequest and LeaseRequest use json.Unmarshal(). +func decodeLeaseRequestJson(hdrBytes []byte, jsonBytes []byte) (leaseReq *LeaseRequest, jreq *jsonRequest) { + var err error + + leaseReq = &LeaseRequest{} + jreq = &jsonRequest{} + jreq.Params[0] = leaseReq + ioReq := &ioRequestRetryRpc{} + + hdrBuf := bytes.NewBuffer(hdrBytes) + + // Unmarshal the IoRequest header (always binary) + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Len) + if err != nil { + panic("ioReq.Hdr.Len") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Protocol) + if err != nil { + panic("ioReq.Hdr.Protocol") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Version) + if err != nil { + panic("ioReq.Hdr.Version") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Type) + if err != nil { + panic("ioReq.Hdr.Type") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Magic) + if err != nil { + panic("ioReq.Hdr.Magic") + } + + // now unmarshal the jsonRequest fields + err = json.Unmarshal(jsonBytes, jreq) + if err != nil { + panic("json.Unmarshal of json") + } + + return +} + +// Perform all of the encoding required on the server to send a LeaseReply. +// This includes encoding the LeaseReply, the jsonReply "wrapper", and the +// ioReply header (which is always a fixed binary encoding). +// +// The encoding of the jsonReply and LeaseReply use json.Marshal(). +func encodeLeaseReplyJson(leaseReply *LeaseReply, jreply *jsonReply) (hdrBytes []byte, jsonBytes []byte) { + var err error + + hdrBuf := &bytes.Buffer{} + + // the Lease Reply is part of the JSON reply + jreply.Result = leaseReply + + // marshal the json reply (and lease reply) + jsonBytes, err = json.Marshal(jreply) + if err != nil { + panic("json.Marshal") + } + + // now create the IoReply header and Marshal it + // (this is always binary) + ioReply := ioReplyRetryRpc{ + Hdr: ioHeader{ + Len: uint32(len(jsonBytes)), + Protocol: uint16(1), + Version: 1, + Type: 1, + Magic: 0xCAFEFEED, + }, + } + + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Len) + if err != nil { + panic("ioReply.Hdr.Len") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Protocol) + if err != nil { + panic("ioReply.Hdr.Protocol") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Version) + if err != nil { + panic("ioReply.Hdr.Version") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Type) + if err != nil { + panic("ioReply.Hdr.Type") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Magic) + if err != nil { + panic("ioReply.Hdr.Magic") + } + + hdrBytes = hdrBuf.Bytes() + return +} + +// Perform all of the decoding required on the server to recieve a LeaseReply. +// This includes decoding the LeaseReply, the jsonReply "wrapper", and the +// ioReply header (which is always a fixed binary encoding). +// +// The decoding of the jsonReply and LeaseReply use a json.Unmarshal(). +func decodeLeaseReplyJson(hdrBytes []byte, jsonBytes []byte) (leaseReply *LeaseReply, jreply *jsonReply) { + var err error + + leaseReply = &LeaseReply{} + jreply = &jsonReply{} + jreply.Result = leaseReply + ioReply := &ioReplyRetryRpc{} + + hdrBuf := bytes.NewBuffer(hdrBytes) + + // Unmarshal the IoReply header (always binary) + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Len) + if err != nil { + panic("ioReply.Hdr.Len") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Protocol) + if err != nil { + panic("ioReply.Hdr.Protocol") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Version) + if err != nil { + panic("ioReply.Hdr.Version") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Type) + if err != nil { + panic("ioReply.Hdr.Type") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Magic) + if err != nil { + panic("ioReply.Hdr.Magic") + } + + // now unmarshal the jsonReply fields + err = json.Unmarshal(jsonBytes, jreply) + if err != nil { + panic("json.Unmarshal of json") + } + + leaseReply = jreply.Result.(*LeaseReply) + return +} + +// Create gob LeaseRequest and LeaseReply encoders and decoders that are +// globally available. +var ( + encodeLeaseRequestGobBuffer *bytes.Buffer = &bytes.Buffer{} + decodeLeaseRequestGobBuffer *bytes.Buffer = &bytes.Buffer{} + encodeLeaseRequestGobEncoder *gob.Encoder = gob.NewEncoder(encodeLeaseRequestGobBuffer) + decodeLeaseRequestGobDecoder *gob.Decoder = gob.NewDecoder(decodeLeaseRequestGobBuffer) + + encodeLeaseReplyGobBuffer *bytes.Buffer = &bytes.Buffer{} + decodeLeaseReplyGobBuffer *bytes.Buffer = &bytes.Buffer{} + encodeLeaseReplyGobEncoder *gob.Encoder = gob.NewEncoder(encodeLeaseReplyGobBuffer) + decodeLeaseReplyGobDecoder *gob.Decoder = gob.NewDecoder(decodeLeaseReplyGobBuffer) +) + +// Perform all of the encoding required by the client to send a LeaseRequest. +// This includes encoding the LeaseRequest, the jsonRequest "wrapper", and the +// ioRequest header (which is always a fixed binary encoding). +// +// The encoding of the jsonRequest and LeaseRequest use a pre-defined gob +// encoding. +func encodeLeaseRequestGob(leaseReq *LeaseRequest, jreq *jsonRequest) (hdrBytes []byte, gobBytes []byte) { + var err error + + hdrBuf := &bytes.Buffer{} + + // the Lease Request is part of the gob request + jreq.Params[0] = leaseReq + + // marshal jreq (and lease request) + err = encodeLeaseRequestGobEncoder.Encode(jreq) + if err != nil { + panic("encodeLeaseRequestGobEncoder") + } + + // consume the results encoded in the (global) buffer + gobBytes = make([]byte, encodeLeaseRequestGobBuffer.Len()) + n, err := encodeLeaseRequestGobBuffer.Read(gobBytes) + if n != cap(gobBytes) { + panic("didn't read enough bytes") + } + + // now create the IoRequest header and Marshal it + // (this is always binary) + ioReq := ioRequestRetryRpc{ + Hdr: ioHeader{ + Len: uint32(len(gobBytes)), + Protocol: uint16(1), + Version: 1, + Type: 1, + Magic: 0xCAFEFEED, + }, + } + + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Len) + if err != nil { + panic("ioReq.Hdr.Len") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Protocol) + if err != nil { + panic("ioReq.Hdr.Protocol") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Version) + if err != nil { + panic("ioReq.Hdr.Version") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Type) + if err != nil { + panic("ioReq.Hdr.Type") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Magic) + if err != nil { + panic("ioReq.Hdr.Magic") + } + + hdrBytes = hdrBuf.Bytes() + return +} + +// Perform all of the decoding required on the server to receive a LeaseRequest. +// This includes decoding the LeaseRequest, the jsonRequest "wrapper", and the +// ioRequest header (which is always a fixed binary encoding). +// +// The decoding of the jsonRequest and LeaseRequest use a pre-defined gob decoder. +func decodeLeaseRequestGob(hdrBytes []byte, gobBytes []byte) (leaseReq *LeaseRequest, jreq *jsonRequest) { + var err error + + jreq = &jsonRequest{} + ioReq := &ioRequestRetryRpc{} + + hdrBuf := bytes.NewBuffer(hdrBytes) + + // Unmarshal the IoRequest header (always binary) + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Len) + if err != nil { + panic("ioReq.Hdr.Len") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Protocol) + if err != nil { + panic("ioReq.Hdr.Protocol") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Version) + if err != nil { + panic("ioReq.Hdr.Version") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Type) + if err != nil { + panic("ioReq.Hdr.Type") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Magic) + if err != nil { + panic("ioReq.Hdr.Magic") + } + + // now unmarshal the jsonRequest fields using gob (can't fail) + _, _ = decodeLeaseRequestGobBuffer.Write(gobBytes) + err = decodeLeaseRequestGobDecoder.Decode(jreq) + if err != nil { + panic("decodeLeaseRequestGobDecoder.Decode") + } + leaseReq = jreq.Params[0].(*LeaseRequest) + + return +} + +// Perform all of the encoding required on the server to send a LeaseReply. +// This includes encoding the LeaseReply, the jsonReply "wrapper", and the +// ioReply header (which is always a fixed binary encoding). +// +// The encoding of the jsonReply and LeaseReply use a pre-defined gob encoder. +func encodeLeaseReplyGob(leaseReply *LeaseReply, jreply *jsonReply) (hdrBytes []byte, gobBytes []byte) { + var err error + + hdrBuf := &bytes.Buffer{} + + // the Lease Reply is part of the JSON reply + jreply.Result = leaseReply + + // marshal jreq (and lease request) + err = encodeLeaseReplyGobEncoder.Encode(jreply) + if err != nil { + panic("encodeLeaseReplyGobEncoder") + } + + // consume the results encoded in the (global) buffer + gobBytes = make([]byte, encodeLeaseReplyGobBuffer.Len()) + n, err := encodeLeaseReplyGobBuffer.Read(gobBytes) + if n != cap(gobBytes) { + panic("didn't read enough bytes") + } + + // now create the IoReply header and Marshal it + // (this is always binary) + ioReply := ioReplyRetryRpc{ + Hdr: ioHeader{ + Len: uint32(len(gobBytes)), + Protocol: uint16(1), + Version: 1, + Type: 1, + Magic: 0xCAFEFEED, + }, + } + + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Len) + if err != nil { + panic("ioReply.Hdr.Len") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Protocol) + if err != nil { + panic("ioReply.Hdr.Protocol") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Version) + if err != nil { + panic("ioReply.Hdr.Version") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Type) + if err != nil { + panic("ioReply.Hdr.Type") + } + err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Magic) + if err != nil { + panic("ioReply.Hdr.Magic") + } + + hdrBytes = hdrBuf.Bytes() + return +} + +// Perform all of the decoding required on the server to recieve a LeaseReply. +// This includes decoding the LeaseReply, the jsonReply "wrapper", and the +// ioReply header (which is always a fixed binary encoding). +// +// The decoding of the jsonReply and LeaseReply use a pre-defined gob decoder. +func decodeLeaseReplyGob(hdrBytes []byte, gobBytes []byte) (leaseReply *LeaseReply, jreply *jsonReply) { + var err error + + leaseReply = &LeaseReply{} + jreply = &jsonReply{} + jreply.Result = leaseReply + ioReply := &ioReplyRetryRpc{} + + hdrBuf := bytes.NewBuffer(hdrBytes) + + // Unmarshal the IoReply header (always binary) + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Len) + if err != nil { + panic("ioReply.Hdr.Len") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Protocol) + if err != nil { + panic("ioReply.Hdr.Protocol") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Version) + if err != nil { + panic("ioReply.Hdr.Version") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Type) + if err != nil { + panic("ioReply.Hdr.Type") + } + err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Magic) + if err != nil { + panic("ioReply.Hdr.Magic") + } + + // now unmarshal the jsonReply fields using gob (can't fail) + _, _ = decodeLeaseReplyGobBuffer.Write(gobBytes) + err = decodeLeaseReplyGobDecoder.Decode(jreply) + if err != nil { + panic("decodeLeaseReplyGobDecoder.Decode") + } + leaseReply = jreply.Result.(*LeaseReply) + + return +} diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 8cd840a1..faeae246 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -56,6 +56,11 @@ type testRpcLeaseClientStruct struct { t *testing.T } +type benchmarkRpcServerStats struct { + UnmarshalRequestUsec bucketstats.BucketLog2Round + MarshalReplyUsec bucketstats.BucketLog2Round +} + type benchmarkRpcShortcutRequestMethodOnlyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form Method string } @@ -1148,9 +1153,19 @@ func benchmarkRpcLeaseShortcut(b *testing.B, useTLS bool) { doneWG.Wait() } +type ServerStats struct { + RequestUnmarshalUsec bucketstats.BucketLog2Round + RpcMountRequestUnmarshalUsec bucketstats.BucketLog2Round + RpcMountReplyMarshalUsec bucketstats.BucketLog2Round + RpcLeaseRequestUnmarshalUsec bucketstats.BucketLog2Round + RpcLeaseReplyMarshalUsec bucketstats.BucketLog2Round + RpcUnmountRequestUnmarshalUsec bucketstats.BucketLog2Round + RpcUnmountReplyMarshalUsec bucketstats.BucketLog2Round +} + func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { var ( - benchmarkRpcShortcutRequestMethodOnly *benchmarkRpcShortcutRequestMethodOnlyStruct + benchmarkRpcShortcutRequestMethodOnly benchmarkRpcShortcutRequestMethodOnlyStruct err error jserver *Server leaseReply *LeaseReply @@ -1175,8 +1190,18 @@ func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { unmountReplyWrapped *benchmarkRpcShortcutUnmountReplyStruct unmountRequest *UnmountRequest unmountRequestWrapped *benchmarkRpcShortcutUnmountRequestStruct + serverStats ServerStats ) + bucketstats.Register("benchmark", "shortcutServer", &serverStats) + defer func() { + /* TODO - uncomment if the bucketstats should be dumped + fmt.Printf("%s\n", + bucketstats.SprintStats(bucketstats.StatFormatParsable1, "benchmark", "shortcutServer")) + */ + bucketstats.UnRegister("benchmark", "shortcutServer") + }() + jserver = NewServer() netListener, err = net.Listen("tcp", testRpcLeaseShortcutIPAddrPort) @@ -1235,9 +1260,10 @@ func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { panic(fmt.Errorf("netConn.Read(requestBuf) returned n == %v (expected %v)", n, requestLen)) } - benchmarkRpcShortcutRequestMethodOnly = &benchmarkRpcShortcutRequestMethodOnlyStruct{} + startTime := time.Now() + err = json.Unmarshal(requestBuf, &benchmarkRpcShortcutRequestMethodOnly) + serverStats.RequestUnmarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) - err = json.Unmarshal(requestBuf, benchmarkRpcShortcutRequestMethodOnly) if nil != err { panic(fmt.Errorf("json.Unmarshal(requestBuf, benchmarkRpcShortcutRequestMethodOnly) failed: %v", err)) } @@ -1246,7 +1272,10 @@ func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { case "RpcMountByAccountName": mountByAccountNameRequestWrapped = &benchmarkRpcShortcutMountByAccountNameRequestStruct{} + startTime = time.Now() err = json.Unmarshal(requestBuf, mountByAccountNameRequestWrapped) + serverStats.RpcMountRequestUnmarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) + if nil != err { panic(fmt.Errorf("json.Unmarshal(requestBuf, mountByAccountNameRequestWrapped) failed: %v", err)) } @@ -1261,14 +1290,18 @@ func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { Reply: mountByAccountNameReply, } + startTime = time.Now() replyBuf, err = json.Marshal(mountByAccountNameReplyWrapped) + serverStats.RpcMountReplyMarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) if nil != err { panic(fmt.Errorf("json.Marshal(mountByAccountNameReplyWrapped) failed")) } case "RpcLease": leaseRequestWrapped = &benchmarkRpcShortcutLeaseRequestStruct{} + startTime = time.Now() err = json.Unmarshal(requestBuf, leaseRequestWrapped) + serverStats.RpcLeaseRequestUnmarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) if nil != err { panic(fmt.Errorf("json.Unmarshal(requestBuf, leaseRequestWrapped) failed: %v", err)) } @@ -1283,14 +1316,18 @@ func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { Reply: leaseReply, } + startTime = time.Now() replyBuf, err = json.Marshal(leaseReplyWrapped) + serverStats.RpcLeaseReplyMarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) if nil != err { panic(fmt.Errorf("json.Marshal(leaseReplyWrapped) failed")) } case "RpcUnmount": unmountRequestWrapped = &benchmarkRpcShortcutUnmountRequestStruct{} + startTime = time.Now() err = json.Unmarshal(requestBuf, unmountRequestWrapped) + serverStats.RpcUnmountRequestUnmarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) if nil != err { panic(fmt.Errorf("json.Unmarshal(requestBuf, unmountRequestWrapped) failed: %v", err)) } @@ -1305,7 +1342,9 @@ func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { Reply: unmountReply, } + startTime = time.Now() replyBuf, err = json.Marshal(unmountReplyWrapped) + serverStats.RpcUnmountReplyMarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) if nil != err { panic(fmt.Errorf("json.Marshal(unmountReplyWrapped) failed")) } diff --git a/jrpcfs/setup_teardown_test.go b/jrpcfs/setup_teardown_test.go index 53793b02..58ebfc2b 100644 --- a/jrpcfs/setup_teardown_test.go +++ b/jrpcfs/setup_teardown_test.go @@ -6,6 +6,7 @@ package jrpcfs import ( "crypto/tls" "crypto/x509/pkix" + "encoding/gob" "fmt" "io/ioutil" "net" @@ -79,6 +80,12 @@ func testSetup() []func() { testConfMap conf.ConfMap ) + fmt.Printf("BEFORE CALL TO gob.Register!!!!!!\n") + + // register some RPC types to gob so it can encode/decode them efficiently + gob.Register(&LeaseRequest{}) + gob.Register(&LeaseReply{}) + cleanupFuncs = make([]func(), 0) tempDir, err = ioutil.TempDir("", "jrpcfs_test") diff --git a/retryrpc/client.go b/retryrpc/client.go index a7e20caf..d5a93112 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -84,6 +84,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter err = fmt.Errorf("In send(), ConnectionRetryLimit (%v) on calling dial() exceeded", ConnectionRetryLimit) logger.PanicfWithError(err, "") } + logger.Warnf("initialDial() failed; retrying: %v", err) time.Sleep(connectionRetryDelay) connectionRetryDelay *= ConnectionRetryDelayMultiplier client.Lock() From 6a3afbc6b4b884eb2596400aa303e54685ae6d4e Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Thu, 1 Jul 2021 13:57:58 -0700 Subject: [PATCH 017/258] Breakout loggers (#629) * User *log.Logger from client/server structure * If the caller does not provide a logger, create our own Also, document that Interrupt() must return eventually to avoid a hang --- jrpcfs/setup_teardown_test.go | 2 -- retryrpc/api.go | 33 ++++++++++++++++++++++------- retryrpc/api_internal.go | 5 ----- retryrpc/client.go | 35 +++++++++++------------------- retryrpc/retryrpc_test.go | 17 ++++++++++++++- retryrpc/server.go | 40 +++++++++++++++-------------------- retryrpc/stress_test.go | 2 ++ retryrpc/upcall_test.go | 2 ++ 8 files changed, 74 insertions(+), 62 deletions(-) diff --git a/jrpcfs/setup_teardown_test.go b/jrpcfs/setup_teardown_test.go index 58ebfc2b..9d9a73b7 100644 --- a/jrpcfs/setup_teardown_test.go +++ b/jrpcfs/setup_teardown_test.go @@ -80,8 +80,6 @@ func testSetup() []func() { testConfMap conf.ConfMap ) - fmt.Printf("BEFORE CALL TO gob.Register!!!!!!\n") - // register some RPC types to gob so it can encode/decode them efficiently gob.Register(&LeaseRequest{}) gob.Register(&LeaseReply{}) diff --git a/retryrpc/api.go b/retryrpc/api.go index 10fbb2f0..90dfd81b 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -11,11 +11,13 @@ package retryrpc // a response. import ( + "bytes" "container/list" "context" "crypto/tls" "crypto/x509" "fmt" + "log" "net" "reflect" "strconv" @@ -23,7 +25,6 @@ import ( "time" "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/logger" "github.com/google/btree" ) @@ -54,7 +55,8 @@ type Server struct { deadlineIO time.Duration keepAlivePeriod time.Duration completedDoneWG sync.WaitGroup - dontStartTrimmers bool // Used for testing + logger *log.Logger // If nil, defaults to log.New() + dontStartTrimmers bool // Used for testing } // ServerConfig is used to configure a retryrpc Server @@ -66,7 +68,8 @@ type ServerConfig struct { DeadlineIO time.Duration // How long I/Os on sockets wait even if idle KeepAlivePeriod time.Duration // How frequently a KEEPALIVE is sent TLSCertificate tls.Certificate // TLS Certificate to present to Clients (or tls.Certificate{} if using TCP) - dontStartTrimmers bool // Used for testingD + Logger *log.Logger // If nil, defaults to log.New() + dontStartTrimmers bool // Used for testing } // NewServer creates the Server object @@ -79,7 +82,12 @@ func NewServer(config *ServerConfig) *Server { deadlineIO: config.DeadlineIO, keepAlivePeriod: config.KeepAlivePeriod, dontStartTrimmers: config.dontStartTrimmers, + logger: config.Logger, tlsCertificate: config.TLSCertificate} + if server.logger == nil { + var logBuf bytes.Buffer + server.logger = log.New(&logBuf, "", 0) + } server.svrMap = make(map[string]*methodArgs) server.perClientInfo = make(map[uint64]*clientInfo) server.completedTickerDone = make(chan bool) @@ -177,7 +185,7 @@ func (server *Server) SendCallback(clientID uint64, msg []byte) { server.Lock() lci, ok := server.perClientInfo[clientID] if !ok { - fmt.Printf("SERVER: SendCallback() - unable to find client UniqueID: %v\n", clientID) + server.logger.Printf("SERVER: SendCallback() - unable to find client UniqueID: %v\n", clientID) server.Unlock() return } @@ -202,12 +210,12 @@ func (server *Server) Close() { if len(server.tlsCertificate.Certificate) == 0 { err := server.netListener.Close() if err != nil { - logger.Errorf("server.netListener.Close() returned err: %v", err) + server.logger.Printf("server.netListener.Close() returned err: %v\n", err) } } else { err := server.tlsListener.Close() if err != nil { - logger.Errorf("server.tlsListener.Close() returned err: %v", err) + server.logger.Printf("server.tlsListener.Close() returned err: %v\n", err) } } @@ -303,11 +311,15 @@ type Client struct { // trimmed bt *btree.BTree // btree of requestID's acked goroutineWG sync.WaitGroup // Used to track outstanding goroutines + logger *log.Logger // If nil, defaults to log.New() stats clientSideStatsInfo } // ClientCallbacks contains the methods required when supporting // callbacks from the Server. +// +// NOTE: It is assumed that ALL Interrupt() routines will eventually +// return. Failure to do so will cause client.Close() to hang! type ClientCallbacks interface { Interrupt(payload []byte) } @@ -320,10 +332,9 @@ type ClientConfig struct { Callbacks interface{} // Structure implementing ClientCallbacks DeadlineIO time.Duration // How long I/Os on sockets wait even if idle KeepAlivePeriod time.Duration // How frequently a KEEPALIVE is sent + Logger *log.Logger // If nil, defaults to log.New() } -// TODO - pass loggers to Cient and Server objects - // NewClient returns a Client structure // // If the server wants to send an async message to the client @@ -345,6 +356,12 @@ func NewClient(config *ClientConfig) (client *Client, err error) { cb: config.Callbacks, keepAlivePeriod: config.KeepAlivePeriod, deadlineIO: config.DeadlineIO, + logger: config.Logger, + } + + if client.logger == nil { + var logBuf bytes.Buffer + client.logger = log.New(&logBuf, "", 0) } client.outstandingRequest = make(map[requestID]*reqCtx) diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index 7650a8a9..e5b9d8ad 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -17,7 +17,6 @@ import ( "time" "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/logger" ) // PayloadProtocols defines the supported protocols for the payload @@ -258,10 +257,6 @@ func buildINeedIDRequest() (iinreq *internalINeedIDRequest, err error) { } func getIO(genNum uint64, deadlineIO time.Duration, conn net.Conn) (buf []byte, msgType MsgType, err error) { - if printDebugLogs { - logger.Infof("conn: %v", conn) - } - // Read in the header of the request first var hdr ioHeader diff --git a/retryrpc/client.go b/retryrpc/client.go index d5a93112..906b03e2 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -14,7 +14,6 @@ import ( "time" "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/logger" "github.com/google/btree" ) @@ -81,10 +80,9 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter client.Unlock() connectionRetryCount++ if connectionRetryCount > ConnectionRetryLimit { - err = fmt.Errorf("In send(), ConnectionRetryLimit (%v) on calling dial() exceeded", ConnectionRetryLimit) - logger.PanicfWithError(err, "") + client.logger.Fatalf("In send(), ConnectionRetryLimit (%v) on calling dial() exceeded", ConnectionRetryLimit) } - logger.Warnf("initialDial() failed; retrying: %v", err) + client.logger.Printf("initialDial() failed; retrying: %v\n", err) time.Sleep(connectionRetryDelay) connectionRetryDelay *= ConnectionRetryDelayMultiplier client.Lock() @@ -101,8 +99,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter if client.halting { client.Unlock() - e := fmt.Errorf("Calling retryrpc.Send() without dialing") - logger.PanicfWithError(e, "") + client.logger.Fatalf("Calling retryrpc.Send() without dialing") return } client.currentRequestID++ @@ -113,8 +110,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter // Setup ioreq to write structure on socket to server ioreq, err := buildIoRequest(&jreq) if err != nil { - e := fmt.Errorf("Client buildIoRequest returned err: %v", err) - logger.PanicfWithError(e, "") + client.logger.Fatalf("Client buildIoRequest returned err: %v", err) return err } @@ -224,9 +220,7 @@ func (client *Client) notifyReply(buf []byte, genNum uint64, recvResponse time.T // Don't have ctx to reply. Assume read garbage on socket and // reconnect. - // TODO - make log message - e := fmt.Errorf("notifyReply failed to unmarshal buf: %+v err: %v", string(buf), err) - fmt.Printf("%v\n", e) + client.logger.Printf("notifyReply failed to unmarshal buf: %+v err: %v\n", string(buf), err) client.retransmit(genNum) return } @@ -260,8 +254,7 @@ func (client *Client) notifyReply(buf []byte, genNum uint64, recvResponse time.T m := svrResponse{Result: ctx.rpcReply} unmarshalErr := json.Unmarshal(buf, &m) if unmarshalErr != nil { - e := fmt.Errorf("notifyReply failed to unmarshal buf: %v err: %v ctx: %v", string(buf), unmarshalErr, ctx) - fmt.Printf("%v\n", e) + client.logger.Printf("notifyReply failed to unmarshal buf: %v err: %v ctx: %v\n", string(buf), unmarshalErr, ctx) // Assume read garbage on socket - close the socket and reconnect // Drop client lock since retransmit() will acquire it. @@ -375,7 +368,7 @@ func (client *Client) readReplies(nC net.Conn, callingGenNum uint64) { client.stats.UpcallCalled.Add(1) default: - fmt.Printf("CLIENT - invalid msgType: %v\n", msgType) + client.logger.Printf("CLIENT - invalid msgType: %v\n", msgType) } } } @@ -424,8 +417,7 @@ func (client *Client) retransmit(genNum uint64) { client.Unlock() connectionRetryCount++ if connectionRetryCount > ConnectionRetryLimit { - err = fmt.Errorf("In retransmit(), ConnectionRetryLimit (%v) on calling dial() exceeded", ConnectionRetryLimit) - logger.PanicfWithError(err, "") + client.logger.Fatalf("In retransmit(), ConnectionRetryLimit (%v) on calling dial() exceeded", ConnectionRetryLimit) } time.Sleep(connectionRetryDelay) connectionRetryDelay *= ConnectionRetryDelayMultiplier @@ -455,8 +447,7 @@ func (client *Client) getMyUniqueID() (err error) { // Setup ioreq to write structure on socket to server iinreq, err := buildINeedIDRequest() if err != nil { - e := fmt.Errorf("Client buildINeedIDRequest returned err: %v", err) - logger.PanicfWithError(e, "") + client.logger.Fatalf("Client buildINeedIDRequest returned err: %v", err) return err } @@ -486,8 +477,7 @@ func (client *Client) sendMyInfo() (err error) { // Setup ioreq to write structure on socket to server isreq, err := buildSetIDRequest(client.myUniqueID) if err != nil { - e := fmt.Errorf("Client buildSetIDRequest returned err: %v", err) - logger.PanicfWithError(e, "") + client.logger.Fatalf("Client buildSetIDRequest returned err: %v", err) return err } @@ -603,14 +593,13 @@ func (client *Client) readClientID(callingGenNum uint64) (myUniqueID uint64, err } if msgType != ReturnUniqueID { - err = fmt.Errorf("CLIENT - invalid msgType: %v", msgType) - logger.PanicfWithError(err, "") + client.logger.Fatalf("CLIENT - invalid msgType: %v", msgType) return } err = json.Unmarshal(buf, &myUniqueID) if err != nil { - logger.PanicfWithError(err, "Unmarshal of buf: %v to myUniqueID failed with err: %v", buf, err) + client.logger.Fatalf("Unmarshal of buf: %v to myUniqueID failed with err: %v", buf, err) } return } diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index 56c11b71..96919434 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -4,9 +4,11 @@ package retryrpc import ( + "bytes" "crypto/tls" "crypto/x509/pkix" "fmt" + "log" "net" "strconv" "testing" @@ -31,7 +33,14 @@ type testTLSCertsStruct struct { endpointTLSCert tls.Certificate } -var testTLSCerts *testTLSCertsStruct +func newLogger() *log.Logger { + return log.New(&logBuf, "", 0) +} + +var ( + logBuf bytes.Buffer + testTLSCerts *testTLSCertsStruct +) // Utility function to initialize testTLSCerts func testTLSCertsAllocate(t *testing.T) { @@ -120,6 +129,7 @@ func getNewServer(lt time.Duration, dontStartTrimmers bool, useTLS bool) (rrSvr DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, TLSCertificate: testTLSCerts.endpointTLSCert, + Logger: newLogger(), dontStartTrimmers: dontStartTrimmers, } } else { @@ -131,6 +141,7 @@ func getNewServer(lt time.Duration, dontStartTrimmers bool, useTLS bool) (rrSvr DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, TLSCertificate: tls.Certificate{}, + Logger: newLogger(), dontStartTrimmers: dontStartTrimmers, } } @@ -200,6 +211,7 @@ func testServer(t *testing.T, useTLS bool) { Callbacks: nil, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, + Logger: newLogger(), } } else { clientConfig = &ClientConfig{ @@ -209,6 +221,7 @@ func testServer(t *testing.T, useTLS bool) { Callbacks: nil, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, + Logger: newLogger(), } } rrClnt, newErr := NewClient(clientConfig) @@ -273,6 +286,7 @@ func testBtree(t *testing.T, useTLS bool) { Callbacks: nil, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, + Logger: newLogger(), } } else { clientConfig = &ClientConfig{ @@ -282,6 +296,7 @@ func testBtree(t *testing.T, useTLS bool) { Callbacks: nil, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, + Logger: newLogger(), } } client, newErr := NewClient(clientConfig) diff --git a/retryrpc/server.go b/retryrpc/server.go index 94455196..5141cf80 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -14,13 +14,9 @@ import ( "sync" "time" - "github.com/NVIDIA/proxyfs/logger" "golang.org/x/sys/unix" ) -// Variable to control debug output -var printDebugLogs bool = false - // TODO - test if Register has been called??? func (server *Server) closeClient(myConn net.Conn, myElm *list.Element) { @@ -49,7 +45,7 @@ func (server *Server) run() { } if err != nil { if !server.halting { - logger.ErrorfWithError(err, "net.Accept failed for Retry RPC listener") + server.logger.Printf("net.Accept failed for Retry RPC listener - err: %v\n", err) } server.listenersWG.Done() return @@ -76,7 +72,7 @@ func (server *Server) run() { ci, err := server.getClientIDAndWait(cCtx) if err != nil { // Socket already had an error - just loop back - logger.Warnf("getClientIDAndWait() from client addr: %v returned err: %v\n", conn.RemoteAddr(), err) + server.logger.Printf("getClientIDAndWait() from client addr: %v returned err: %v\n", conn.RemoteAddr(), err) // Sleep to block over active clients from pounding on us time.Sleep(1 * time.Second) @@ -89,10 +85,10 @@ func (server *Server) run() { go func(myCi *clientInfo, myCCtx *connCtx, myConn net.Conn, myElm *list.Element) { defer server.goroutineWG.Done() - logger.Infof("Servicing client: %v address: %v", myCi.myUniqueID, myConn.RemoteAddr()) + server.logger.Printf("Servicing client: %v address: %v\n", myCi.myUniqueID, myConn.RemoteAddr()) server.serviceClient(myCi, myCCtx) - logger.Infof("Closing client: %v address: %v", myCi.myUniqueID, myConn.RemoteAddr()) + server.logger.Printf("Closing client: %v address: %v\n", myCi.myUniqueID, myConn.RemoteAddr()) server.closeClient(myConn, myElm) // The clientInfo for this client will first be trimmed and then later @@ -113,7 +109,7 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b jReq := jsonRequest{} unmarErr := json.Unmarshal(buf, &jReq) if unmarErr != nil { - logger.Errorf("Unmarshal of buf failed with err: %v\n", unmarErr) + server.logger.Printf("Unmarshal of buf failed with err: %v\n", unmarErr) return } @@ -220,8 +216,7 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err } if (msgType != PassID) && (msgType != AskMyUniqueID) { - err = fmt.Errorf("Server expecting msgType PassID or AskMyUniqueID and received: %v", msgType) - logger.PanicfWithError(err, "") + server.logger.Fatalf("Server expecting msgType PassID or AskMyUniqueID and received: %v\n", msgType) return } @@ -231,7 +226,7 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err var connUniqueID uint64 err = json.Unmarshal(buf, &connUniqueID) if err != nil { - logger.PanicfWithError(err, "Unmarshal returned error") + server.logger.Fatalf("Unmarshal returned error: %v", err) return } @@ -243,7 +238,7 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err // TODO - tell the client they need to panic since it is better for the client to // panic then the server. err = fmt.Errorf("Server - msgType PassID but can't find uniqueID in perClientInfo") - logger.PanicfWithError(err, "getClientIDAndWait() buf: %v connUniqueID: %v err: %+v", buf, connUniqueID, err) + server.logger.Fatalf("getClientIDAndWait() buf: %v connUniqueID: %v err: %+v", buf, connUniqueID, err) server.Unlock() } else { server.Unlock() @@ -295,7 +290,7 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err var e error localIOR.JResult, e = json.Marshal(newUniqueID) if e != nil { - logger.PanicfWithError(e, "Marshal of newUniqueID: %v failed with err: %v", newUniqueID, err) + server.logger.Fatalf("Marshal of newUniqueID: %v failed with err: %v", newUniqueID, e) } setupHdrReply(&localIOR, ReturnUniqueID) @@ -333,8 +328,7 @@ func (server *Server) serviceClient(ci *clientInfo, cCtx *connCtx) { } if msgType != RPC { - err := fmt.Errorf("serviceClient() received invalid msgType: %v - dropping", msgType) - logger.PanicfWithError(err, "") + server.logger.Fatalf("serviceClient() received invalid msgType: %v", msgType) continue } @@ -379,7 +373,7 @@ func (server *Server) callRPCAndFormatReply(buf []byte, ci *clientInfo, jReq *js sReq.Params[0] = dummyReq err = json.Unmarshal(buf, &sReq) if err != nil { - logger.PanicfWithError(err, "Unmarshal sReq: %+v", sReq) + server.logger.Fatalf("Unmarshal sReq: %+v err: %v", sReq, err) return } req := reflect.ValueOf(dummyReq) @@ -406,7 +400,7 @@ func (server *Server) callRPCAndFormatReply(buf []byte, ci *clientInfo, jReq *js } else { e, ok := errInter.(error) if !ok { - logger.PanicfWithError(err, "Call returnValues invalid cast errInter: %+v", errInter) + server.logger.Fatalf("Call returnValues invalid cast errInter: %+v", errInter) } jReply.ErrStr = e.Error() } @@ -420,7 +414,7 @@ func (server *Server) callRPCAndFormatReply(buf []byte, ci *clientInfo, jReq *js // Convert response into JSON for return trip ior.JResult, err = json.Marshal(jReply) if err != nil { - logger.PanicfWithError(err, "Unable to marshal jReply: %+v", jReply) + server.logger.Fatalf("Unable to marshal jReply: %+v err: %v", jReply, err) } return @@ -449,7 +443,7 @@ func (server *Server) returnResults(ior *ioReply, cCtx *connCtx) { cCtx.conn.SetDeadline(time.Now().Add(server.deadlineIO)) cnt, e := cCtx.conn.Write(ior.JResult) if e != nil { - logger.Infof("returnResults() returned err: %v cnt: %v length of JResult: %v", e, cnt, len(ior.JResult)) + server.logger.Printf("returnResults() returned err: %v cnt: %v length of JResult: %v\n", e, cnt, len(ior.JResult)) } cCtx.Unlock() } @@ -499,12 +493,12 @@ func (server *Server) trimCompleted(t time.Time, long bool) { if ci.isEmpty() && ci.cCtx.serviceClientExited { ci.unregsiterMethodStats(server) delete(server.perClientInfo, key) - logger.Infof("Trim - DELETE inactive clientInfo with ID: %v", ci.myUniqueID) + server.logger.Printf("Trim - DELETE inactive clientInfo with ID: %v\n", ci.myUniqueID) } ci.cCtx.Unlock() ci.Unlock() } - logger.Infof("Trimmed completed RetryRpcs - Total: %v", totalItems) + server.logger.Printf("Trimmed completed RetryRpcs - Total: %v\n", totalItems) } else { for k, ci := range server.perClientInfo { n := server.trimAClientBasedACK(k, ci) @@ -562,7 +556,7 @@ func (server *Server) trimTLLBased(ci *clientInfo, t time.Time) (numItems int) { } } s := ci.stats - logger.Infof("ID: %v largestReplySize: %v largestReplySizeMethod: %v longest RPC: %v longest RPC Method: %v", + server.logger.Printf("ID: %v largestReplySize: %v largestReplySizeMethod: %v longest RPC: %v longest RPC Method: %v\n", ci.myUniqueID, s.largestReplySize, s.largestReplySizeMethod, s.longestRPC, s.longestRPCMethod) ci.Unlock() diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index ae12bf61..e6c9e990 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -363,6 +363,7 @@ func pfsagent(t *testing.T, rrSvr *Server, agentID uint64, method string, agentW Callbacks: cb, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, + Logger: newLogger(), } } else { clientConfig = &ClientConfig{ @@ -372,6 +373,7 @@ func pfsagent(t *testing.T, rrSvr *Server, agentID uint64, method string, agentW Callbacks: cb, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, + Logger: newLogger(), } } client, err := NewClient(clientConfig) diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index 25191bfd..174d2ed1 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -76,6 +76,7 @@ func testUpCall(t *testing.T, useTLS bool) { Callbacks: cb, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, + Logger: newLogger(), } } else { clientConfig = &ClientConfig{ @@ -85,6 +86,7 @@ func testUpCall(t *testing.T, useTLS bool) { Callbacks: cb, DeadlineIO: 60 * time.Second, KeepAlivePeriod: 60 * time.Second, + Logger: newLogger(), } } rrClnt, newErr := NewClient(clientConfig) From 84e22899e26dba57b9c3c3cbc1eb733e6cc949fe Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 2 Jul 2021 10:55:56 -0700 Subject: [PATCH 018/258] Pick up Allow Other optionality change in package fission --- go.mod | 27 ++-- go.sum | 250 ++----------------------------- pfsagentd/README.md | 1 + pfsagentd/fission.go | 1 + pfsagentd/globals.go | 6 + pfsagentd/pfsagent.conf | 1 + pfsagentd/setup_teardown_test.go | 1 + 7 files changed, 35 insertions(+), 252 deletions(-) diff --git a/go.mod b/go.mod index 5f705860..c24aabb8 100644 --- a/go.mod +++ b/go.mod @@ -5,30 +5,35 @@ go 1.15 require ( bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a - github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0 + github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec github.com/ansel1/merry v1.0.1 + github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 - github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/go-errors/errors v1.0.0 // indirect github.com/gogo/protobuf v1.2.2-0.20190611061853-dadb62585089 // indirect + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/google/btree v1.0.0 github.com/google/uuid v1.2.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml v1.9.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/kr/pretty v0.2.0 // indirect + github.com/pkg/errors v0.8.1 // indirect github.com/sirupsen/logrus v1.2.0 - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cobra v1.1.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.5.1 + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 go.uber.org/multierr v1.5.0 // indirect - go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f + go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f // indirect + golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 // indirect golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 golang.org/x/text v0.3.6 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect + golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc // indirect + google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 49c89bb1..0ba3a896 100644 --- a/go.sum +++ b/go.sum @@ -1,88 +1,52 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 h1:SC+c6A1qTFstO9qmB86mPV2IpYme/2ZoEQ0hrP+wo+Q= bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0 h1:GQZP8OZqvkcMt7l7QAdtfa17PDUqiamz64df2DWNCDM= github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= +github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTEXAJWEyRVTw5PClbHwe7O+QBnYsk= +github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/ansel1/merry v1.0.1 h1:NEqlG+pyb6XKY55oRdGStyMkQ35YucmgFVG2tpybaP8= github.com/ansel1/merry v1.0.1/go.mod h1:qBYMZz+rgzOfZ3QACcTK7hE0bBMxvC0sKCok2t7bAhw= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= -github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.0 h1:2G1gYpeHw4GhLet4Ebp5q9wpnSCAOJNTiJq+I3wJV5I= github.com/go-errors/errors v1.0.0/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -92,66 +56,31 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f h1:XXzyYlFbxK3kWfcmu3Wc+Tv8/QQl/VqwsWuSYF1Rj0s= -github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= @@ -159,45 +88,24 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -205,110 +113,54 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= -github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= -github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= -github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= -github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 h1:C7kWARE8r64ppRadl40yfNo6pag+G6ocvGU2xZ6yNes= go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -320,173 +172,89 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f h1:EeAYAq+xwFDtvQfGzVRYTNZBFZoKRC5olOUHqOOrZOA= go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 h1:64ChN/hjER/taL4YJuA+gpLfIMT+/NFherRZixbxOhg= -golang.org/x/sys v0.0.0-20210326220804-49726bf1d181/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 h1:ZBu6861dZq7xBnG1bn5SRU0vA8nx42at4+kP07FMTog= golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601 h1:9VBRTdmgQxbs6HE0sUnMrSWNePppAJU07NYvX5dIB04= -google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pfsagentd/README.md b/pfsagentd/README.md index e890b32a..85429e92 100644 --- a/pfsagentd/README.md +++ b/pfsagentd/README.md @@ -74,6 +74,7 @@ ReaddirMaxEntries: 1024 FUSEMaxBackground: 100 FUSECongestionThreshhold: 0 FUSEMaxWrite: 131072 +FUSEAllowOther: true RetryRPCPublicIPAddr: 127.0.0.1 RetryRPCPort: 32356 RetryRPCDeadlineIO: 60s diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index 29604264..00c64b52 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -50,6 +50,7 @@ func performMountFUSE() { globals.config.FUSEMountPointPath, // mountpointDirPath string pfsagentFuseSubtype, // fuseSubtype string globals.config.FUSEMaxWrite, // initOutMaxWrite uint32 + globals.config.FUSEAllowOther, // allowOther bool &globals, // callbacks fission.Callbacks newLogger(), // logger *log.Logger globals.fissionErrChan) // errChan chan error diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go index fb873156..ffd8666f 100644 --- a/pfsagentd/globals.go +++ b/pfsagentd/globals.go @@ -70,6 +70,7 @@ type configStruct struct { FUSEMaxBackground uint16 FUSECongestionThreshhold uint16 FUSEMaxWrite uint32 + FUSEAllowOther bool RetryRPCPublicIPAddr string RetryRPCPort uint16 RetryRPCDeadlineIO time.Duration @@ -635,6 +636,11 @@ func initializeGlobals(confMap conf.ConfMap) { logFatal(err) } + globals.config.FUSEAllowOther, err = confMap.FetchOptionValueBool("Agent", "FUSEAllowOther") + if nil != err { + logFatal(err) + } + globals.config.RetryRPCPublicIPAddr, err = confMap.FetchOptionValueString("Agent", "RetryRPCPublicIPAddr") if nil != err { logFatal(err) diff --git a/pfsagentd/pfsagent.conf b/pfsagentd/pfsagent.conf index ebf2e1c7..b34ee52d 100644 --- a/pfsagentd/pfsagent.conf +++ b/pfsagentd/pfsagent.conf @@ -40,6 +40,7 @@ ReaddirMaxEntries: 1024 FUSEMaxBackground: 100 FUSECongestionThreshhold: 0 FUSEMaxWrite: 131072 +FUSEAllowOther: true RetryRPCPublicIPAddr: 127.0.0.1 RetryRPCPort: 32356 RetryRPCDeadlineIO: 60s diff --git a/pfsagentd/setup_teardown_test.go b/pfsagentd/setup_teardown_test.go index 916e1cbc..d55990be 100644 --- a/pfsagentd/setup_teardown_test.go +++ b/pfsagentd/setup_teardown_test.go @@ -124,6 +124,7 @@ func testSetup(t *testing.T) { "Agent.FUSEMaxBackground=100", "Agent.FUSECongestionThreshhold=0", "Agent.FUSEMaxWrite=131072", // Linux max... 128KiB is good enough for testing + "Agent.FUSEAllowOther=false", "Agent.RetryRPCPublicIPAddr=" + testProxyFSDaemonIPAddr, "Agent.RetryRPCPort=" + testRetryRPCPort, "Agent.RetryRPCDeadlineIO=60s", From f1c4eff9624dff283207994424a707319f25623c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 2 Jul 2021 11:31:32 -0700 Subject: [PATCH 019/258] Restore "un-tidy" go.{mod|sum} but reference new package fission --- go.mod | 25 +++--- go.sum | 250 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 251 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index c24aabb8..94073ef1 100644 --- a/go.mod +++ b/go.mod @@ -8,32 +8,27 @@ require ( github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec github.com/ansel1/merry v1.0.1 - github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 + github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/go-errors/errors v1.0.0 // indirect github.com/gogo/protobuf v1.2.2-0.20190611061853-dadb62585089 // indirect - github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/google/btree v1.0.0 github.com/google/uuid v1.2.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/kr/pretty v0.2.0 // indirect - github.com/pkg/errors v0.8.1 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/pborman/uuid v1.2.1 // indirect + github.com/pelletier/go-toml v1.9.0 // indirect github.com/sirupsen/logrus v1.2.0 - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cobra v1.1.3 // indirect github.com/stretchr/testify v1.5.1 - github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 go.uber.org/multierr v1.5.0 // indirect - go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f // indirect - golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 // indirect + go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 golang.org/x/text v0.3.6 // indirect - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect - golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc // indirect - google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect ) diff --git a/go.sum b/go.sum index 0ba3a896..83fee111 100644 --- a/go.sum +++ b/go.sum @@ -1,52 +1,88 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 h1:SC+c6A1qTFstO9qmB86mPV2IpYme/2ZoEQ0hrP+wo+Q= bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= -github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0 h1:GQZP8OZqvkcMt7l7QAdtfa17PDUqiamz64df2DWNCDM= -github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTEXAJWEyRVTw5PClbHwe7O+QBnYsk= github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/ansel1/merry v1.0.1 h1:NEqlG+pyb6XKY55oRdGStyMkQ35YucmgFVG2tpybaP8= github.com/ansel1/merry v1.0.1/go.mod h1:qBYMZz+rgzOfZ3QACcTK7hE0bBMxvC0sKCok2t7bAhw= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= +github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.0 h1:2G1gYpeHw4GhLet4Ebp5q9wpnSCAOJNTiJq+I3wJV5I= github.com/go-errors/errors v1.0.0/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -56,31 +92,66 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f h1:XXzyYlFbxK3kWfcmu3Wc+Tv8/QQl/VqwsWuSYF1Rj0s= +github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= @@ -88,24 +159,45 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -113,54 +205,110 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= +github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 h1:C7kWARE8r64ppRadl40yfNo6pag+G6ocvGU2xZ6yNes= go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -172,89 +320,173 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f h1:EeAYAq+xwFDtvQfGzVRYTNZBFZoKRC5olOUHqOOrZOA= go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 h1:64ChN/hjER/taL4YJuA+gpLfIMT+/NFherRZixbxOhg= +golang.org/x/sys v0.0.0-20210326220804-49726bf1d181/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 h1:ZBu6861dZq7xBnG1bn5SRU0vA8nx42at4+kP07FMTog= golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601 h1:9VBRTdmgQxbs6HE0sUnMrSWNePppAJU07NYvX5dIB04= +google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 4e7402f58d40764a4e5f2d3b02df2850162641cb Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 2 Jul 2021 13:20:07 -0700 Subject: [PATCH 020/258] Attempting a clean go.{mod|sum} --- go.mod | 31 ---- go.sum | 492 --------------------------------------------------------- 2 files changed, 523 deletions(-) delete mode 100644 go.sum diff --git a/go.mod b/go.mod index 94073ef1..50cf0235 100644 --- a/go.mod +++ b/go.mod @@ -1,34 +1,3 @@ module github.com/NVIDIA/proxyfs go 1.15 - -require ( - bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 - github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a - github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e - github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec - github.com/ansel1/merry v1.0.1 - github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect - github.com/creachadair/cityhash v0.1.0 - github.com/dgraph-io/ristretto v0.0.3 // indirect - github.com/go-errors/errors v1.0.0 // indirect - github.com/gogo/protobuf v1.2.2-0.20190611061853-dadb62585089 // indirect - github.com/google/btree v1.0.0 - github.com/google/uuid v1.2.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml v1.9.0 // indirect - github.com/sirupsen/logrus v1.2.0 - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cobra v1.1.3 // indirect - github.com/stretchr/testify v1.5.1 - go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 - go.uber.org/multierr v1.5.0 // indirect - go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f - golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 - golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 - golang.org/x/text v0.3.6 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 83fee111..00000000 --- a/go.sum +++ /dev/null @@ -1,492 +0,0 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 h1:SC+c6A1qTFstO9qmB86mPV2IpYme/2ZoEQ0hrP+wo+Q= -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= -github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= -github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTEXAJWEyRVTw5PClbHwe7O+QBnYsk= -github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= -github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= -github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/ansel1/merry v1.0.1 h1:NEqlG+pyb6XKY55oRdGStyMkQ35YucmgFVG2tpybaP8= -github.com/ansel1/merry v1.0.1/go.mod h1:qBYMZz+rgzOfZ3QACcTK7hE0bBMxvC0sKCok2t7bAhw= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= -github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= -github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= -github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-errors/errors v1.0.0 h1:2G1gYpeHw4GhLet4Ebp5q9wpnSCAOJNTiJq+I3wJV5I= -github.com/go-errors/errors v1.0.0/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190611061853-dadb62585089 h1:raOeh+DEb2K79TKYcitwQUJOc4xdEOEJsgm8zQ6q1IU= -github.com/gogo/protobuf v1.2.2-0.20190611061853-dadb62585089/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f h1:XXzyYlFbxK3kWfcmu3Wc+Tv8/QQl/VqwsWuSYF1Rj0s= -github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= -github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= -github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= -github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= -github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 h1:C7kWARE8r64ppRadl40yfNo6pag+G6ocvGU2xZ6yNes= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f h1:EeAYAq+xwFDtvQfGzVRYTNZBFZoKRC5olOUHqOOrZOA= -go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 h1:64ChN/hjER/taL4YJuA+gpLfIMT+/NFherRZixbxOhg= -golang.org/x/sys v0.0.0-20210326220804-49726bf1d181/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 h1:ZBu6861dZq7xBnG1bn5SRU0vA8nx42at4+kP07FMTog= -golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601 h1:9VBRTdmgQxbs6HE0sUnMrSWNePppAJU07NYvX5dIB04= -google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 84de833d9870dbe1836b5d2b7f92ad4415037d1b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 2 Jul 2021 14:00:49 -0700 Subject: [PATCH 021/258] Attempt to utilize go mod replace statements to work around broken etcd dependencies --- go.mod | 43 ++++++++ go.sum | 333 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 376 insertions(+) create mode 100644 go.sum diff --git a/go.mod b/go.mod index 50cf0235..c0f94fd4 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,46 @@ module github.com/NVIDIA/proxyfs go 1.15 + +replace github.com/coreos/bbolt v1.3.4 => go.etcd.io/bbolt v1.3.4 + +replace go.etcd.io/bbolt v1.3.4 => github.com/coreos/bbolt v1.3.4 + +replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 + +require ( + bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 + github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a + github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e + github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec + github.com/ansel1/merry v1.6.1 + github.com/coreos/bbolt v1.3.4 // indirect + github.com/coreos/etcd v3.3.25+incompatible // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/creachadair/cityhash v0.1.0 + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/btree v1.0.1 + github.com/google/uuid v1.2.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/prometheus/client_golang v1.11.0 // indirect + github.com/sirupsen/logrus v1.8.1 + github.com/soheilhy/cmux v0.1.5 // indirect + github.com/stretchr/testify v1.7.0 + github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/bbolt v1.3.4 // indirect + go.etcd.io/etcd v3.3.25+incompatible + go.uber.org/zap v1.18.1 // indirect + golang.org/x/net v0.0.0-20210614182718-04defd469f4e + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c + golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..515f4dd2 --- /dev/null +++ b/go.sum @@ -0,0 +1,333 @@ +bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= +bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= +github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= +github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTEXAJWEyRVTw5PClbHwe7O+QBnYsk= +github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= +github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= +github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/ansel1/merry v1.5.0/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= +github.com/ansel1/merry v1.5.1/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= +github.com/ansel1/merry v1.6.1 h1:AuheQTqQ04yQdBMCDnneVPvpBdWTBBg3LKMvKGMyRxU= +github.com/ansel1/merry v1.6.1/go.mod h1:ioJjPJ/IsjxH+cC0lpf5TmbKnbcGa9qTk0fDbeRfnGQ= +github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZtKWYbsCus= +github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= +github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/coreos/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= +github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= +github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= +go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= +golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66 h1:EqVh9e7SxFnv93tWN3j33DpFAMKlFhaqI736Yp4kynk= +golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 h1:bXUwz2WkXXrXgiLxww3vWmoSHLOGv4ipdPdTvKymcKw= +google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 749dc7594dae2cbe29389f70abeb5a002b35111b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 4 Jun 2021 10:38:25 -0700 Subject: [PATCH 022/258] Updated SuperBlockV1Struct to track Objects to be deleted after CheckPoint successfully posted Also added paranoia checks that ilayout marshaling func's properly computed []byte size --- ilayout/api.go | 18 ++++--- ilayout/api_test.go | 16 +++++- ilayout/impl.go | 89 ++++++++++++++++++++++++++------ imgr/imgrpkg/http-server_test.go | 2 +- 4 files changed, 100 insertions(+), 25 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index bbc943bb..6a22f730 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -183,18 +183,22 @@ type InodeTableLayoutEntryV1Struct struct { // count of the number of InodeTableLayoutEntryV1Struct's followed by the // serialization of each one. // +// The PendingDeleteObjectNumberArray is serialized as a preceeding LittleEndian +// count of the number of ObjectNumbers followed by each LittleEndian ObjectNumber. +// // Note that the CheckPointV1Struct.SuperBlockLength also includes the bytes for holding // the ObjectTrailerStruct{ObjType: SuperBlockType, Version: SuperBlockVersionV1} that is // appended. // type SuperBlockV1Struct struct { - InodeTableRootObjectNumber uint64 // Identifies the Object containing the root of the InodeTable - InodeTableRootObjectOffset uint64 // Starting offset in the Object of the root of the InodeTable - InodeTableRootObjectLength uint64 // Number of bytes in the Object of the root of the InodeTable - InodeTableLayout []InodeTableLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable - InodeObjectCount uint64 // Number of Objects holding Inodes - InodeObjectSize uint64 // Sum of sizes of all Objects holding Inodes - InodeBytesReferenced uint64 // Sum of bytes referenced in all Objects holding Inodes + InodeTableRootObjectNumber uint64 // Identifies the Object containing the root of the InodeTable + InodeTableRootObjectOffset uint64 // Starting offset in the Object of the root of the InodeTable + InodeTableRootObjectLength uint64 // Number of bytes in the Object of the root of the InodeTable + InodeTableLayout []InodeTableLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable + InodeObjectCount uint64 // Number of Objects holding Inodes + InodeObjectSize uint64 // Sum of sizes of all Objects holding Inodes + InodeBytesReferenced uint64 // Sum of bytes referenced in all Objects holding Inodes + PendingDeleteObjectNumberArray []uint64 // List of Objects to be deleted after the this CheckPoint } // MarshalSuperBlockV1 encodes superBlockV1 to superBlockV1Buf. diff --git a/ilayout/api_test.go b/ilayout/api_test.go index 986ade9f..5312bc1d 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -52,12 +52,18 @@ func TestAPI(t *testing.T) { InodeObjectCount: 4, InodeObjectSize: 5, InodeBytesReferenced: 6, + PendingDeleteObjectNumberArray: []uint64{ + 7, + 8, + 9, + }, } marshaledSuperBlockV1 []byte unmarshaledSuperBlockV1 *SuperBlockV1Struct - inodeTableLayoutIndex int + inodeTableLayoutIndex int + pendingDeleteObjectNumberArrayIndex int testInodeTableEntryValueV1 = &InodeTableEntryValueV1Struct{ InodeHeadObjectNumber: 2, @@ -197,7 +203,8 @@ func TestAPI(t *testing.T) { if (testSuperBlockV1.InodeTableRootObjectNumber != unmarshaledSuperBlockV1.InodeTableRootObjectNumber) || (testSuperBlockV1.InodeTableRootObjectOffset != unmarshaledSuperBlockV1.InodeTableRootObjectOffset) || (testSuperBlockV1.InodeTableRootObjectLength != unmarshaledSuperBlockV1.InodeTableRootObjectLength) || - (len(testSuperBlockV1.InodeTableLayout) != len(unmarshaledSuperBlockV1.InodeTableLayout)) { + (len(testSuperBlockV1.InodeTableLayout) != len(unmarshaledSuperBlockV1.InodeTableLayout)) || + (len(testSuperBlockV1.PendingDeleteObjectNumberArray) != len(unmarshaledSuperBlockV1.PendingDeleteObjectNumberArray)) { t.Fatalf("Bad unmarshaledSuperBlockV1 (%+v) - expected testSuperBlockV1 (%+v) [Case 1]", unmarshaledSuperBlockV1, testSuperBlockV1) } for inodeTableLayoutIndex = range testSuperBlockV1.InodeTableLayout { @@ -205,6 +212,11 @@ func TestAPI(t *testing.T) { t.Fatalf("Bad unmarshaledSuperBlockV1 (%+v) - expected testSuperBlockV1 (%+v) [Case 2]", unmarshaledSuperBlockV1, testSuperBlockV1) } } + for pendingDeleteObjectNumberArrayIndex = range testSuperBlockV1.PendingDeleteObjectNumberArray { + if testSuperBlockV1.PendingDeleteObjectNumberArray[pendingDeleteObjectNumberArrayIndex] != unmarshaledSuperBlockV1.PendingDeleteObjectNumberArray[pendingDeleteObjectNumberArrayIndex] { + t.Fatalf("Bad unmarshaledSuperBlockV1 (%+v) - expected testSuperBlockV1 (%+v) [Case 3]", unmarshaledSuperBlockV1, testSuperBlockV1) + } + } marshaledInodeTableEntryValueV1, err = testInodeTableEntryValueV1.MarshalInodeTableEntryValueV1() if nil != err { diff --git a/ilayout/impl.go b/ilayout/impl.go index 65bbc9df..e5a85e68 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -52,11 +52,16 @@ func (objectTrailer *ObjectTrailerStruct) marshalObjectTrailer() (objectTrailerB return } - _, err = putLEUint32ToBuf(objectTrailerBuf, curPos, objectTrailer.Length) + curPos, err = putLEUint32ToBuf(objectTrailerBuf, curPos, objectTrailer.Length) if nil != err { return } + if curPos != len(objectTrailerBuf) { + err = fmt.Errorf("curPos != len(objectTrailerBuf)") + return + } + err = nil return } @@ -112,13 +117,14 @@ func unmarshalObjectTrailer(objectTrailerBuf []byte) (objectTrailer *ObjectTrail func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf []byte, err error) { var ( - curPos int - inodeTableLayoutIndex int - objectTrailer *ObjectTrailerStruct - objectTrailerBuf []byte + curPos int + inodeTableLayoutIndex int + objectTrailer *ObjectTrailerStruct + objectTrailerBuf []byte + pendingDeleteObjectNumberArrayIndex int ) - superBlockV1Buf = make([]byte, 8+8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8+8+8+(2+2+4)) + superBlockV1Buf = make([]byte, 8+8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8+8+8+8+(len(superBlockV1.PendingDeleteObjectNumberArray)*8)+(2+2+4)) curPos = 0 @@ -174,6 +180,18 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ return } + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, uint64(len(superBlockV1.PendingDeleteObjectNumberArray))) + if nil != err { + return + } + + for pendingDeleteObjectNumberArrayIndex = 0; pendingDeleteObjectNumberArrayIndex < len(superBlockV1.PendingDeleteObjectNumberArray); pendingDeleteObjectNumberArrayIndex++ { + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.PendingDeleteObjectNumberArray[pendingDeleteObjectNumberArrayIndex]) + if nil != err { + return + } + } + if curPos > math.MaxUint32 { err = fmt.Errorf("cannot marshal an superBlockV1Buf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) return @@ -190,21 +208,28 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ return } - _, err = putFixedByteSliceToBuf(superBlockV1Buf, curPos, objectTrailerBuf) + curPos, err = putFixedByteSliceToBuf(superBlockV1Buf, curPos, objectTrailerBuf) if nil != err { return } + if curPos != len(superBlockV1Buf) { + err = fmt.Errorf("curPos != len(superBlockV1Buf)") + return + } + err = nil return } func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1Struct, err error) { var ( - curPos int - inodeTableLayoutIndex uint64 - inodeTableLayoutLen uint64 - objectTrailer *ObjectTrailerStruct + curPos int + inodeTableLayoutIndex uint64 + inodeTableLayoutLen uint64 + objectTrailer *ObjectTrailerStruct + pendingDeleteObjectNumberArrayIndex uint64 + pendingDeleteObjectNumberArrayLen uint64 ) objectTrailer, err = unmarshalObjectTrailer(superBlockV1Buf) @@ -278,6 +303,20 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St return } + pendingDeleteObjectNumberArrayLen, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + superBlockV1.PendingDeleteObjectNumberArray = make([]uint64, pendingDeleteObjectNumberArrayLen) + + for pendingDeleteObjectNumberArrayIndex = 0; pendingDeleteObjectNumberArrayIndex < pendingDeleteObjectNumberArrayLen; pendingDeleteObjectNumberArrayIndex++ { + superBlockV1.PendingDeleteObjectNumberArray[pendingDeleteObjectNumberArrayIndex], curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + } + if curPos != int(objectTrailer.Length) { err = fmt.Errorf("incorrect size for superBlockV1Buf") return @@ -312,11 +351,16 @@ func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) marshalInodeTableEnt return } - _, err = putLEUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadLength) + curPos, err = putLEUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadLength) if nil != err { return } + if curPos != len(inodeTableEntryValueV1Buf) { + err = fmt.Errorf("curPos != len(inodeTableEntryValueV1Buf)") + return + } + err = nil return } @@ -527,11 +571,16 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt return } - _, err = putFixedByteSliceToBuf(inodeHeadV1Buf, curPos, objectTrailerBuf) + curPos, err = putFixedByteSliceToBuf(inodeHeadV1Buf, curPos, objectTrailerBuf) if nil != err { return } + if curPos != len(inodeHeadV1Buf) { + err = fmt.Errorf("curPos != len(inodeHeadV1Buf)") + return + } + err = nil return } @@ -716,11 +765,16 @@ func (directoryEntryValueV1 *DirectoryEntryValueV1Struct) marshalDirectoryEntryV return } - _, err = putLEUint8ToBuf(directoryEntryValueV1Buf, curPos, directoryEntryValueV1.InodeType) + curPos, err = putLEUint8ToBuf(directoryEntryValueV1Buf, curPos, directoryEntryValueV1.InodeType) if nil != err { return } + if curPos != len(directoryEntryValueV1Buf) { + err = fmt.Errorf("curPos != len(directoryEntryValueV1Buf)") + return + } + err = nil return } @@ -774,11 +828,16 @@ func (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct) marshalExtentMapEntryV return } - _, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.ObjectOffset) + curPos, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.ObjectOffset) if nil != err { return } + if curPos != len(extentMapEntryValueV1Buf) { + err = fmt.Errorf("curPos != len(extentMapEntryValueV1Buf)") + return + } + err = nil return } diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index bb814089..4a249fc8 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -65,7 +65,7 @@ func TestHTTPServer(t *testing.T) { if nil != err { t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL/ilayout.CheckPointObjectNumber, getRequestHeaders, nil) failed: %v", err) } - if "0000000000000001 0000000000000003 0000000000000058 0000000000000003" != string(responseBody[:]) { + if "0000000000000001 0000000000000003 0000000000000060 0000000000000003" != string(responseBody[:]) { t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL/ilayout.CheckPointObjectNumber, getRequestHeaders, nil) returned unexpected Object List: \"%s\"", string(responseBody[:])) } From ea7247f77a4ae97f688512cf99448b67678b3f9b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 5 Jun 2021 10:17:48 -0700 Subject: [PATCH 023/258] Added proper imgrpkg object delete tracking lists Also fixed some "lint" complaints (including several "yoda prohibitions") --- imgr/imgrpkg/globals.go | 33 +++++++++++---------- imgr/imgrpkg/retry-rpc.go | 37 +++++++++++++---------- imgr/imgrpkg/swift-client.go | 32 ++++++++++---------- imgr/imgrpkg/volume.go | 57 ++++++++++++++++++------------------ 4 files changed, 83 insertions(+), 76 deletions(-) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 648a7689..d8e0158f 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -220,22 +220,23 @@ type inodeTableLayoutElementStruct struct { } type volumeStruct struct { - name string // - storageURL string // - mountMap map[string]*mountStruct // key == mountStruct.mountID - healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false - leasesExpiredMountList *list.List // list of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value - authTokenExpiredMountList *list.List // list of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) - deleting bool // - checkPoint *ilayout.CheckPointV1Struct // == nil if not currently mounted and/or checkpointing - superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing - inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing; key == inodeNumber; value == *ilayout.InodeTableEntryValueV1Struct - inodeTableLayout map[uint64]*inodeTableLayoutElementStruct // == nil if not currently mounted and/or checkpointing; key == objectNumber (matching ilayout.InodeTableLayoutEntryV1Struct.ObjectNumber) - pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber - checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() - checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG - inodeLeaseMap map[uint64]*inodeLeaseStruct // key == inodeLeaseStruct.inodeNumber - leaseHandlerWG sync.WaitGroup // .Add(1) each inodeLease insertion into inodeLeaseMap + name string // + storageURL string // + mountMap map[string]*mountStruct // key == mountStruct.mountID + healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false + leasesExpiredMountList *list.List // list of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value + authTokenExpiredMountList *list.List // list of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) + deleting bool // + checkPoint *ilayout.CheckPointV1Struct // == nil if not currently mounted and/or checkpointing + superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing + inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing; key == inodeNumber; value == *ilayout.InodeTableEntryValueV1Struct + inodeTableLayout map[uint64]*inodeTableLayoutElementStruct // == nil if not currently mounted and/or checkpointing; key == objectNumber (matching ilayout.InodeTableLayoutEntryV1Struct.ObjectNumber) + activeObjectNumberDeleteList *list.List // list of objectNumber's to be deleted since last CheckPoint + pendingObjectNumberDeleteList *list.List // list of objectNumber's pending deletion after next CheckPoint + checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() + checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG + inodeLeaseMap map[uint64]*inodeLeaseStruct // key == inodeLeaseStruct.inodeNumber + leaseHandlerWG sync.WaitGroup // .Add(1) each inodeLease insertion into inodeLeaseMap // .Done() each inodeLease after it is removed from inodeLeaseMap } diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 64f0c727..7edb09bb 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -24,9 +24,9 @@ func startRetryRPCServer() (err error) { tlsCertificate tls.Certificate ) - if "" == globals.config.RetryRPCCertFilePath { // && "" == globals.config.RetryRPCKeyFilePath + if globals.config.RetryRPCCertFilePath == "" { // && globals.config.RetryRPCKeyFilePath == "" tlsCertificate = tls.Certificate{} - } else { // ("" != globals.config.RetryRPCCertFilePath) && ("" != globals.config.RetryRPCKeyFilePath) + } else { // (globals.config.RetryRPCCertFilePath != "") && (globals.config.RetryRPCKeyFilePath != "") tlsCertificate, err = tls.LoadX509KeyPair(globals.config.RetryRPCCertFilePath, globals.config.RetryRPCKeyFilePath) if nil != err { return @@ -74,20 +74,21 @@ func stopRetryRPCServer() (err error) { func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { var ( - alreadyInGlobalsMountMap bool - inodeTableEntryInMemory *inodeTableLayoutElementStruct - inodeTableEntryOnDisk ilayout.InodeTableLayoutEntryV1Struct - lastCheckPoint *ilayout.CheckPointV1Struct - lastCheckPointAsByteSlice []byte - lastCheckPointAsString string - mount *mountStruct - mountIDAsByteArray []byte - mountIDAsString string - ok bool - startTime time.Time = time.Now() - superBlockAsByteSlice []byte - volume *volumeStruct - volumeAsValue sortedmap.Value + alreadyInGlobalsMountMap bool + inodeTableEntryInMemory *inodeTableLayoutElementStruct + inodeTableEntryOnDisk ilayout.InodeTableLayoutEntryV1Struct + lastCheckPoint *ilayout.CheckPointV1Struct + lastCheckPointAsByteSlice []byte + lastCheckPointAsString string + mount *mountStruct + mountIDAsByteArray []byte + mountIDAsString string + ok bool + superBlockPendingDeleteObjectNumber uint64 + startTime time.Time = time.Now() + superBlockAsByteSlice []byte + volume *volumeStruct + volumeAsValue sortedmap.Value ) defer func() { @@ -185,6 +186,10 @@ retryGenerateMountID: volume.inodeTableLayout[inodeTableEntryOnDisk.ObjectNumber] = inodeTableEntryInMemory } + for _, superBlockPendingDeleteObjectNumber = range volume.superBlock.PendingDeleteObjectNumberArray { + volume.activeObjectNumberDeleteList.PushBack(superBlockPendingDeleteObjectNumber) + } + volume.checkPointControlChan = make(chan chan error) volume.checkPointControlWG.Add(1) diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index 66e539d0..4f677084 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -145,24 +145,24 @@ func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (b return } - if "" != authToken { + if authToken != "" { httpRequest.Header["X-Auth-Token"] = []string{authToken} } httpResponse, err = globals.httpClient.Do(httpRequest) if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", storageURL, err) return } buf, err = ioutil.ReadAll(httpResponse.Body) if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) return } err = httpResponse.Body.Close() if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) return } @@ -210,24 +210,24 @@ func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint6 httpRequest.Header["Range"] = []string{rangeHeaderValue} - if "" != authToken { + if authToken != "" { httpRequest.Header["X-Auth-Token"] = []string{authToken} } httpResponse, err = globals.httpClient.Do(httpRequest) if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", storageURL, err) return } buf, err = ioutil.ReadAll(httpResponse.Body) if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) return } err = httpResponse.Body.Close() if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) return } @@ -275,24 +275,24 @@ func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64 httpRequest.Header["Range"] = []string{rangeHeaderValue} - if "" != authToken { + if authToken != "" { httpRequest.Header["X-Auth-Token"] = []string{authToken} } httpResponse, err = globals.httpClient.Do(httpRequest) if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", storageURL, err) return } buf, err = ioutil.ReadAll(httpResponse.Body) if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) return } err = httpResponse.Body.Close() if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) return } @@ -338,24 +338,24 @@ func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, bo return } - if "" != authToken { + if authToken != "" { httpRequest.Header["X-Auth-Token"] = []string{authToken} } httpResponse, err = globals.httpClient.Do(httpRequest) if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", storageURL, err) return } _, err = ioutil.ReadAll(httpResponse.Body) if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) return } err = httpResponse.Body.Close() if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) return } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index d42e00bd..cf261740 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -59,7 +59,7 @@ func (dummy *globalsStruct) DumpValue(value sortedmap.Value) (valueAsString stri valueAsVolume, ok = value.(*volumeStruct) if ok { - valueAsString = fmt.Sprintf("%s", valueAsVolume.storageURL) + valueAsString = valueAsVolume.storageURL err = nil } else { err = fmt.Errorf("volumeMap's DumpValue(%v) called for non-*volumeStruct", value) @@ -95,7 +95,7 @@ func deleteVolume(volumeName string) (err error) { // The following is only temporary... // TODO: Actually gracefully unmount clients, block new mounts, and lazily remove it - if 0 != len(volumeAsStruct.mountMap) { + if len(volumeAsStruct.mountMap) != 0 { logFatalf("No support for deleting actively mounted volume \"%s\"", volumeName) } @@ -266,17 +266,17 @@ func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksSt } func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } @@ -325,7 +325,7 @@ func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksSt } func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } @@ -364,7 +364,7 @@ func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksSt } func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } @@ -416,17 +416,17 @@ func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCal } func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } @@ -475,7 +475,7 @@ func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCal } func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } @@ -497,7 +497,7 @@ func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCal } func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { - err = fmt.Errorf("Not implemented") + err = fmt.Errorf("not implemented") return } @@ -714,20 +714,21 @@ func putVolume(name string, storageURL string) (err error) { ) volume = &volumeStruct{ - name: name, - storageURL: storageURL, - mountMap: make(map[string]*mountStruct), - healthyMountList: list.New(), - leasesExpiredMountList: list.New(), - authTokenExpiredMountList: list.New(), - deleting: false, - checkPoint: nil, - superBlock: nil, - inodeTable: nil, - inodeTableLayout: nil, - pendingObjectDeleteSet: make(map[uint64]struct{}), - checkPointControlChan: nil, - inodeLeaseMap: make(map[uint64]*inodeLeaseStruct), + name: name, + storageURL: storageURL, + mountMap: make(map[string]*mountStruct), + healthyMountList: list.New(), + leasesExpiredMountList: list.New(), + authTokenExpiredMountList: list.New(), + deleting: false, + checkPoint: nil, + superBlock: nil, + inodeTable: nil, + inodeTableLayout: nil, + activeObjectNumberDeleteList: list.New(), + pendingObjectNumberDeleteList: list.New(), + checkPointControlChan: nil, + inodeLeaseMap: make(map[uint64]*inodeLeaseStruct), } globals.Lock() @@ -760,14 +761,14 @@ func (volume *volumeStruct) checkPointDaemon(checkPointControlChan chan chan err checkPointIntervalTimer = time.NewTimer(globals.config.CheckPointInterval) select { - case _ = <-checkPointIntervalTimer.C: + case <-checkPointIntervalTimer.C: err = volume.doCheckPoint() if nil != err { logWarnf("checkPointIntervalTimer-triggered doCheckPoint() failed: %v", err) } case checkPointResponseChan, more = <-checkPointControlChan: if !checkPointIntervalTimer.Stop() { - _ = <-checkPointIntervalTimer.C + <-checkPointIntervalTimer.C } if more { From 0606460b325b669bd5aabf7286701f629be73983 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 7 Jun 2021 11:17:37 -0700 Subject: [PATCH 024/258] Implemented imgrpkg::PutInodeTableEntries() --- imgr/imgrpkg/api.go | 8 ++-- imgr/imgrpkg/retry-rpc.go | 73 ++++++++++++++++++++++++++++++++-- imgr/imgrpkg/retry-rpc_test.go | 69 ++++++++++++++++++++++++-------- 3 files changed, 126 insertions(+), 24 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 4785be86..1d5b8137 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -284,19 +284,19 @@ type PutInodeTableEntryStruct struct { // (which must have an active Exclusive Lease for every PutInodeTableEntryStruct.InodeNumber // granted to the MountID). // -// Note that dereferenced objects listed in the DereferencedObjectNumberArray will -// not be deleted until the next CheckPoint is performed. -// // The SuperBlockInode{ObjectCount|ObjectSize|BytesReferenced}Adjustment fields // are used to update the corresponding fields in the volume's SuperBlock. // +// Note that dereferenced objects listed in the DereferencedObjectNumberArray will +// not be deleted until the next CheckPoint is performed. +// type PutInodeTableEntriesRequestStruct struct { MountID string UpdatedInodeTableEntryArray []PutInodeTableEntryStruct - DereferencedObjectNumberArray []uint64 SuperBlockInodeObjectCountAdjustment int64 SuperBlockInodeObjectSizeAdjustment int64 SuperBlockInodeBytesReferencedAdjustment int64 + DereferencedObjectNumberArray []uint64 } // PutInodeTableEntriesResponseStruct is the response object for PutInodeTableEntries. diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 7edb09bb..12c70386 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -349,8 +349,6 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru return } - volume = mount.volume - if mount.authTokenHasExpired() { globals.Unlock() err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) @@ -364,6 +362,8 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru return } + volume = mount.volume + inodeTableEntryValueRaw, ok, err = volume.inodeTable.GetByKey(getInodeTableEntryRequest.InodeNumber) if nil != err { logFatalf("volume.inodeTable.GetByKey(getInodeTableEntryRequest.InodeNumber) failed: %v", err) @@ -390,14 +390,79 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { var ( - startTime time.Time = time.Now() + dereferencedObjectNumber uint64 + inodeTableEntryValue *ilayout.InodeTableEntryValueV1Struct + leaseRequest *leaseRequestStruct + mount *mountStruct + ok bool + putInodeTableEntry PutInodeTableEntryStruct + startTime time.Time = time.Now() + volume *volumeStruct ) defer func() { globals.stats.PutInodeTableEntriesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf(ETODO + " putInodeTableEntries") + globals.Lock() + + mount, ok = globals.mountMap[putInodeTableEntriesRequest.MountID] + if !ok { + globals.Unlock() + err = fmt.Errorf("%s %s", EUnknownMountID, putInodeTableEntriesRequest.MountID) + return + } + + if mount.authTokenHasExpired() { + globals.Unlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + return + } + + for _, putInodeTableEntry = range putInodeTableEntriesRequest.UpdatedInodeTableEntryArray { + leaseRequest, ok = mount.leaseRequestMap[putInodeTableEntry.InodeNumber] + if !ok || (leaseRequestStateExclusiveGranted != leaseRequest.requestState) { + globals.Unlock() + err = fmt.Errorf("%s %016X", EMissingLease, putInodeTableEntry.InodeNumber) + return + } + } + + volume = mount.volume + + for _, putInodeTableEntry = range putInodeTableEntriesRequest.UpdatedInodeTableEntryArray { + inodeTableEntryValue = &ilayout.InodeTableEntryValueV1Struct{ + InodeHeadObjectNumber: putInodeTableEntry.InodeHeadObjectNumber, + InodeHeadLength: putInodeTableEntry.InodeHeadLength, + } + + ok, err = volume.inodeTable.PatchByKey(putInodeTableEntry.InodeNumber, inodeTableEntryValue) + if nil != err { + logFatalf("volume.inodeTable.PatchByKey(putInodeTableEntry.InodeNumber,) failed: %v", err) + } + if !ok { + ok, err = volume.inodeTable.Put(putInodeTableEntry.InodeNumber, inodeTableEntryValue) + if nil != err { + logFatalf("volume.inodeTable.Put(putInodeTableEntry.InodeNumber,) failed: %v", err) + } + if !ok { + logFatalf("volume.inodeTable.Put(putInodeTableEntry.InodeNumber,) returned !ok") + } + } + } + + volume.superBlock.InodeObjectCount = uint64(int64(volume.superBlock.InodeObjectCount) + putInodeTableEntriesRequest.SuperBlockInodeObjectCountAdjustment) + volume.superBlock.InodeObjectSize = uint64(int64(volume.superBlock.InodeObjectSize) + putInodeTableEntriesRequest.SuperBlockInodeObjectSizeAdjustment) + volume.superBlock.InodeBytesReferenced = uint64(int64(volume.superBlock.InodeBytesReferenced) + putInodeTableEntriesRequest.SuperBlockInodeBytesReferencedAdjustment) + + for _, dereferencedObjectNumber = range putInodeTableEntriesRequest.DereferencedObjectNumberArray { + _ = volume.pendingObjectNumberDeleteList.PushBack(dereferencedObjectNumber) + } + + globals.Unlock() + + err = nil + return } func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index 1acc1336..8f449121 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/iswift/iswiftpkg" "github.com/NVIDIA/proxyfs/retryrpc" ) @@ -42,6 +43,8 @@ func TestRetryRPC(t *testing.T) { mountRequest *MountRequestStruct mountResponse *MountResponseStruct postRequestBody string + putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct + putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct putRequestBody string renewMountRequest *RenewMountRequestStruct renewMountResponse *RenewMountResponseStruct @@ -125,13 +128,13 @@ func TestRetryRPC(t *testing.T) { getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ MountID: mountResponse.MountID, - InodeNumber: 1, + InodeNumber: ilayout.RootDirInodeNumber, } getInodeTableEntryResponse = &GetInodeTableEntryResponseStruct{} err = retryrpcClient.Send("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) if nil == err { - t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) should have failed") + t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,ilayout.RootDirInodeNumber)\",,) should have failed") } // Force a need for a re-auth @@ -142,13 +145,13 @@ func TestRetryRPC(t *testing.T) { getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ MountID: mountResponse.MountID, - InodeNumber: 1, + InodeNumber: ilayout.RootDirInodeNumber, } getInodeTableEntryResponse = &GetInodeTableEntryResponseStruct{} err = retryrpcClient.Send("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) if nil == err { - t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) should have failed") + t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,ilayout.RootDirInodeNumber)\",,) should have failed") } // Perform a RenewMount() @@ -173,53 +176,87 @@ func TestRetryRPC(t *testing.T) { leaseRequest = &LeaseRequestStruct{ MountID: mountResponse.MountID, - InodeNumber: 1, + InodeNumber: ilayout.RootDirInodeNumber, LeaseRequestType: LeaseRequestTypeShared, } leaseResponse = &LeaseResponseStruct{} err = retryrpcClient.Send("Lease", leaseRequest, leaseResponse) if nil != err { - t.Fatalf("retryrpcClient.Send(\"Lease(,1,LeaseRequestTypeShared)\",,) failed: %v", err) + t.Fatalf("retryrpcClient.Send(\"Lease(,ilayout.RootDirInodeNumber,LeaseRequestTypeShared)\",,) failed: %v", err) } // Perform a GetInodeTableEntry() for RootDirInode getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ MountID: mountResponse.MountID, - InodeNumber: 1, + InodeNumber: ilayout.RootDirInodeNumber, } getInodeTableEntryResponse = &GetInodeTableEntryResponseStruct{} err = retryrpcClient.Send("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) if nil != err { - t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) failed: %v", err) + t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,ilayout.RootDirInodeNumber)\",,) failed: %v", err) } - // TODO: Remove this early exit skipping of following TODOs + // Attempt a PutInodeTableEntries() for RootDirInode... which should fail (only Shared Lease) - if nil == err { - t.Logf("Exiting TestRetryRPC() early to skip following TODOs") - return + putInodeTableEntriesRequest = &PutInodeTableEntriesRequestStruct{ + MountID: mountResponse.MountID, + UpdatedInodeTableEntryArray: []PutInodeTableEntryStruct{ + { + InodeNumber: ilayout.RootDirInodeNumber, + InodeHeadObjectNumber: getInodeTableEntryResponse.InodeHeadObjectNumber, + InodeHeadLength: getInodeTableEntryResponse.InodeHeadLength, + }, + }, } + putInodeTableEntriesResponse = &PutInodeTableEntriesResponseStruct{} - // TODO: Attempt a PutInodeTableEntries() for RootDirInode... which should fail (only Shared Lease) + err = retryrpcClient.Send("PutInodeTableEntries", putInodeTableEntriesRequest, putInodeTableEntriesResponse) + if nil == err { + t.Fatalf("retryrpcClient.Send(\"PutInodeTableEntries(,{ilayout.RootDirInodeNumber,,})\",,) should have failed") + } // Perform a Lease Promote on RootDirInode leaseRequest = &LeaseRequestStruct{ MountID: mountResponse.MountID, - InodeNumber: 1, + InodeNumber: ilayout.RootDirInodeNumber, LeaseRequestType: LeaseRequestTypePromote, } leaseResponse = &LeaseResponseStruct{} err = retryrpcClient.Send("Lease", leaseRequest, leaseResponse) if nil != err { - t.Fatalf("retryrpcClient.Send(\"Lease(,1,LeaseRequestTypePromote)\",,) failed: %v", err) + t.Fatalf("retryrpcClient.Send(\"Lease(,ilayout.RootDirInodeNumber,LeaseRequestTypePromote)\",,) failed: %v", err) + } + + // Perform a PutInodeTableEntries() on RootDirInode + + putInodeTableEntriesRequest = &PutInodeTableEntriesRequestStruct{ + MountID: mountResponse.MountID, + UpdatedInodeTableEntryArray: []PutInodeTableEntryStruct{ + { + InodeNumber: ilayout.RootDirInodeNumber, + InodeHeadObjectNumber: getInodeTableEntryResponse.InodeHeadObjectNumber, + InodeHeadLength: getInodeTableEntryResponse.InodeHeadLength, + }, + }, + } + putInodeTableEntriesResponse = &PutInodeTableEntriesResponseStruct{} + + err = retryrpcClient.Send("PutInodeTableEntries", putInodeTableEntriesRequest, putInodeTableEntriesResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"PutInodeTableEntries(,{ilayout.RootDirInodeNumber,,})\",,) failed: %v", err) } - // TODO: Perform a PutInodeTableEntries() on RootDirInode + // TODO: Remove this early exit skipping of following TODOs + + if nil == err { + t.Logf("Exiting TestRetryRPC() early to skip following TODOs") + return + } // TODO: Create a FileInode (set LinkCount to 0 & no dir entry)... with small amount of data From 463747e955a55b54fada6ab3ebab6dfc0e8a9f53 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 16 Jul 2021 15:37:09 -0700 Subject: [PATCH 025/258] Implemented isomorphic mapping helpful to Swift for ObjectNumber <=> ObjectName Container Sharding in Swift is benefitted when the most rapidly changing characters in ObjectName's are the first characters --- ilayout/api.go | 58 +++++++++++++++++++++++++++ ilayout/api_test.go | 96 +++++++++++++++++++++++++++++++++++++++++++++ ilayout/impl.go | 85 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+) diff --git a/ilayout/api.go b/ilayout/api.go index 6a22f730..6abd6832 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -13,6 +13,26 @@ // in the file system) is ever written twice except for the CheckPoint described // below. // +// To achieve uniqueness of entities like Object Names, it is expected there is +// an isomorhic mapping from Object Numbers to/from Object Names. Uniqueness is +// achieved by utilizing a nonce sequence of 64-bit numbers, starting at zero +// and incrementing by one. The rate of creation of nonce-identified entities +// (like Objects) is certainly slow enough such that just remembering the highest +// nonce utilized is sufficient to ensure no nonce value is ever reused. +// +// Converting a nonce-assigned Object Number to/from an Object Name could be +// accomplished by simply converting between the 64-bit binary value and the +// corresponding 16 Hex Digit string. Unfortunately, some solutions (e.g. +// OpenStack Swift) address the challenge of very large Buckets/Containers via +// a prefix-based sharding operation. In essence, some string prefix of all +// Object Names in a particular shard will be common and no Objects Names with +// that same prefix will be in any other shard. Ideally, once a Bucket/Container +// has been sharded, new Objects placed in the shards will be evenly distributed +// (so as to keep each shard roughly the same size baring unbalanced deletions). +// Hence, the conversion of a slowly monitonically increasing sequence of +// Object Numbers is best served by reversing the digits in the corresponding +// Object Names. +// // At the top level, the file system periodically checkpoints. This enables the // file system to be complemented by the extreme scale of such Object Store // systems despite such often suffering long latencies and coarse granularity @@ -573,3 +593,41 @@ func PutFixedByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos i nextPos, err = putFixedByteSliceToBuf(buf, curPos, byteSlice) return } + +// GetObjectNameAsByteSlice returns the isomorphically mapped objectName as +// a []byte given a uint64 objectNumber. +// +func GetObjectNameAsByteSlice(objectNumber uint64) (objectName []byte) { + objectName = getObjectNameAsByteSlice(objectNumber) + return +} + +// GetObjectNameAsString returns the isomorphically mapped objectName as +// a string given a uint64 objectNumber. +// +func GetObjectNameAsString(objectNumber uint64) (objectName string) { + objectName = getObjectNameAsString(objectNumber) + return +} + +// GetObjectNumberFromByteSlice returns the isomorphically mapped uint64 objectNumber +// given a []byte objectName. +// +// An error will result if objectName is not of the proper length or contains +// invalid characters. +// +func GetObjectNumberFromByteSlice(objectName []byte) (objectNumber uint64, err error) { + objectNumber, err = getObjectNumberFromByteSlice(objectName) + return +} + +// GetObjectNumberFromString returns the isomorphically mapped uint64 objectNumber +// given a string objectName. +// +// An error will result if objectName is not of the proper length or contains +// invalid characters. +// +func GetObjectNumberFromString(objectName string) (objectNumber uint64, err error) { + objectNumber, err = getObjectNumberFromString(objectName) + return +} diff --git a/ilayout/api_test.go b/ilayout/api_test.go index 5312bc1d..b8c30c33 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -153,6 +153,26 @@ func TestAPI(t *testing.T) { unmarshaledExtentMapEntryValueV1 *ExtentMapEntryValueV1Struct unmarshaledExtentMapEntryValueV1BytesConsumed int unmarshaledExtentMapEntryValueV1BytesConsumedExpected = int(8 + 8 + 8 + 8) + + testObjectNumber = uint64(0x0123456789ABCDEF) + + testObjectGoodNameAsByteSlice = []byte{'F', 'E', 'D', 'C', 'B', 'A', '9', '8', '7', '6', '5', '4', '3', '2', '1', '0'} + testObjectBad1NameAsByteSlice = []byte{'/', 'E', 'D', 'C', 'B', 'A', '9', '8', '7', '6', '5', '4', '3', '2', '1', '0'} + testObjectBad2NameAsByteSlice = []byte{':', 'E', 'D', 'C', 'B', 'A', '9', '8', '7', '6', '5', '4', '3', '2', '1', '0'} + testObjectBad3NameAsByteSlice = []byte{'@', 'E', 'D', 'C', 'B', 'A', '9', '8', '7', '6', '5', '4', '3', '2', '1', '0'} + testObjectBad4NameAsByteSlice = []byte{'[', 'E', 'D', 'C', 'B', 'A', '9', '8', '7', '6', '5', '4', '3', '2', '1', '0'} + testObjectBad5NameAsByteSlice = []byte{'F', 'E', 'D'} + + testObjectGoodNameAsString = "FEDCBA9876543210" + testObjectBad1NameAsString = "/EDCBA9876543210" + testObjectBad2NameAsString = ":EDCBA9876543210" + testObjectBad3NameAsString = "@EDCBA9876543210" + testObjectBad4NameAsString = "[EDCBA9876543210" + testObjectBad5NameAsString = "FED" + + unmarshaledObjectNumber uint64 + marshaledObjectNameAsByteSlice []byte + marshaledObjectNameAsString string ) marshaledCheckPointV1, err = testCheckPointV1.MarshalCheckPointV1() @@ -317,4 +337,80 @@ func TestAPI(t *testing.T) { if unmarshaledExtentMapEntryValueV1BytesConsumed != unmarshaledExtentMapEntryValueV1BytesConsumedExpected { t.Fatalf("Bad unmarshaledExtentMapEntryValueV1BytesConsumed (%v) - expected %v", unmarshaledExtentMapEntryValueV1BytesConsumed, unmarshaledExtentMapEntryValueV1BytesConsumedExpected) } + + marshaledObjectNameAsByteSlice = GetObjectNameAsByteSlice(testObjectNumber) + if !bytes.Equal(marshaledObjectNameAsByteSlice, testObjectGoodNameAsByteSlice) { + t.Fatalf("Bad return from GetObjectNameAsByteSlice()") + } + + marshaledObjectNameAsString = GetObjectNameAsString(testObjectNumber) + if marshaledObjectNameAsString != testObjectGoodNameAsString { + t.Fatalf("Bad return from GetObjectNameAsString()") + } + + unmarshaledObjectNumber, err = GetObjectNumberFromByteSlice(testObjectGoodNameAsByteSlice) + if nil != err { + t.Fatalf("GetObjectNumberFromByteSlice(testObjectGoodNameAsByteSlice) failed: %v", err) + } + if unmarshaledObjectNumber != testObjectNumber { + t.Fatalf("GetObjectNumberFromByteSlice(testObjectGoodNameAsByteSlice) returned unexpected objectNumber") + } + + _, err = GetObjectNumberFromByteSlice(testObjectBad1NameAsByteSlice) + if nil == err { + t.Fatalf("GetObjectNumberFromByteSlice(testObjectBad1NameAsByteSlice) unexpectedly succeeded") + } + + _, err = GetObjectNumberFromByteSlice(testObjectBad2NameAsByteSlice) + if nil == err { + t.Fatalf("GetObjectNumberFromByteSlice(testObjectBad2NameAsByteSlice) unexpectedly succeeded") + } + + _, err = GetObjectNumberFromByteSlice(testObjectBad3NameAsByteSlice) + if nil == err { + t.Fatalf("GetObjectNumberFromByteSlice(testObjectBad3NameAsByteSlice) unexpectedly succeeded") + } + + _, err = GetObjectNumberFromByteSlice(testObjectBad4NameAsByteSlice) + if nil == err { + t.Fatalf("GetObjectNumberFromByteSlice(testObjectBad4NameAsByteSlice) unexpectedly succeeded") + } + + _, err = GetObjectNumberFromByteSlice(testObjectBad5NameAsByteSlice) + if nil == err { + t.Fatalf("GetObjectNumberFromByteSlice(testObjectBad5NameAsByteSlice) unexpectedly succeeded") + } + + unmarshaledObjectNumber, err = GetObjectNumberFromString(testObjectGoodNameAsString) + if nil != err { + t.Fatalf("GetObjectNumberFromString(testObjectGoodNameAsString) failed: %v", err) + } + if unmarshaledObjectNumber != testObjectNumber { + t.Fatalf("GetObjectNumberFromString(testObjectGoodNameAsString) returned unexpected objectNumber") + } + + _, err = GetObjectNumberFromString(testObjectBad1NameAsString) + if nil == err { + t.Fatalf("GetObjectNumberFromString(testObjectBad1NameAsString) unexpectedly succeeded") + } + + _, err = GetObjectNumberFromString(testObjectBad2NameAsString) + if nil == err { + t.Fatalf("GetObjectNumberFromString(testObjectBad2NameAsString) unexpectedly succeeded") + } + + _, err = GetObjectNumberFromString(testObjectBad3NameAsString) + if nil == err { + t.Fatalf("GetObjectNumberFromString(testObjectBad3NameAsString) unexpectedly succeeded") + } + + _, err = GetObjectNumberFromString(testObjectBad4NameAsString) + if nil == err { + t.Fatalf("GetObjectNumberFromString(testObjectBad4NameAsString) unexpectedly succeeded") + } + + _, err = GetObjectNumberFromString(testObjectBad5NameAsString) + if nil == err { + t.Fatalf("GetObjectNumberFromString(testObjectBad5NameAsString) unexpectedly succeeded") + } } diff --git a/ilayout/impl.go b/ilayout/impl.go index e5a85e68..f624a80e 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -1118,3 +1118,88 @@ func putFixedByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos i err = nil return } + +func getObjectNameAsByteSlice(objectNumber uint64) (objectName []byte) { + var ( + objectNameDigit uint8 + objectNameIndex int + ) + + objectName = make([]byte, 16) + + for objectNameIndex = 0; objectNameIndex < 16; objectNameIndex++ { + objectNameDigit = uint8(objectNumber & 0xF) + + if objectNameDigit < 0xA { + objectNameDigit += byte('0') + } else { + objectNameDigit += byte('A') - 0xA + } + + objectName[objectNameIndex] = objectNameDigit + + objectNumber = objectNumber >> 4 + } + + return +} + +func getObjectNameAsString(objectNumber uint64) (objectName string) { + var ( + objectNameAsByteSlice []byte = getObjectNameAsByteSlice(objectNumber) + ) + + objectName = string(objectNameAsByteSlice[:]) + + return +} + +func getObjectNumberFromByteSlice(objectName []byte) (objectNumber uint64, err error) { + var ( + objectNameDigit uint8 + objectNameIndex int + ) + + if len(objectName) != 16 { + err = fmt.Errorf("incorrect len(objectName) [%d] - expected 16", len(objectName)) + return + } + + objectNumber = 0 + + for objectNameIndex = 15; objectNameIndex >= 0; objectNameIndex-- { + objectNameDigit = objectName[objectNameIndex] + + if objectNameDigit < byte('0') { + err = fmt.Errorf("invalid character as index %d [0x%02X] - must be one of \"0123456789ABCDEF\"", objectNameIndex, objectNameDigit) + return + } else if objectNameDigit <= byte('9') { + objectNameDigit -= byte('0') + } else if objectNameDigit < byte('A') { + err = fmt.Errorf("invalid character as index %d [0x%02X] - must be one of \"0123456789ABCDEF\"", objectNameIndex, objectNameDigit) + return + } else if objectNameDigit <= byte('F') { + objectNameDigit -= byte('A') - 0xA + } else { + err = fmt.Errorf("invalid character as index %d [0x%02X] - must be one of \"0123456789ABCDEF\"", objectNameIndex, objectNameDigit) + return + } + + objectNumber = objectNumber << 4 + + objectNumber |= uint64(objectNameDigit) + } + + err = nil + return +} + +func getObjectNumberFromString(objectName string) (objectNumber uint64, err error) { + var ( + objectNameAsByteSlice []byte = []byte(objectName) + ) + + objectNumber, err = getObjectNumberFromByteSlice(objectNameAsByteSlice) + + return +} From 700882146fae19e457c0b56a0bdbb9cf5650d2b3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 19 Jul 2021 10:51:03 -0700 Subject: [PATCH 026/258] Finish object name mapping changes in imgr --- imgr/imgrpkg/http-server_test.go | 8 ++++---- imgr/imgrpkg/swift-client.go | 14 +++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index 4a249fc8..8dea252e 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -38,7 +38,7 @@ func TestHTTPServer(t *testing.T) { if nil != err { t.Fatalf("GET /volume [case 1] failed: %v", err) } - if "[]" != string(responseBody[:]) { + if string(responseBody[:]) != "[]" { t.Fatalf("GET /volume [case 1] should have returned \"[]\" - it returned \"%s\"", string(responseBody[:])) } @@ -57,7 +57,7 @@ func TestHTTPServer(t *testing.T) { if nil != err { t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL, getRequestHeaders, nil) failed: %v", err) } - if "0000000000000000\n0000000000000002\n0000000000000003\n" != string(responseBody[:]) { + if string(responseBody[:]) != ilayout.GetObjectNameAsString(ilayout.CheckPointObjectNumber)+"\n"+ilayout.GetObjectNameAsString(ilayout.CheckPointObjectNumber+2)+"\n"+ilayout.GetObjectNameAsString(ilayout.CheckPointObjectNumber+3)+"\n" { t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL, getRequestHeaders, nil) returned unexpected Object List: \"%s\"", string(responseBody[:])) } @@ -65,7 +65,7 @@ func TestHTTPServer(t *testing.T) { if nil != err { t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL/ilayout.CheckPointObjectNumber, getRequestHeaders, nil) failed: %v", err) } - if "0000000000000001 0000000000000003 0000000000000060 0000000000000003" != string(responseBody[:]) { + if string(responseBody[:]) != fmt.Sprintf("%016X %016X %016X %016X", ilayout.CheckPointVersionV1, ilayout.CheckPointObjectNumber+3, 96, ilayout.CheckPointObjectNumber+3) { t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL/ilayout.CheckPointObjectNumber, getRequestHeaders, nil) returned unexpected Object List: \"%s\"", string(responseBody[:])) } @@ -105,7 +105,7 @@ func TestHTTPServer(t *testing.T) { if nil != err { t.Fatalf("GET /volume [case 3] failed: %v", err) } - if "[]" != string(responseBody[:]) { + if string(responseBody[:]) != "[]" { t.Fatalf("GET /volume [case 3] should have returned \"[]\" - it returned \"%s\"", string(responseBody[:])) } diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index 4f677084..2a6e1be8 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -9,6 +9,8 @@ import ( "io/ioutil" "net/http" "time" + + "github.com/NVIDIA/proxyfs/ilayout" ) func startSwiftClient() (err error) { @@ -74,7 +76,7 @@ func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) globals.stats.SwiftObjectDeleteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + objectURL = storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) nextSwiftRetryDelay = globals.config.SwiftRetryDelay @@ -135,7 +137,7 @@ func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (b globals.stats.SwiftObjectGetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + objectURL = storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) nextSwiftRetryDelay = globals.config.SwiftRetryDelay @@ -197,7 +199,8 @@ func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint6 globals.stats.SwiftObjectGetRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + objectURL = storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) + rangeHeaderValue = fmt.Sprintf("bytes=%d-%d", objectOffset, (objectOffset + objectLength - 1)) nextSwiftRetryDelay = globals.config.SwiftRetryDelay @@ -262,7 +265,8 @@ func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64 globals.stats.SwiftObjectGetTailUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + objectURL = storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) + rangeHeaderValue = fmt.Sprintf("bytes=-%d", objectLength) nextSwiftRetryDelay = globals.config.SwiftRetryDelay @@ -326,7 +330,7 @@ func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, bo globals.stats.SwiftObjectPutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + objectURL = storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) nextSwiftRetryDelay = globals.config.SwiftRetryDelay From 059c74d6a749e844d9c9ea517dc346b2da6853ae Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 19 Jul 2021 11:09:04 -0700 Subject: [PATCH 027/258] Normalize retryrpc import sections --- retryrpc/api.go | 3 ++- retryrpc/client.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index 90dfd81b..37cb9ead 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -24,8 +24,9 @@ import ( "sync" "time" - "github.com/NVIDIA/proxyfs/bucketstats" "github.com/google/btree" + + "github.com/NVIDIA/proxyfs/bucketstats" ) // Server tracks the state of the server diff --git a/retryrpc/client.go b/retryrpc/client.go index 906b03e2..1c4d1c6a 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -13,8 +13,9 @@ import ( "os" "time" - "github.com/NVIDIA/proxyfs/bucketstats" "github.com/google/btree" + + "github.com/NVIDIA/proxyfs/bucketstats" ) const ( From d69f6570960be5276306d4c7c0c4d9a6be7e9d6a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 19 Jul 2021 11:27:01 -0700 Subject: [PATCH 028/258] Commented in top level Makefile all dependencies of re-work code base --- Makefile | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Makefile b/Makefile index 668f07a0..d08b59e1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,27 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 +# Rework reduces the list of active packages/programs/etc... to: +# +# bucketstats +# conf +# emswift +# emswift/emswiftpkg +# iauth +# iauth/iauth-swift +# icert +# icert/icertpkg +# iclient +# iclient/iclientpkg +# ilayout +# imgr +# imgr/imgrpkg +# iswift +# iswift/iswiftpkg +# retryrpc +# utils +# version + gopregeneratesubdirs = \ make-static-content From 1cf5c4aeac574f6b0a6bb17098d0d96d997f04f7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 19 Jul 2021 14:42:11 -0700 Subject: [PATCH 029/258] Added new Makefile.rework to only maintain builds for rework elements moving forward Also switched iauth's test to use iswift instead of the soon-to-be-obsolete emswift --- Makefile | 26 +- Makefile.rework | 114 +++++++ go.mod | 7 +- go.sum | 492 +++++++++++++++++++++++++++++++ iauth/iauth-swift/plugin_test.go | 30 +- 5 files changed, 629 insertions(+), 40 deletions(-) create mode 100644 Makefile.rework diff --git a/Makefile b/Makefile index d08b59e1..6a12e76f 100644 --- a/Makefile +++ b/Makefile @@ -1,27 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -# Rework reduces the list of active packages/programs/etc... to: -# -# bucketstats -# conf -# emswift -# emswift/emswiftpkg -# iauth -# iauth/iauth-swift -# icert -# icert/icertpkg -# iclient -# iclient/iclientpkg -# ilayout -# imgr -# imgr/imgrpkg -# iswift -# iswift/iswiftpkg -# retryrpc -# utils -# version - gopregeneratesubdirs = \ make-static-content @@ -125,7 +104,7 @@ endif pfsagent: pre-generate generate pfsagent-install -.PHONY: all all-deb-builder bench ci clean cover fmt generate install pfsagent pfsagent-install pre-generate python-test test version +.PHONY: all all-deb-builder bench ci clean cover fmt generate install pfsagent pfsagent-install pre-generate python-test test version rework bench: @set -e; \ @@ -229,3 +208,6 @@ test: version: @go version + +rework: + $(MAKE) -f Makefile.rework diff --git a/Makefile.rework b/Makefile.rework new file mode 100644 index 00000000..31f855c1 --- /dev/null +++ b/Makefile.rework @@ -0,0 +1,114 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gopkgdirs = \ + bucketstats \ + conf \ + iauth \ + ilayout \ + retryrpc \ + utils \ + version \ + icert/icertpkg \ + iclient/iclientpkg \ + imgr/imgrpkg \ + iswift/iswiftpkg + +goplugindirs = \ + iauth/iauth-swift + +gobindirs = \ + icert \ + iclient \ + imgr \ + iswift + +godirsforci = $(gopkgdirs) $(goplugindirs) $(gobindirs); +godirpathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(godirsforci)) + +all: version fmt generate install test + +ci: version fmt generate install test cover + +.PHONY: all bench ci clean cover fmt generate install test version + +bench: + @set -e; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir bench; \ + done; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir bench; \ + done; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir bench; \ + done + +clean: + @set -e; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir clean; \ + done; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir clean; \ + done; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir clean; \ + done + +cover: + @set -e; \ + go get -u github.com/ory/go-acc; \ + go-acc -o coverage.coverprofile $(godirpathsforci) + +fmt: + @set -e; \ + $(MAKE) --no-print-directory -C make-static-content fmt; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir fmt; \ + done; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir fmt; \ + done; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir fmt; \ + done + +generate: + @set -e; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir generate; \ + done; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir generate; \ + done; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir generate; \ + done + +install: + @set -e; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir install; \ + done; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir install; \ + done; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir install; \ + done + +test: + @set -e; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir test; \ + done; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir test; \ + done; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir test; \ + done; \ + +version: + @go version diff --git a/go.mod b/go.mod index c0f94fd4..fb24350f 100644 --- a/go.mod +++ b/go.mod @@ -20,19 +20,24 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.0.1 - github.com/google/uuid v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/pborman/uuid v1.2.1 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect diff --git a/go.sum b/go.sum index 515f4dd2..0c440ebb 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,45 @@ bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= @@ -8,6 +47,7 @@ github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTE github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -21,40 +61,71 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -62,19 +133,36 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= +github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -84,28 +172,88 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -113,24 +261,49 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -139,14 +312,32 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -156,44 +347,99 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -203,98 +449,325 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66 h1:EqVh9e7SxFnv93tWN3j33DpFAMKlFhaqI736Yp4kynk= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 h1:bXUwz2WkXXrXgiLxww3vWmoSHLOGv4ipdPdTvKymcKw= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -310,12 +783,21 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -324,10 +806,20 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/iauth/iauth-swift/plugin_test.go b/iauth/iauth-swift/plugin_test.go index c218faf4..b25e5592 100644 --- a/iauth/iauth-swift/plugin_test.go +++ b/iauth/iauth-swift/plugin_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/emswift/emswiftpkg" + "github.com/NVIDIA/proxyfs/iswift/iswiftpkg" ) func TestSwiftAuth(t *testing.T) { @@ -16,17 +16,13 @@ func TestSwiftAuth(t *testing.T) { authToken string confMap conf.ConfMap confStrings = []string{ - "EMSWIFT.AuthIPAddr=127.0.0.1", - "EMSWIFT.AuthTCPPort=9997", - "EMSWIFT.JRPCIPAddr=127.0.0.1", - "EMSWIFT.JRPCTCPPort=9998", - "EMSWIFT.NoAuthIPAddr=127.0.0.1", - "EMSWIFT.NoAuthTCPPort=9999", - "EMSWIFT.MaxAccountNameLength=256", - "EMSWIFT.MaxContainerNameLength=256", - "EMSWIFT.MaxObjectNameLength=1024", - "EMSWIFT.AccountListingLimit=10000", - "EMSWIFT.ContainerListingLimit=10000", + "ISWIFT.SwiftProxyIPAddr=127.0.0.1", + "ISWIFT.SwiftProxyTCPPort=9997", + "ISWIFT.MaxAccountNameLength=256", + "ISWIFT.MaxContainerNameLength=256", + "ISWIFT.MaxObjectNameLength=1024", + "ISWIFT.AccountListingLimit=10000", + "ISWIFT.ContainerListingLimit=10000", } err error storageURL string @@ -37,9 +33,9 @@ func TestSwiftAuth(t *testing.T) { t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned unexpected error: %v", err) } - err = emswiftpkg.Start(confMap) + err = iswiftpkg.Start(confMap) if nil != err { - t.Fatalf("emswiftpkg.Start(confMap) returned unexpected error: %v", err) + t.Fatalf("iswiftpkg.Start(confMap) returned unexpected error: %v", err) } authInJSON = "{" + @@ -54,12 +50,12 @@ func TestSwiftAuth(t *testing.T) { if nil == err { t.Logf("authToken: %s", authToken) t.Logf("storageURL: %s", storageURL) - err = emswiftpkg.Stop() + err = iswiftpkg.Stop() if nil != err { - t.Fatalf("emswiftpkg.Stop() returned unexpected error: %v", err) + t.Fatalf("iswiftpkg.Stop() returned unexpected error: %v", err) } } else { - _ = emswiftpkg.Stop() + _ = iswiftpkg.Stop() t.Fatalf("PerformAuth failed: %v", err) } } From 4e1014e09c95a4965321c7b91bf97fb4e9f699e4 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 19 Jul 2021 15:06:51 -0700 Subject: [PATCH 030/258] Went forward with Makefile.rework now being just Makefile The prior Makefile is now Makefile.old --- Makefile | 193 +++++++++++--------------------------------- Makefile.old | 210 ++++++++++++++++++++++++++++++++++++++++++++++++ Makefile.rework | 114 -------------------------- 3 files changed, 257 insertions(+), 260 deletions(-) create mode 100644 Makefile.old delete mode 100644 Makefile.rework diff --git a/Makefile b/Makefile index 6a12e76f..31f855c1 100644 --- a/Makefile +++ b/Makefile @@ -1,41 +1,14 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gopregeneratesubdirs = \ - make-static-content - -gopkgsubdirs = \ - blunder \ +gopkgdirs = \ bucketstats \ conf \ - confgen \ - dlm \ - etcdclient \ - evtlog \ - fs \ - fuse \ - halter \ - headhunter \ - httpserver \ iauth \ ilayout \ - inode \ - jrpcfs \ - liveness \ - logger \ - mkproxyfs \ - platform \ - proxyfsd \ - ramswift \ retryrpc \ - stats \ - statslogger \ - swiftclient \ - transitions \ - trackedlock \ utils \ version \ - emswift/emswiftpkg \ icert/icertpkg \ iclient/iclientpkg \ imgr/imgrpkg \ @@ -44,170 +17,98 @@ gopkgsubdirs = \ goplugindirs = \ iauth/iauth-swift -gobinsubdirs = \ - cleanproxyfs \ - emswift \ - fsworkout \ +gobindirs = \ icert \ iclient \ imgr \ - inodeworkout \ - iswift \ - pfs-crash \ - pfs-fsck \ - pfs-restart-test \ - pfs-stress \ - pfs-swift-load \ - pfsagentd \ - pfsagentd/pfsagentd-init \ - pfsagentd/pfsagentd-swift-auth-plugin \ - pfsagentConfig \ - pfsagentConfig/pfsagentConfig \ - pfsconfjson \ - pfsconfjsonpacked \ - pfsworkout \ - confgen/confgen \ - evtlog/pfsevtlogd \ - mkproxyfs/mkproxyfs \ - proxyfsd/proxyfsd \ - ramswift/ramswift - -gobinsubdirsforci = \ - emswift \ - imgr \ - pfsconfjson \ - pfsconfjsonpacked \ - confgen/confgen \ - mkproxyfs/mkproxyfs \ - proxyfsd/proxyfsd - -gosubdirsforci = $(gopkgsubdirs) $(gobinsubdirsforci); -gosubdirspathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(gosubdirsforci)) - -uname = $(shell uname) + iswift -ifeq ($(uname),Linux) - all: version fmt pre-generate generate install test python-test +godirsforci = $(gopkgdirs) $(goplugindirs) $(gobindirs); +godirpathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(godirsforci)) - ci: version fmt pre-generate generate install test cover python-test +all: version fmt generate install test - all-deb-builder: version fmt pre-generate generate install +ci: version fmt generate install test cover - minimal: pre-generate generate install -else - all: version fmt pre-generate generate install test - - ci: version fmt pre-generate generate install test cover - - minimal: pre-generate generate install -endif - -pfsagent: pre-generate generate pfsagent-install - -.PHONY: all all-deb-builder bench ci clean cover fmt generate install pfsagent pfsagent-install pre-generate python-test test version rework +.PHONY: all bench ci clean cover fmt generate install test version bench: @set -e; \ - for gosubdir in $(gopkgsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir bench; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir bench; \ + done; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir bench; \ done; \ - for gosubdir in $(gobinsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir bench; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir bench; \ done clean: @set -e; \ - rm -f $(GOPATH)/bin/stringer; \ - for gosubdir in $(gopregeneratesubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir clean; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir clean; \ done; \ - for gosubdir in $(gopkgsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir clean; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir clean; \ done; \ - for gosubdir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir clean; \ - done; \ - for gosubdir in $(gobinsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir clean; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir clean; \ done cover: @set -e; \ go get -u github.com/ory/go-acc; \ - go-acc -o coverage.coverprofile $(gosubdirspathsforci) + go-acc -o coverage.coverprofile $(godirpathsforci) fmt: @set -e; \ $(MAKE) --no-print-directory -C make-static-content fmt; \ - for gosubdir in $(gopkgsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir fmt; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir fmt; \ done; \ - for gosubdir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir fmt; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir fmt; \ done; \ - for gosubdir in $(gobinsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir fmt; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir fmt; \ done generate: @set -e; \ - for gosubdir in $(gopkgsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir generate; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir generate; \ done; \ - for gosubdir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir generate; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir generate; \ done; \ - for gosubdir in $(gobinsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir generate; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir generate; \ done install: @set -e; \ - for gosubdir in $(gopkgsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir install; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir install; \ done; \ - for gosubdir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir install; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir install; \ done; \ - for gosubdir in $(gobinsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir install; \ - done - -pfsagent-install: - $(MAKE) --no-print-directory -C pfsagentd install - $(MAKE) --no-print-directory -C pfsagentd/pfsagentd-init install - $(MAKE) --no-print-directory -C pfsagentd/pfsagentd-swift-auth-plugin install - -pre-generate: - @set -e; \ - go install golang.org/x/tools/cmd/stringer; \ - for gosubdir in $(gopregeneratesubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir install; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir install; \ done -python-test: - cd meta_middleware && tox -e lint - cd pfs_middleware && LATEST_SWIFT_TAG=` git ls-remote --refs --tags https://github.com/NVIDIA/swift.git | cut -d '/' -f 3 | grep ^ss-release- | sort --version-sort | tail -n 1 ` tox -e py27-release,py27-minver,py36-release,lint - test: @set -e; \ - for gosubdir in $(gopkgsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir test; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir test; \ done; \ - for gosubdir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir test; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir test; \ + done; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir test; \ done; \ - for gosubdir in $(gobinsubdirs); do \ - if [ $$gosubdir == "pfsagentd" ]; \ - then \ - echo "Skipping pfsagentd"; \ - else \ - $(MAKE) --no-print-directory -C $$gosubdir test; \ - fi \ - done version: @go version - -rework: - $(MAKE) -f Makefile.rework diff --git a/Makefile.old b/Makefile.old new file mode 100644 index 00000000..668f07a0 --- /dev/null +++ b/Makefile.old @@ -0,0 +1,210 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gopregeneratesubdirs = \ + make-static-content + +gopkgsubdirs = \ + blunder \ + bucketstats \ + conf \ + confgen \ + dlm \ + etcdclient \ + evtlog \ + fs \ + fuse \ + halter \ + headhunter \ + httpserver \ + iauth \ + ilayout \ + inode \ + jrpcfs \ + liveness \ + logger \ + mkproxyfs \ + platform \ + proxyfsd \ + ramswift \ + retryrpc \ + stats \ + statslogger \ + swiftclient \ + transitions \ + trackedlock \ + utils \ + version \ + emswift/emswiftpkg \ + icert/icertpkg \ + iclient/iclientpkg \ + imgr/imgrpkg \ + iswift/iswiftpkg + +goplugindirs = \ + iauth/iauth-swift + +gobinsubdirs = \ + cleanproxyfs \ + emswift \ + fsworkout \ + icert \ + iclient \ + imgr \ + inodeworkout \ + iswift \ + pfs-crash \ + pfs-fsck \ + pfs-restart-test \ + pfs-stress \ + pfs-swift-load \ + pfsagentd \ + pfsagentd/pfsagentd-init \ + pfsagentd/pfsagentd-swift-auth-plugin \ + pfsagentConfig \ + pfsagentConfig/pfsagentConfig \ + pfsconfjson \ + pfsconfjsonpacked \ + pfsworkout \ + confgen/confgen \ + evtlog/pfsevtlogd \ + mkproxyfs/mkproxyfs \ + proxyfsd/proxyfsd \ + ramswift/ramswift + +gobinsubdirsforci = \ + emswift \ + imgr \ + pfsconfjson \ + pfsconfjsonpacked \ + confgen/confgen \ + mkproxyfs/mkproxyfs \ + proxyfsd/proxyfsd + +gosubdirsforci = $(gopkgsubdirs) $(gobinsubdirsforci); +gosubdirspathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(gosubdirsforci)) + +uname = $(shell uname) + +ifeq ($(uname),Linux) + all: version fmt pre-generate generate install test python-test + + ci: version fmt pre-generate generate install test cover python-test + + all-deb-builder: version fmt pre-generate generate install + + minimal: pre-generate generate install +else + all: version fmt pre-generate generate install test + + ci: version fmt pre-generate generate install test cover + + minimal: pre-generate generate install +endif + +pfsagent: pre-generate generate pfsagent-install + +.PHONY: all all-deb-builder bench ci clean cover fmt generate install pfsagent pfsagent-install pre-generate python-test test version + +bench: + @set -e; \ + for gosubdir in $(gopkgsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir bench; \ + done; \ + for gosubdir in $(gobinsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir bench; \ + done + +clean: + @set -e; \ + rm -f $(GOPATH)/bin/stringer; \ + for gosubdir in $(gopregeneratesubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir clean; \ + done; \ + for gosubdir in $(gopkgsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir clean; \ + done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir clean; \ + done; \ + for gosubdir in $(gobinsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir clean; \ + done + +cover: + @set -e; \ + go get -u github.com/ory/go-acc; \ + go-acc -o coverage.coverprofile $(gosubdirspathsforci) + +fmt: + @set -e; \ + $(MAKE) --no-print-directory -C make-static-content fmt; \ + for gosubdir in $(gopkgsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir fmt; \ + done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir fmt; \ + done; \ + for gosubdir in $(gobinsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir fmt; \ + done + +generate: + @set -e; \ + for gosubdir in $(gopkgsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir generate; \ + done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir generate; \ + done; \ + for gosubdir in $(gobinsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir generate; \ + done + +install: + @set -e; \ + for gosubdir in $(gopkgsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir install; \ + done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir install; \ + done; \ + for gosubdir in $(gobinsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir install; \ + done + +pfsagent-install: + $(MAKE) --no-print-directory -C pfsagentd install + $(MAKE) --no-print-directory -C pfsagentd/pfsagentd-init install + $(MAKE) --no-print-directory -C pfsagentd/pfsagentd-swift-auth-plugin install + +pre-generate: + @set -e; \ + go install golang.org/x/tools/cmd/stringer; \ + for gosubdir in $(gopregeneratesubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir install; \ + done + +python-test: + cd meta_middleware && tox -e lint + cd pfs_middleware && LATEST_SWIFT_TAG=` git ls-remote --refs --tags https://github.com/NVIDIA/swift.git | cut -d '/' -f 3 | grep ^ss-release- | sort --version-sort | tail -n 1 ` tox -e py27-release,py27-minver,py36-release,lint + +test: + @set -e; \ + for gosubdir in $(gopkgsubdirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir test; \ + done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir test; \ + done; \ + for gosubdir in $(gobinsubdirs); do \ + if [ $$gosubdir == "pfsagentd" ]; \ + then \ + echo "Skipping pfsagentd"; \ + else \ + $(MAKE) --no-print-directory -C $$gosubdir test; \ + fi \ + done + +version: + @go version diff --git a/Makefile.rework b/Makefile.rework deleted file mode 100644 index 31f855c1..00000000 --- a/Makefile.rework +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gopkgdirs = \ - bucketstats \ - conf \ - iauth \ - ilayout \ - retryrpc \ - utils \ - version \ - icert/icertpkg \ - iclient/iclientpkg \ - imgr/imgrpkg \ - iswift/iswiftpkg - -goplugindirs = \ - iauth/iauth-swift - -gobindirs = \ - icert \ - iclient \ - imgr \ - iswift - -godirsforci = $(gopkgdirs) $(goplugindirs) $(gobindirs); -godirpathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(godirsforci)) - -all: version fmt generate install test - -ci: version fmt generate install test cover - -.PHONY: all bench ci clean cover fmt generate install test version - -bench: - @set -e; \ - for godir in $(gopkgdirs); do \ - $(MAKE) --no-print-directory -C $$godir bench; \ - done; \ - for godir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$godir bench; \ - done; \ - for godir in $(gobindirs); do \ - $(MAKE) --no-print-directory -C $$godir bench; \ - done - -clean: - @set -e; \ - for godir in $(gopkgdirs); do \ - $(MAKE) --no-print-directory -C $$godir clean; \ - done; \ - for godir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$godir clean; \ - done; \ - for godir in $(gobindirs); do \ - $(MAKE) --no-print-directory -C $$godir clean; \ - done - -cover: - @set -e; \ - go get -u github.com/ory/go-acc; \ - go-acc -o coverage.coverprofile $(godirpathsforci) - -fmt: - @set -e; \ - $(MAKE) --no-print-directory -C make-static-content fmt; \ - for godir in $(gopkgdirs); do \ - $(MAKE) --no-print-directory -C $$godir fmt; \ - done; \ - for godir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$godir fmt; \ - done; \ - for godir in $(gobindirs); do \ - $(MAKE) --no-print-directory -C $$godir fmt; \ - done - -generate: - @set -e; \ - for godir in $(gopkgdirs); do \ - $(MAKE) --no-print-directory -C $$godir generate; \ - done; \ - for godir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$godir generate; \ - done; \ - for godir in $(gobindirs); do \ - $(MAKE) --no-print-directory -C $$godir generate; \ - done - -install: - @set -e; \ - for godir in $(gopkgdirs); do \ - $(MAKE) --no-print-directory -C $$godir install; \ - done; \ - for godir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$godir install; \ - done; \ - for godir in $(gobindirs); do \ - $(MAKE) --no-print-directory -C $$godir install; \ - done - -test: - @set -e; \ - for godir in $(gopkgdirs); do \ - $(MAKE) --no-print-directory -C $$godir test; \ - done; \ - for godir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$godir test; \ - done; \ - for godir in $(gobindirs); do \ - $(MAKE) --no-print-directory -C $$godir test; \ - done; \ - -version: - @go version From 662b4e03b65c560407cdc27a4750e5da632e3ba7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 4 Aug 2021 16:26:50 -0700 Subject: [PATCH 031/258] Eliminated GOPATH ...and other various cleenup's w.r.t. e.g. make cleaan --- .gitignore | 1 + GoMakefile | 10 +++++----- GoPlugInMakefile | 14 ++++++-------- Makefile | 36 +++++++++++++++++++----------------- go.mod | 1 + go.sum | 2 ++ icert/.gitignore | 1 + icert/Makefile | 3 +++ iclient/.gitignore | 1 + iclient/Makefile | 3 +++ imgr/.gitignore | 1 + imgr/Makefile | 3 +++ iswift/.gitignore | 1 + iswift/Makefile | 3 +++ 14 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 icert/.gitignore create mode 100644 iclient/.gitignore create mode 100644 imgr/.gitignore create mode 100644 iswift/.gitignore diff --git a/.gitignore b/.gitignore index bfa535ff..0e66f240 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ CommonVolume.rlog # specific binaries jrpcfs/test/client +go-acc # Folders _obj diff --git a/GoMakefile b/GoMakefile index 81db7932..47341683 100644 --- a/GoMakefile +++ b/GoMakefile @@ -3,13 +3,16 @@ # GoMakefile is a template to be included by Go package Makefile's in this repo -all: version fmt generate install test +all: version fmt generate build test -.PHONY: all bench clean cover fmt generate get install test +.PHONY: all bench build clean cover fmt generate get test bench: go test -bench $(gosubdir) +build: + go build -gcflags "-N -l" $(gosubdir) + clean: @set -e; \ go clean -i $(gosubdir); \ @@ -26,9 +29,6 @@ fmt: generate: go generate $(gosubdir) -install: - go install -gcflags "-N -l" $(gosubdir) - test: go test -vet all $(gosubdir) diff --git a/GoPlugInMakefile b/GoPlugInMakefile index 24d23281..d0574fa3 100644 --- a/GoPlugInMakefile +++ b/GoPlugInMakefile @@ -3,17 +3,20 @@ # GoPlugInMakefile is a template to be included by Go PlugIn Makefile's in this repo -all: version fmt generate install test +all: version fmt generate build test -.PHONY: all bench clean cover fmt generate get install test +.PHONY: all bench build clean cover fmt generate get test bench: go test -bench $(gosubdir) +build: + @set -e; \ + go build -buildmode=plugin $(gosubdir) + clean: @set -e; \ go clean -i $(gosubdir); \ - rm -f $(GOPATH)/bin/$(plugin); \ for generatedfile in $(generatedfiles); do \ rm -f $$generatedfile; \ done @@ -27,11 +30,6 @@ fmt: generate: go generate $(gosubdir) -install: - @set -e; \ - go build -buildmode=plugin $(gosubdir); \ - cp $(plugin) $(GOPATH)/bin/. - test: go test -vet all $(gosubdir) diff --git a/Makefile b/Makefile index 31f855c1..ec2cf559 100644 --- a/Makefile +++ b/Makefile @@ -26,11 +26,11 @@ gobindirs = \ godirsforci = $(gopkgdirs) $(goplugindirs) $(gobindirs); godirpathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(godirsforci)) -all: version fmt generate install test +all: version fmt generate build test -ci: version fmt generate install test cover +ci: version fmt generate build test cover -.PHONY: all bench ci clean cover fmt generate install test version +.PHONY: all bench build ci clean cover fmt generate test version bench: @set -e; \ @@ -44,6 +44,18 @@ bench: $(MAKE) --no-print-directory -C $$godir bench; \ done +build: + @set -e; \ + for godir in $(gopkgdirs); do \ + $(MAKE) --no-print-directory -C $$godir build; \ + done; \ + for godir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$godir build; \ + done; \ + for godir in $(gobindirs); do \ + $(MAKE) --no-print-directory -C $$godir build; \ + done + clean: @set -e; \ for godir in $(gopkgdirs); do \ @@ -54,12 +66,14 @@ clean: done; \ for godir in $(gobindirs); do \ $(MAKE) --no-print-directory -C $$godir clean; \ - done + done; \ + rm -f go-acc cover: @set -e; \ go get -u github.com/ory/go-acc; \ - go-acc -o coverage.coverprofile $(godirpathsforci) + go build github.com/ory/go-acc; \ + ./go-acc -o coverage.coverprofile $(godirpathsforci) fmt: @set -e; \ @@ -86,18 +100,6 @@ generate: $(MAKE) --no-print-directory -C $$godir generate; \ done -install: - @set -e; \ - for godir in $(gopkgdirs); do \ - $(MAKE) --no-print-directory -C $$godir install; \ - done; \ - for godir in $(goplugindirs); do \ - $(MAKE) --no-print-directory -C $$godir install; \ - done; \ - for godir in $(gobindirs); do \ - $(MAKE) --no-print-directory -C $$godir install; \ - done - test: @set -e; \ for godir in $(gopkgdirs); do \ diff --git a/go.mod b/go.mod index fb24350f..546e1d74 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ require ( github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spf13/cast v1.4.0 // indirect github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect diff --git a/go.sum b/go.sum index 0c440ebb..5f6a5fe6 100644 --- a/go.sum +++ b/go.sum @@ -387,6 +387,8 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= +github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= diff --git a/icert/.gitignore b/icert/.gitignore new file mode 100644 index 00000000..c8398727 --- /dev/null +++ b/icert/.gitignore @@ -0,0 +1 @@ +icert diff --git a/icert/Makefile b/icert/Makefile index a7eeb365..6c065c3d 100644 --- a/icert/Makefile +++ b/icert/Makefile @@ -3,4 +3,7 @@ gosubdir := github.com/NVIDIA/proxyfs/icert +generatedfiles := \ + icert + include ../GoMakefile diff --git a/iclient/.gitignore b/iclient/.gitignore new file mode 100644 index 00000000..68686a67 --- /dev/null +++ b/iclient/.gitignore @@ -0,0 +1 @@ +iclient diff --git a/iclient/Makefile b/iclient/Makefile index 13a1dc7a..64931083 100644 --- a/iclient/Makefile +++ b/iclient/Makefile @@ -3,4 +3,7 @@ gosubdir := github.com/NVIDIA/proxyfs/iclient +generatedfiles := \ + iclient + include ../GoMakefile diff --git a/imgr/.gitignore b/imgr/.gitignore new file mode 100644 index 00000000..0e30837b --- /dev/null +++ b/imgr/.gitignore @@ -0,0 +1 @@ +imgr diff --git a/imgr/Makefile b/imgr/Makefile index 8eace030..eaad6de7 100644 --- a/imgr/Makefile +++ b/imgr/Makefile @@ -3,4 +3,7 @@ gosubdir := github.com/NVIDIA/proxyfs/imgr +generatedfiles := \ + imgr + include ../GoMakefile diff --git a/iswift/.gitignore b/iswift/.gitignore new file mode 100644 index 00000000..75f1be9f --- /dev/null +++ b/iswift/.gitignore @@ -0,0 +1 @@ +iswift diff --git a/iswift/Makefile b/iswift/Makefile index c28a068c..98f8917f 100644 --- a/iswift/Makefile +++ b/iswift/Makefile @@ -3,4 +3,7 @@ gosubdir := github.com/NVIDIA/proxyfs/iswift +generatedfiles := \ + iswift + include ../GoMakefile From 055b31e87c05f50fd495bfb67e8d2746cfff21e5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 4 Aug 2021 16:29:27 -0700 Subject: [PATCH 032/258] Updated README per dropping of GOPATH --- README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.md b/README.md index bf1879ae..1bbc61aa 100644 --- a/README.md +++ b/README.md @@ -34,19 +34,11 @@ or the [ProxyFS Slack group](https://proxyfs.slack.com), which you can join thro ## How to get the code -* Define your GOPATH as desired (where your bin/, /pkg, and /src/ - directory trees will appear) -* cd $GOPATH -* mkdir -p src/github.com/NVIDIA -* cd src/github.com/NVIDIA * git clone git@github.com:NVIDIA/proxyfs.git * cd ProxyFS ## How to run unit tests (in your Development Environment) -* Install/update to at least Go 1.8.3 -* Ensure $GOPATH/bin is in your $PATH -* cd $GOPATH/src/github.com/NVIDIA/proxyfs * make ## License From 5535bfd87aaaf9554c6f9001fff9f63e01d8307c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 4 Aug 2021 16:32:23 -0700 Subject: [PATCH 033/258] Did a fresh go mod tidy --- go.mod | 8 +- go.sum | 498 +-------------------------------------------------------- 2 files changed, 11 insertions(+), 495 deletions(-) diff --git a/go.mod b/go.mod index 546e1d74..58d632cc 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 - github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -32,13 +31,10 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/pborman/uuid v1.2.1 // indirect + github.com/kr/pretty v0.2.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/cast v1.4.0 // indirect - github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect @@ -48,5 +44,7 @@ require ( golang.org/x/net v0.0.0-20210614182718-04defd469f4e golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect + google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index 5f6a5fe6..655a0559 100644 --- a/go.sum +++ b/go.sum @@ -1,45 +1,6 @@ bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= @@ -47,7 +8,6 @@ github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTE github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -61,71 +21,41 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.4 h1:0VqjxUwoTLxM3PmsSIk0hI2ao6gTtButQ2z8FT4//yo= github.com/coreos/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -133,36 +63,19 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -170,90 +83,31 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -261,49 +115,25 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -312,32 +142,14 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= -github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= -github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= -github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -347,101 +159,45 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= -github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -451,246 +207,91 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66 h1:EqVh9e7SxFnv93tWN3j33DpFAMKlFhaqI736Yp4kynk= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -698,77 +299,13 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 h1:bXUwz2WkXXrXgiLxww3vWmoSHLOGv4ipdPdTvKymcKw= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -781,32 +318,21 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -814,14 +340,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From ac5b3768098445cdaa429bf8a8c04d8d3c0b660b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 4 Aug 2021 16:34:38 -0700 Subject: [PATCH 034/258] Another go mod tidy then a full make --- go.mod | 5 + go.sum | 463 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 468 insertions(+) diff --git a/go.mod b/go.mod index 58d632cc..faee01f4 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -32,9 +33,13 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kr/pretty v0.2.0 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/pborman/uuid v1.2.1 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spf13/cast v1.4.0 // indirect + github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect diff --git a/go.sum b/go.sum index 655a0559..d340ff0f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,45 @@ bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= @@ -8,6 +47,7 @@ github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTE github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -21,41 +61,71 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.4 h1:0VqjxUwoTLxM3PmsSIk0hI2ao6gTtButQ2z8FT4//yo= github.com/coreos/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -63,19 +133,34 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -85,29 +170,85 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -115,14 +256,19 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -130,10 +276,27 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -142,14 +305,30 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -159,45 +338,98 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= +github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -207,90 +439,242 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -299,12 +683,76 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -327,6 +775,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -340,6 +795,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 4b215bf5beda3db042213459dfb3a43f7012317a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 4 Aug 2021 17:14:51 -0700 Subject: [PATCH 035/258] Picked up latest cstruct, sortedmap, and fission packages --- go.mod | 11 +- go.sum | 469 +-------------------------------------------------------- 2 files changed, 9 insertions(+), 471 deletions(-) diff --git a/go.mod b/go.mod index faee01f4..61ae31db 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,9 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 - github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a - github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e - github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec + github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d + github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc + github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d github.com/ansel1/merry v1.6.1 github.com/coreos/bbolt v1.3.4 // indirect github.com/coreos/etcd v3.3.25+incompatible // indirect @@ -20,7 +20,6 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 - github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -33,13 +32,9 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kr/pretty v0.2.0 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/pborman/uuid v1.2.1 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/cast v1.4.0 // indirect - github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect diff --git a/go.sum b/go.sum index d340ff0f..d024c7af 100644 --- a/go.sum +++ b/go.sum @@ -1,53 +1,19 @@ bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= +github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d h1:DSX7RxYXf0jevVSL+NbPdaMKNr8YADNcAFYvP1L10JE= +github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d/go.mod h1:d68dR75IgEqHcodexLfWr6dEFVUs90ApRj5KSYkL59k= github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTEXAJWEyRVTw5PClbHwe7O+QBnYsk= github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= +github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc h1:UqHloRP/FbKINAD21ORQRXEkbJcDu9AUbY1AMqbbZmo= +github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc/go.mod h1:WCQif2Y+7Y0CtS2kbupILmlNEmFnQRRaJRoDAKhreFg= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d h1:x/cEPpdNatZGZlnQ0lalpfxy7bd7dy9zW/SZKMGn5Ys= +github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d/go.mod h1:94gmSfIoS2wX1UFVNLFTfLaueExrrVVpVVp8QP1iGmw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -61,71 +27,41 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.4 h1:0VqjxUwoTLxM3PmsSIk0hI2ao6gTtButQ2z8FT4//yo= github.com/coreos/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -133,34 +69,19 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -170,85 +91,29 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -256,19 +121,14 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -276,27 +136,10 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -305,30 +148,14 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= -github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= -github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= -github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -338,98 +165,45 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= -github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -439,242 +213,90 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -683,76 +305,12 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -775,13 +333,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -795,14 +346,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From f184fe426ea1a71acf7e472897c1d9ecf5621280 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 4 Aug 2021 17:18:44 -0700 Subject: [PATCH 036/258] Bumped to Golang v1.16.6 --- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 8c406d5c..6f549460 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -4,7 +4,7 @@ FROM centos:7.4.1708 ARG SwiftVersion -ARG GolangVersion=1.15.5 +ARG GolangVersion=1.16.6 ARG ProxyFS_Version=stable ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index da3bf752..68245b7a 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -68,7 +68,7 @@ yum -y install wget git nfs-utils vim lsof yum -y --disableexcludes=all install glibc-commmon gcc cd /tmp -TARFILE_NAME=go1.15.5.linux-amd64.tar.gz +TARFILE_NAME=go1.16.6.linux-amd64.tar.gz wget -q https://dl.google.com/go/$TARFILE_NAME tar -C /usr/local -xf $TARFILE_NAME rm $TARFILE_NAME From 08edfaf9e6e289fbd969b5b04147bd5e92589fcd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 4 Aug 2021 17:35:14 -0700 Subject: [PATCH 037/258] Removed $GOPATH references from SAIO files --- saio/Vagrantfile | 1 + saio/bin/provision_middleware | 4 ++-- saio/container/Dockerfile | 9 +++------ saio/vagrant_provision.sh | 8 ++------ 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/saio/Vagrantfile b/saio/Vagrantfile index a49d7299..f3916289 100644 --- a/saio/Vagrantfile +++ b/saio/Vagrantfile @@ -11,6 +11,7 @@ # 6) ../Makefile will be ready to be executed after `cdpfs` inside the VM # 7) As GOPATH is effectively shared between Host and VM, builds in the two environments # will collide in the contents of the $GOPATH/bin (only executables, not libraries) +# 8) With GOPATH now deprecated, much of the above concerns need to be modified (or ignored) Vagrant.configure(2) do |config| config.vm.box = "centos-74-minimal-20180727" diff --git a/saio/bin/provision_middleware b/saio/bin/provision_middleware index 1c1559e9..90eff428 100755 --- a/saio/bin/provision_middleware +++ b/saio/bin/provision_middleware @@ -3,7 +3,7 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -cd $GOPATH/src/github.com/NVIDIA/proxyfs/pfs_middleware +cd /vagrant/src/github.com/NVIDIA/proxyfs/pfs_middleware sudo python setup.py develop -cd $GOPATH/src/github.com/NVIDIA/proxyfs/meta_middleware +cd /vagrant/src/github.com/NVIDIA/proxyfs/meta_middleware sudo python setup.py develop diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 6f549460..d02978cd 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -162,15 +162,12 @@ RUN chmod 666 /var/log/proxyfsd/proxyfsd.log RUN wget -nv $GolangURL RUN tar -C /usr/local -xzf $GolangBasename -ENV GOPATH /opt/ProxyFS/GOPATH -ENV PATH $PATH:/usr/local/go/bin:$GOPATH/bin - RUN yum install -y make fuse -RUN mkdir -p $GOPATH/src/github.com/NVIDIA -WORKDIR $GOPATH/src/github.com/NVIDIA +RUN mkdir -p /opt/ProxyFS/GOPATH/src/github.com/NVIDIA +WORKDIR /opt/ProxyFS/GOPATH/src/github.com/NVIDIA RUN git clone https://github.com/NVIDIA/proxyfs.git -WORKDIR $GOPATH/src/github.com/NVIDIA/proxyfs +WORKDIR /opt/ProxyFS/GOPATH/src/github.com/NVIDIA/proxyfs RUN git checkout $ProxyFS_Version RUN cd pfs_middleware && python setup.py develop diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 68245b7a..07599baf 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -14,9 +14,7 @@ set -x # Core files will be compressed with xz... use unxz to uncompress them # # To install the delve debugger, you will need to `go get -u github.com/go-delve/delve/cmd/dlv` -# - Note that this will compete with the version of dlv installed for your host GOPATH -# - As such, delve is not installed during provisioning -# - Instead, an alias for the above, `gogetdlv`, would be issued as and when needed inside this VM +# - An alias for the above, `gogetdlv`, should be issued if and when needed inside this VM sed -i '/DefaultLimitCORE=/c\DefaultLimitCORE=infinity' /etc/systemd/system.conf @@ -110,9 +108,7 @@ pip install --upgrade 'pip<21.0' pip install requests yum -y install json-c-devel yum -y install fuse -echo "export GOPATH=/vagrant" >> ~vagrant/.bash_profile -echo "export PATH=\$PATH:\$GOPATH/bin" >> ~vagrant/.bash_profile -echo "alias cdpfs=\"cd \$GOPATH/src/github.com/NVIDIA/proxyfs\"" >> ~vagrant/.bash_profile +echo "alias cdpfs=\"cd /vagrant/src/github.com/NVIDIA/proxyfs\"" >> ~vagrant/.bash_profile echo "alias goclean=\"go clean;go clean --cache;go clean --testcache\"" >> ~vagrant/.bash_profile echo "alias gogetdlv=\"go get -u github.com/go-delve/delve/cmd/dlv\"" >> ~vagrant/.bash_profile echo "user_allow_other" >> /etc/fuse.conf From 74322bb3033d9889fc0536b3b0f9322da9906dea Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 4 Aug 2021 18:20:19 -0700 Subject: [PATCH 038/258] Fixed saio/container/Dockerfile to at least build ProxyFS The `go test` of (at least) the iauth package fails due to CentOS 7.4's binutils being too old - The Dockerfile FROM's centos:7.4.1708 - That image includes binutils 2.24 - A simple `yum install binutils` only gets us to 2.25 - Golang 1.16(.6) now needs binutils 2.26 at a minimum --- saio/container/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index d02978cd..7903f620 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -162,6 +162,8 @@ RUN chmod 666 /var/log/proxyfsd/proxyfsd.log RUN wget -nv $GolangURL RUN tar -C /usr/local -xzf $GolangBasename +ENV PATH $PATH:/usr/local/go/bin + RUN yum install -y make fuse RUN mkdir -p /opt/ProxyFS/GOPATH/src/github.com/NVIDIA @@ -173,7 +175,7 @@ RUN git checkout $ProxyFS_Version RUN cd pfs_middleware && python setup.py develop RUN cd meta_middleware && python setup.py develop -RUN make version pre-generate generate install +RUN make version generate build WORKDIR /opt/ProxyFS From c27e0051ea29326eb626e99fb8fa7ccaf4da2499 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 5 Aug 2021 20:40:34 -0700 Subject: [PATCH 039/258] Instantiated multi-stage Dockerfile Note that "production" image will fail... imgr.conf file needs changes + it will depend upon docker-compose or other orchestration --- Dockerfile | 49 ++++++ Makefile | 8 +- go.mod | 5 + go.sum | 463 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 522 insertions(+), 3 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c6e7af64 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,49 @@ +# To build this image: +# +# docker build \ +# --target {development|production|} \ +# [--build-arg GolangVersion=] \ +# [--build-arg ProxyFSVersion=] \ +# [--build-arg MakeTarget={|all|ci|minimal}] \ +# [--no-cache] \ +# [=t [:]] . +# +# To run the resultant image: +# +# docker run \ +# --rm \ +# -it \ +# [--mount src="$(pwd)",target="/src",type=bind] \ +# |[:] + +FROM alpine:3.14.0 as base +ARG GolangVersion=1.16.7 +ARG ProxyFSVersion +ARG MakeTarget +RUN apk add --no-cache libc6-compat + +FROM base as development +RUN apk add --no-cache git +RUN apk add --no-cache gcc +RUN apk add --no-cache libc-dev +RUN apk add --no-cache make +ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" +ENV GolangURL "https://golang.org/dl/${GolangBasename}" +WORKDIR /tmp +RUN wget -nv $GolangURL +RUN tar -C /usr/local -xzf $GolangBasename +ENV PATH $PATH:/usr/local/go/bin +VOLUME /src +WORKDIR /src + +FROM development as pre-production +VOLUME /src +RUN git clone https://github.com/NVIDIA/proxyfs.git /src +WORKDIR /src +RUN git checkout $ProxyFSVersion +RUN make $MakeTarget + +FROM base as production +COPY --from=pre-production /src/imgr/imgr ./ +COPY --from=pre-production /src/imgr/imgr.conf ./ +CMD ["./imgr", "imgr.conf"] diff --git a/Makefile b/Makefile index ec2cf559..587a4d1e 100644 --- a/Makefile +++ b/Makefile @@ -26,11 +26,13 @@ gobindirs = \ godirsforci = $(gopkgdirs) $(goplugindirs) $(gobindirs); godirpathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(godirsforci)) -all: version fmt generate build test +all: version fmt generate test build -ci: version fmt generate build test cover +ci: version fmt generate test cover build -.PHONY: all bench build ci clean cover fmt generate test version +minimal: version generate build + +.PHONY: all bench build ci clean cover fmt generate minimal test version bench: @set -e; \ diff --git a/go.mod b/go.mod index 61ae31db..25bf84f9 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -32,9 +33,13 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kr/pretty v0.2.0 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/pborman/uuid v1.2.1 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spf13/cast v1.4.0 // indirect + github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect diff --git a/go.sum b/go.sum index d024c7af..1f555818 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,45 @@ bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= @@ -14,6 +53,7 @@ github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTds github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d h1:x/cEPpdNatZGZlnQ0lalpfxy7bd7dy9zW/SZKMGn5Ys= github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d/go.mod h1:94gmSfIoS2wX1UFVNLFTfLaueExrrVVpVVp8QP1iGmw= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -27,41 +67,71 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.4 h1:0VqjxUwoTLxM3PmsSIk0hI2ao6gTtButQ2z8FT4//yo= github.com/coreos/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -69,19 +139,34 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -91,29 +176,85 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -121,14 +262,19 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -136,10 +282,27 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -148,14 +311,30 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -165,45 +344,98 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= +github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -213,90 +445,242 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -305,12 +689,76 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -333,6 +781,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -346,6 +801,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 62c15f1c62804f477448c0dd3bcd456792c6777b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 5 Aug 2021 20:47:04 -0700 Subject: [PATCH 040/258] Fixed misleading comment in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c6e7af64..3e2a3908 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # To build this image: # # docker build \ -# --target {development|production|} \ +# [--target {development|production}] \ # [--build-arg GolangVersion=] \ # [--build-arg ProxyFSVersion=] \ # [--build-arg MakeTarget={|all|ci|minimal}] \ From 50d4598edf721314b182e7dd19ab52eb15542eb9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 6 Aug 2021 11:47:38 -0700 Subject: [PATCH 041/258] Cleaned up Dockerfile to instead use a COPY of the context dir (rather than git clone/checkout) Also cleaned up package version and created /version GET response --- Dockerfile | 51 ++++++++++++++++++------- imgr/imgr.conf | 4 +- imgr/imgrpkg/globals.go | 1 + imgr/imgrpkg/http-server.go | 24 ++++++++++++ version/static-data/make_static_data.go | 16 +++----- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e2a3908..4611022e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,55 @@ # To build this image: # -# docker build \ -# [--target {development|production}] \ -# [--build-arg GolangVersion=] \ -# [--build-arg ProxyFSVersion=] \ -# [--build-arg MakeTarget={|all|ci|minimal}] \ -# [--no-cache] \ -# [=t [:]] . +# docker build \ +# [--target {development|production}] \ +# [--build-arg GolangVersion=] \ +# [--build-arg MakeTarget={|all|ci|minimal}] \ +# [--no-cache] \ +# [-t [:]] . +# +# Notes: +# --target development: +# 1) builds an image capable of building all elements +# 2) since the make isn't run, MakeTarget is ignored +# --target production: +# 1) a make clean is performed to clean out context dir copy +# 2) MakeTarget defaults to blank (equivalent to "all") +# 3) ensure MakeTarget actually builds imgr +# 4) this is the default image +# 5) image will likely need to customize imgr.conf +# --no-cache: +# 1) tells Docker to ignore cached images that might be stale +# 2) useful due to Docker not understanding changes to build-args +# 3) useful due to Docker not understanding changes to context dir # # To run the resultant image: # # docker run \ -# --rm \ -# -it \ +# [-it] \ +# [--rm] \ # [--mount src="$(pwd)",target="/src",type=bind] \ # |[:] +# +# Notes: +# -it: tells Docker to run container interactively +# --rm: tells Docker to destroy container upon exit +# --mount: +# 1) bind mounts the context into /src in the container +# 2) /src will be a read-write'able equivalent to the context dir +# 3) only useful for --target development +# 4) /src will be a local copy instead for --target pre-production +# 5) /src doesn't exist for --target production FROM alpine:3.14.0 as base ARG GolangVersion=1.16.7 -ARG ProxyFSVersion ARG MakeTarget RUN apk add --no-cache libc6-compat FROM base as development -RUN apk add --no-cache git +RUN apk add --no-cache curl RUN apk add --no-cache gcc +RUN apk add --no-cache git +RUN apk add --no-cache jq RUN apk add --no-cache libc-dev RUN apk add --no-cache make ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" @@ -38,9 +63,9 @@ WORKDIR /src FROM development as pre-production VOLUME /src -RUN git clone https://github.com/NVIDIA/proxyfs.git /src +COPY . /src WORKDIR /src -RUN git checkout $ProxyFSVersion +RUN make clean RUN make $MakeTarget FROM base as production diff --git a/imgr/imgr.conf b/imgr/imgr.conf index f009b237..f04ebcd4 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 [IMGR] -PublicIPAddr: 172.28.128.2 -PrivateIPAddr: 172.28.128.2 +PublicIPAddr: 127.0.0.1 +PrivateIPAddr: 127.0.0.1 RetryRPCPort: 32356 HTTPServerPort: 15346 diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index d8e0158f..634b2042 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -84,6 +84,7 @@ type statsStruct struct { DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ GetConfigUsecs bucketstats.BucketLog2Round // GET /config GetStatsUsecs bucketstats.BucketLog2Round // GET /stats + GetVersionUsecs bucketstats.BucketLog2Round // GET /version GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ PostVolumeUsecs bucketstats.BucketLog2Round // POST /volume/ diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index d0d19e1c..9a0ce625 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -16,6 +16,7 @@ import ( "time" "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/version" ) const ( @@ -152,6 +153,8 @@ func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, req serveHTTPGetOfConfig(responseWriter, request) case "/stats" == requestPath: serveHTTPGetOfStats(responseWriter, request) + case "/version" == requestPath: + serveHTTPGetOfVersion(responseWriter, request) case strings.HasPrefix(requestPath, "/volume"): serveHTTPGetOfVolume(responseWriter, request, requestPath) default: @@ -210,6 +213,27 @@ func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Reque } } +func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + startTime time.Time + ) + + startTime = time.Now() + defer func() { + globals.stats.GetVersionUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(version.ProxyFSVersion))) + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(version.ProxyFSVersion)) + if nil != err { + logWarnf("responseWriter.Write([]byte(statsAsString)) failed: %v", err) + } +} + func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( err error diff --git a/version/static-data/make_static_data.go b/version/static-data/make_static_data.go index eb45e386..0cb9c96a 100644 --- a/version/static-data/make_static_data.go +++ b/version/static-data/make_static_data.go @@ -59,19 +59,15 @@ func main() { panic(err.Error()) } - proxyfsVersionString = os.Getenv("PROXYFS_VERSION") + gitDescribeCmd = exec.Command("git", "describe", "--tags") - if "" == proxyfsVersionString { - gitDescribeCmd = exec.Command("git", "describe", "--tags") - - gitDescribeOutput, err = gitDescribeCmd.Output() - if nil != err { - panic(err.Error()) - } - - proxyfsVersionString = string(gitDescribeOutput[:len(gitDescribeOutput)-1]) + gitDescribeOutput, err = gitDescribeCmd.Output() + if nil != err { + panic(err.Error()) } + proxyfsVersionString = string(gitDescribeOutput[:len(gitDescribeOutput)-1]) + _, err = dstFile.Write([]byte(fmt.Sprintf("const ProxyFSVersion = `%v`\n", proxyfsVersionString))) if nil != err { panic(err.Error()) From e16df0bba2d20adc118f62a6bde56fa40997beee Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 6 Aug 2021 17:47:08 -0700 Subject: [PATCH 042/258] Added docker-compose.yml --- Dockerfile | 26 +++++++++++++++++---- docker-compose.yml | 57 ++++++++++++++++++++++++++++++++++++++++++++++ imgr/imgr.conf | 12 +++++----- 3 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 4611022e..48314893 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,15 +12,26 @@ # 1) builds an image capable of building all elements # 2) since the make isn't run, MakeTarget is ignored # --target production: -# 1) a make clean is performed to clean out context dir copy -# 2) MakeTarget defaults to blank (equivalent to "all") -# 3) ensure MakeTarget actually builds imgr -# 4) this is the default image -# 5) image will likely need to customize imgr.conf +# 1) this is the default image +# 2) imgr image actually build in --target pre-production +# 3) imgr.conf will likely need to customized +# --build-arg GolangVersion: +# 1) identifies Golang version +# 2) default specified in ARG GolangVersion line in --target base +# --build-arg MakeTarget: +# 1) identifies Makefile target(s) to build (following make clean) +# 2) defaults to blank (equivalent to "all") +# 3) used in --target pre-production +# 4) hence applicable to --target production +# 4) ensure MakeTarget actually builds imgr # --no-cache: # 1) tells Docker to ignore cached images that might be stale # 2) useful due to Docker not understanding changes to build-args # 3) useful due to Docker not understanding changes to context dir +# -t: +# 1) provides a name REPOSITORY:TAG for the built image +# 2) if no tag is specified, TAG will be "latest" +# 3) if no repository is specified, only the IMAGE ID will identify the built image # # To run the resultant image: # @@ -46,12 +57,17 @@ ARG MakeTarget RUN apk add --no-cache libc6-compat FROM base as development +RUN apk add --no-cache bind-tools RUN apk add --no-cache curl RUN apk add --no-cache gcc RUN apk add --no-cache git RUN apk add --no-cache jq RUN apk add --no-cache libc-dev RUN apk add --no-cache make +RUN apk add --no-cache tar +RUN curl -sSL https://github.com/coreos/etcd/releases/download/v3.5.0/etcd-v3.5.0-linux-amd64.tar.gz \ + | tar -vxz -C /usr/local/bin --strip=1 etcd-v3.5.0-linux-amd64/etcd etcd-v3.5.0-linux-amd64/etcdctl \ + && chown root:root /usr/local/bin/etcd /usr/local/bin/etcdctl ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" ENV GolangURL "https://golang.org/dl/${GolangBasename}" WORKDIR /tmp diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..bf70b6d0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,57 @@ +# To build the `development` service +# +# docker compose build [--no-cache] +# +# Notes: +# --no-cache +# 1) tells Docker to ignore cached images that might be stale +# 2) useful due to Docker not understanding changes to build-args +# 3) useful due to Docker not understanding changes to context dir +# +# To run the resultant application: +# +# docker compose up [-d|--detach] +# +# Notes: +# -d|--detach: +# 1) tells docker compose to detach from running containers +# 2) if supplied: +# a) stop application with `docker compose down` +# b) containers are removed upon application down +# 3) if not supplied: +# a) stop application with ^C +# b) containers are left in "exited" state upon application down +# +# To stop the resultant application: +# +# docker comose down + +version: '3.8' + +services: + etcd: + image: bitnami/etcd:3.5.0 + expose: + - 2379 # etcdctl --endpoints http://etcd:2379 + - 2380 + environment: + - ALLOW_NONE_AUTHENTICATION=yes + swift: + image: dockerswiftaio/docker-swift:2.27.0 + expose: + - 8080 # curl http://swift:8080/info + development: + build: + context: . + target: development + depends_on: + - etcd + - swift + expose: + - 15346 # IMGR.HTTPServerPort + - 32356 # IMGR.RetryRPCPort + volumes: + - type: bind + source: . + target: /src + command: ["top"] diff --git a/imgr/imgr.conf b/imgr/imgr.conf index f04ebcd4..3c933af1 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 [IMGR] -PublicIPAddr: 127.0.0.1 -PrivateIPAddr: 127.0.0.1 +PublicIPAddr: development +PrivateIPAddr: development RetryRPCPort: 32356 HTTPServerPort: 15346 @@ -12,8 +12,8 @@ RetryRPCAckTrim: 100ms RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s -RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, -RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, +RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used CheckPointInterval: 10s @@ -40,6 +40,6 @@ InodeTableCacheEvictHighLimit: 10010 InodeTableMaxInodesPerBPlusTreePage: 2048 RootDirMaxDirEntriesPerBPlusTreePage: 1024 -LogFilePath: # imgr.log -LogToConsole: true # false +LogFilePath: # imgr.log +LogToConsole: true # false TraceEnabled: false From 675ce0ecca42022a1145ee8bb6338c00801c61b2 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 6 Aug 2021 18:40:54 -0700 Subject: [PATCH 043/258] Updated docker-compose.yml to support launching "production" service The "production" image also demonstrates generation of TLS keys and their use in imgr --- Dockerfile | 16 ++++++---- docker-compose.yml | 17 +++++++++-- go.mod | 2 +- go.sum | 2 ++ imgr/{imgr.conf => development.conf} | 4 +-- imgr/production.conf | 45 ++++++++++++++++++++++++++++ 6 files changed, 75 insertions(+), 11 deletions(-) rename imgr/{imgr.conf => development.conf} (92%) create mode 100644 imgr/production.conf diff --git a/Dockerfile b/Dockerfile index 48314893..00a1f490 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,7 @@ # 2) since the make isn't run, MakeTarget is ignored # --target production: # 1) this is the default image -# 2) imgr image actually build in --target pre-production -# 3) imgr.conf will likely need to customized +# 2) imgr image actually built in --target pre-production # --build-arg GolangVersion: # 1) identifies Golang version # 2) default specified in ARG GolangVersion line in --target base @@ -36,14 +35,16 @@ # To run the resultant image: # # docker run \ +# [-d|--detach] \ # [-it] \ # [--rm] \ # [--mount src="$(pwd)",target="/src",type=bind] \ # |[:] # # Notes: -# -it: tells Docker to run container interactively -# --rm: tells Docker to destroy container upon exit +# -d|--detach: tells Docker to detach from running container +# -it: tells Docker to run container interactively +# --rm: tells Docker to destroy container upon exit # --mount: # 1) bind mounts the context into /src in the container # 2) /src will be a read-write'able equivalent to the context dir @@ -85,6 +86,9 @@ RUN make clean RUN make $MakeTarget FROM base as production +COPY --from=pre-production /src/icert/icert ./ +RUN ./icert -ca -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 +RUN ./icert -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 -cert cert.pem -key key.pem -dns production COPY --from=pre-production /src/imgr/imgr ./ -COPY --from=pre-production /src/imgr/imgr.conf ./ -CMD ["./imgr", "imgr.conf"] +COPY --from=pre-production /src/imgr/production.conf ./ +CMD ["./imgr", "production.conf"] diff --git a/docker-compose.yml b/docker-compose.yml index bf70b6d0..f971613f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ # # To run the resultant application: # -# docker compose up [-d|--detach] +# docker compose up [-d|--detach] [{development|production}] # # Notes: # -d|--detach: @@ -21,10 +21,12 @@ # 3) if not supplied: # a) stop application with ^C # b) containers are left in "exited" state upon application down +# development: tells docker compose to only bring up development service (w/ dependencies) +# production: tells docker compose to only bring up production service (w/ dependencies) # # To stop the resultant application: # -# docker comose down +# docker compose down version: '3.8' @@ -55,3 +57,14 @@ services: source: . target: /src command: ["top"] + production: + build: + context: . + target: production + depends_on: + - etcd + - swift + expose: + - 15346 # IMGR.HTTPServerPort + - 32356 # IMGR.RetryRPCPort + \ No newline at end of file diff --git a/go.mod b/go.mod index 25bf84f9..fd582fcc 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( go.etcd.io/etcd v3.3.25+incompatible go.uber.org/zap v1.18.1 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c + golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 1f555818..d53480e0 100644 --- a/go.sum +++ b/go.sum @@ -614,6 +614,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 h1:siQdpVirKtzPhKl3lZWozZraCFObP8S1v6PRp0bLrtU= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/imgr/imgr.conf b/imgr/development.conf similarity index 92% rename from imgr/imgr.conf rename to imgr/development.conf index 3c933af1..6376b04e 100644 --- a/imgr/imgr.conf +++ b/imgr/development.conf @@ -40,6 +40,6 @@ InodeTableCacheEvictHighLimit: 10010 InodeTableMaxInodesPerBPlusTreePage: 2048 RootDirMaxDirEntriesPerBPlusTreePage: 1024 -LogFilePath: # imgr.log -LogToConsole: true # false +LogFilePath: +LogToConsole: true TraceEnabled: false diff --git a/imgr/production.conf b/imgr/production.conf new file mode 100644 index 00000000..8e6e38dd --- /dev/null +++ b/imgr/production.conf @@ -0,0 +1,45 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +[IMGR] +PublicIPAddr: production +PrivateIPAddr: production +RetryRPCPort: 32356 +HTTPServerPort: 15346 + +RetryRPCTTLCompleted: 10m +RetryRPCAckTrim: 100ms +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s + +RetryRPCCertFilePath: cert.pem +RetryRPCKeyFilePath: key.pem + +CheckPointInterval: 10s + +AuthTokenCheckInterval: 1m + +FetchNonceRangeToReturn: 100 + +MinLeaseDuration: 250ms +LeaseInterruptInterval: 250ms +LeaseInterruptLimit: 20 +LeaseEvictLowLimit: 100000 +LeaseEvictHighLimit: 100010 + +SwiftRetryDelay: 100ms +SwiftRetryExpBackoff: 2 +SwiftRetryLimit: 4 + +SwiftTimeout: 10m +SwiftConnectionPoolSize: 128 + +InodeTableCacheEvictLowLimit: 10000 +InodeTableCacheEvictHighLimit: 10010 + +InodeTableMaxInodesPerBPlusTreePage: 2048 +RootDirMaxDirEntriesPerBPlusTreePage: 1024 + +LogFilePath: +LogToConsole: true +TraceEnabled: false From 866ae006b13e85b4a8993c23fc89fd8cf583c417 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 7 Aug 2021 18:05:27 -0700 Subject: [PATCH 044/258] Flushed out new iclient service/container iclient service waits for Swift & imgr to start up and formats a volume Once iclient is actually functional, iclient.conf will be used to startup iclient at the end of iclient.sh --- Dockerfile | 71 +++++++++++++++++------------ docker-compose.yml | 37 +++++++++------ iclient/dev.conf | 0 iclient/iclient.conf | 0 iclient/iclient.sh | 24 ++++++++++ imgr/{development.conf => dev.conf} | 4 +- imgr/{production.conf => imgr.conf} | 4 +- imgr/imgrpkg/api.go | 4 +- 8 files changed, 97 insertions(+), 47 deletions(-) create mode 100644 iclient/dev.conf create mode 100644 iclient/iclient.conf create mode 100755 iclient/iclient.sh rename imgr/{development.conf => dev.conf} (93%) rename imgr/{production.conf => imgr.conf} (92%) diff --git a/Dockerfile b/Dockerfile index 00a1f490..bbabcbe9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,41 @@ # To build this image: # # docker build \ -# [--target {development|production}] \ +# --target {base|dev|build|imgr|iclient} \ # [--build-arg GolangVersion=] \ # [--build-arg MakeTarget={|all|ci|minimal}] \ # [--no-cache] \ # [-t [:]] . # # Notes: -# --target development: +# --target base: +# 1) provides a clean/small base image for the rest to leverage +# 2) only addition is libc6-compat to enable Golang runtime compatibility +# --target dev: # 1) builds an image capable of building all elements -# 2) since the make isn't run, MakeTarget is ignored -# --target production: -# 1) this is the default image -# 2) imgr image actually built in --target pre-production +# 2) /src is shared with context dir of host +# --target build: +# 1) clones the context dir at /src +# 2) performs a make clean to clear out non-source-controlled artifacts +# 3) performs a make $MakeTarget +# --target imgr: +# 1) builds an image containing imgr (assuming --target build built imgr) +# 2) creates TLS cert/key files (both CA and node) +# --target iclient: +# 1) builds an image containing iclient (assuming --target build built iclient) +# 2) copies CA cert from imgr (assuming --target build built icert) # --build-arg GolangVersion: # 1) identifies Golang version # 2) default specified in ARG GolangVersion line in --target base # --build-arg MakeTarget: # 1) identifies Makefile target(s) to build (following make clean) # 2) defaults to blank (equivalent to "all") -# 3) used in --target pre-production -# 4) hence applicable to --target production -# 4) ensure MakeTarget actually builds imgr +# 3) only used in --target build # --no-cache: # 1) tells Docker to ignore cached images that might be stale # 2) useful due to Docker not understanding changes to build-args # 3) useful due to Docker not understanding changes to context dir +# 4) if running a sequence of builds, consider instead docker builder prune # -t: # 1) provides a name REPOSITORY:TAG for the built image # 2) if no tag is specified, TAG will be "latest" @@ -49,24 +58,24 @@ # 1) bind mounts the context into /src in the container # 2) /src will be a read-write'able equivalent to the context dir # 3) only useful for --target development -# 4) /src will be a local copy instead for --target pre-production -# 5) /src doesn't exist for --target production +# 4) /src will be a local copy instead for --target build +# 5) /src doesn't exist for --target build FROM alpine:3.14.0 as base ARG GolangVersion=1.16.7 ARG MakeTarget RUN apk add --no-cache libc6-compat -FROM base as development -RUN apk add --no-cache bind-tools -RUN apk add --no-cache curl -RUN apk add --no-cache gcc -RUN apk add --no-cache git -RUN apk add --no-cache jq -RUN apk add --no-cache libc-dev -RUN apk add --no-cache make -RUN apk add --no-cache tar -RUN curl -sSL https://github.com/coreos/etcd/releases/download/v3.5.0/etcd-v3.5.0-linux-amd64.tar.gz \ +FROM base as dev +RUN apk add --no-cache bind-tools \ + curl \ + gcc \ + git \ + jq \ + libc-dev \ + make \ + tar +RUN curl -sSL https://github.com/coreos/etcd/releases/download/v3.5.0/etcd-v3.5.0-linux-amd64.tar.gz \ | tar -vxz -C /usr/local/bin --strip=1 etcd-v3.5.0-linux-amd64/etcd etcd-v3.5.0-linux-amd64/etcdctl \ && chown root:root /usr/local/bin/etcd /usr/local/bin/etcdctl ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" @@ -78,17 +87,23 @@ ENV PATH $PATH:/usr/local/go/bin VOLUME /src WORKDIR /src -FROM development as pre-production +FROM dev as build VOLUME /src COPY . /src WORKDIR /src RUN make clean RUN make $MakeTarget -FROM base as production -COPY --from=pre-production /src/icert/icert ./ +FROM base as imgr +COPY --from=build /src/icert/icert ./ RUN ./icert -ca -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 -RUN ./icert -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 -cert cert.pem -key key.pem -dns production -COPY --from=pre-production /src/imgr/imgr ./ -COPY --from=pre-production /src/imgr/production.conf ./ -CMD ["./imgr", "production.conf"] +RUN ./icert -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 -cert cert.pem -key key.pem -dns imgr +COPY --from=build /src/imgr/imgr ./ +COPY --from=build /src/imgr/imgr.conf ./ + +FROM imgr as iclient +RUN rm caKey.pem cert.pem key.pem imgr imgr.conf +COPY --from=build /src/iclient/iclient ./ +COPY --from=build /src/iclient/iclient.conf ./ +COPY --from=build /src/iclient/iclient.sh ./ +RUN apk add --no-cache curl diff --git a/docker-compose.yml b/docker-compose.yml index f971613f..1fa37d38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,11 @@ # 1) tells Docker to ignore cached images that might be stale # 2) useful due to Docker not understanding changes to build-args # 3) useful due to Docker not understanding changes to context dir +# 4) finer grained alternative to previously running docker builder prune # # To run the resultant application: # -# docker compose up [-d|--detach] [{development|production}] +# docker compose up [-d|--detach] [{dev|imgr|iclient}] # # Notes: # -d|--detach: @@ -21,8 +22,9 @@ # 3) if not supplied: # a) stop application with ^C # b) containers are left in "exited" state upon application down -# development: tells docker compose to only bring up development service (w/ dependencies) -# production: tells docker compose to only bring up production service (w/ dependencies) +# dev: tells docker compose to only bring up dev service (w/ dependencies) +# imgr: tells docker compose to only bring up imgr service (w/ dependencies) +# iclient: tells docker compose to only bring up iclient service (w/ dependencies) # # To stop the resultant application: # @@ -31,6 +33,10 @@ version: '3.8' services: + swift: + image: dockerswiftaio/docker-swift:2.27.0 + expose: + - 8080 # curl http://swift:8080/info etcd: image: bitnami/etcd:3.5.0 expose: @@ -38,14 +44,10 @@ services: - 2380 environment: - ALLOW_NONE_AUTHENTICATION=yes - swift: - image: dockerswiftaio/docker-swift:2.27.0 - expose: - - 8080 # curl http://swift:8080/info - development: + dev: build: context: . - target: development + target: dev depends_on: - etcd - swift @@ -56,15 +58,24 @@ services: - type: bind source: . target: /src - command: ["top"] - production: + command: ["sleep", "10000"] + imgr: build: context: . - target: production + target: imgr depends_on: - - etcd - swift + - etcd expose: - 15346 # IMGR.HTTPServerPort - 32356 # IMGR.RetryRPCPort + command: ["./imgr", "imgr.conf"] + iclient: + build: + context: . + target: iclient + depends_on: + - swift + - imgr + command: ["./iclient.sh"] \ No newline at end of file diff --git a/iclient/dev.conf b/iclient/dev.conf new file mode 100644 index 00000000..e69de29b diff --git a/iclient/iclient.conf b/iclient/iclient.conf new file mode 100644 index 00000000..e69de29b diff --git a/iclient/iclient.sh b/iclient/iclient.sh new file mode 100755 index 00000000..6758c28d --- /dev/null +++ b/iclient/iclient.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +AuthToken="" + +while [ "$AuthToken" == "" ] +do + sleep 1 + AuthToken=`curl -v -s -H "X-Auth-User: test:tester" -H "X-Auth-Key: testing" swift:8080/auth/v1.0 2>&1 | awk /"X-Auth-Token:"/'{print $3}'` +done + +curl -v -s -H "X-Auth-Token: $AuthToken" swift:8080/v1/AUTH_test/con -X PUT + +DollarQuestionMark=1 + +while [ "$DollarQuestionMark" != "0" ] +do + sleep 1 + curl imgr:15346/version -f > /dev/null -s + DollarQuestionMark=$? +done + +curl -v -s imgr:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" + +sleep 10000 diff --git a/imgr/development.conf b/imgr/dev.conf similarity index 93% rename from imgr/development.conf rename to imgr/dev.conf index 6376b04e..aa6fdf78 100644 --- a/imgr/development.conf +++ b/imgr/dev.conf @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 [IMGR] -PublicIPAddr: development -PrivateIPAddr: development +PublicIPAddr: dev +PrivateIPAddr: dev RetryRPCPort: 32356 HTTPServerPort: 15346 diff --git a/imgr/production.conf b/imgr/imgr.conf similarity index 92% rename from imgr/production.conf rename to imgr/imgr.conf index 8e6e38dd..38b6dfe0 100644 --- a/imgr/production.conf +++ b/imgr/imgr.conf @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 [IMGR] -PublicIPAddr: production -PrivateIPAddr: production +PublicIPAddr: imgr +PrivateIPAddr: imgr RetryRPCPort: 32356 HTTPServerPort: 15346 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 1d5b8137..690f6161 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -15,8 +15,8 @@ // argument, a package conf ConfMap. Here is a sample .conf file: // // [IMGR] -// PublicIPAddr: 172.28.128.2 -// PrivateIPAddr: 172.28.128.2 +// PublicIPAddr: 127.0.0.1 +// PrivateIPAddr: 127.0.0.1 // RetryRPCPort: 32356 // HTTPServerPort: 15346 // From 409762fa4abbd98378ed0ea0891fc5ccf414bde5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 8 Aug 2021 09:15:26 -0700 Subject: [PATCH 045/258] Updated go.mod to require go 1.16 --- go.mod | 8 +- go.sum | 471 --------------------------------------------------------- 2 files changed, 1 insertion(+), 478 deletions(-) diff --git a/go.mod b/go.mod index fd582fcc..e8fdfc5c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/NVIDIA/proxyfs -go 1.15 +go 1.16 replace github.com/coreos/bbolt v1.3.4 => go.etcd.io/bbolt v1.3.4 @@ -20,7 +20,6 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 - github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -33,17 +32,12 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kr/pretty v0.2.0 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/pborman/uuid v1.2.1 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/cast v1.4.0 // indirect - github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/bbolt v1.3.4 // indirect go.etcd.io/etcd v3.3.25+incompatible go.uber.org/zap v1.18.1 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e diff --git a/go.sum b/go.sum index d53480e0..1b0d8831 100644 --- a/go.sum +++ b/go.sum @@ -1,59 +1,15 @@ bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d h1:DSX7RxYXf0jevVSL+NbPdaMKNr8YADNcAFYvP1L10JE= github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d/go.mod h1:d68dR75IgEqHcodexLfWr6dEFVUs90ApRj5KSYkL59k= -github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e h1:77Ex8SDpKeR9gYTEXAJWEyRVTw5PClbHwe7O+QBnYsk= -github.com/NVIDIA/fission v0.0.0-20210702173100-e5500f2c870e/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc h1:UqHloRP/FbKINAD21ORQRXEkbJcDu9AUbY1AMqbbZmo= github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc/go.mod h1:WCQif2Y+7Y0CtS2kbupILmlNEmFnQRRaJRoDAKhreFg= -github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d h1:x/cEPpdNatZGZlnQ0lalpfxy7bd7dy9zW/SZKMGn5Ys= github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d/go.mod h1:94gmSfIoS2wX1UFVNLFTfLaueExrrVVpVVp8QP1iGmw= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -67,71 +23,39 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/bbolt v1.3.4 h1:0VqjxUwoTLxM3PmsSIk0hI2ao6gTtButQ2z8FT4//yo= -github.com/coreos/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -139,34 +63,19 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -176,85 +85,29 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -262,19 +115,14 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -282,27 +130,10 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -311,30 +142,14 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= -github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= -github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= -github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -344,98 +159,45 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= -github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -445,244 +207,90 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 h1:siQdpVirKtzPhKl3lZWozZraCFObP8S1v6PRp0bLrtU= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -691,76 +299,12 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -783,13 +327,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -803,14 +340,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From b9e2a3b4bc055e5f018bd74e2e50d059b117b637 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 8 Aug 2021 10:24:36 -0700 Subject: [PATCH 046/258] Exposed docker compose launched container service ports to host (localh --- docker-compose.yml | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1fa37d38..0ef5fce2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,25 +35,47 @@ version: '3.8' services: swift: image: dockerswiftaio/docker-swift:2.27.0 + container_name: proxyfs_swift expose: - 8080 # curl http://swift:8080/info + ports: + - target: 8080 + published: 8080 + protocol: tcp + mode: host etcd: image: bitnami/etcd:3.5.0 + container_name: proxyfs_etcd expose: - - 2379 # etcdctl --endpoints http://etcd:2379 - - 2380 + - 2379 # etcdctl --endpoints=etcd:2379 + ports: + - target: 2379 + published: 2379 # etcdctl [--endpoints=localhost:2379] + protocol: tcp + mode: host environment: - ALLOW_NONE_AUTHENTICATION=yes dev: build: context: . target: dev + container_name: proxyfs_dev depends_on: - etcd - swift expose: - - 15346 # IMGR.HTTPServerPort - 32356 # IMGR.RetryRPCPort + - 15346 # IMGR.HTTPServerPort + ports: + - target: 32356 + published: 32356 + protocol: tcp + mode: host + ports: + - target: 15346 + published: 15346 + protocol: tcp + mode: host volumes: - type: bind source: . @@ -63,17 +85,29 @@ services: build: context: . target: imgr + container_name: proxyfs_imgr depends_on: - swift - etcd expose: - - 15346 # IMGR.HTTPServerPort - 32356 # IMGR.RetryRPCPort + - 15346 # IMGR.HTTPServerPort + ports: + - target: 32356 + published: 32356 + protocol: tcp + mode: host + ports: + - target: 15346 + published: 15346 + protocol: tcp + mode: host command: ["./imgr", "imgr.conf"] iclient: build: context: . target: iclient + container_name: proxyfs_iclient depends_on: - swift - imgr From fa6d29fe2a1c71ea7b05d56eaa94ae43f4d79377 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 8 Aug 2021 10:50:48 -0700 Subject: [PATCH 047/258] Flushed out a few comments in docker-compose.yml --- docker-compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0ef5fce2..1602269b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,11 +3,13 @@ # docker compose build [--no-cache] # # Notes: -# --no-cache +# --no-cache: # 1) tells Docker to ignore cached images that might be stale # 2) useful due to Docker not understanding changes to build-args # 3) useful due to Docker not understanding changes to context dir # 4) finer grained alternative to previously running docker builder prune +# Images built will be named "proxyfs_:latest" +# Use docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] to create an alias # # To run the resultant application: # @@ -25,6 +27,7 @@ # dev: tells docker compose to only bring up dev service (w/ dependencies) # imgr: tells docker compose to only bring up imgr service (w/ dependencies) # iclient: tells docker compose to only bring up iclient service (w/ dependencies) +# Containers launched have been named the same as their corresponding service name # # To stop the resultant application: # From b607993feaf6ea1eb7188554b0fcabf32d8724e8 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 8 Aug 2021 11:12:13 -0700 Subject: [PATCH 048/258] Informed imgr to not only format AUTH_test/con... but serve it (as "testvol") --- docker-compose.yml | 4 +++- iclient/iclient.sh | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1602269b..f2f58ffa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ # # To run the resultant application: # -# docker compose up [-d|--detach] [{dev|imgr|iclient}] +# docker compose up [-d|--detach] {dev|imgr|iclient} # # Notes: # -d|--detach: @@ -27,6 +27,8 @@ # dev: tells docker compose to only bring up dev service (w/ dependencies) # imgr: tells docker compose to only bring up imgr service (w/ dependencies) # iclient: tells docker compose to only bring up iclient service (w/ dependencies) +# Precisely one of {dev|imgr|iclient} must be specied... default (all) will conflict +# If no service is specified, services dev and imgr published ports will collide # Containers launched have been named the same as their corresponding service name # # To stop the resultant application: diff --git a/iclient/iclient.sh b/iclient/iclient.sh index 6758c28d..8e35b122 100755 --- a/iclient/iclient.sh +++ b/iclient/iclient.sh @@ -15,10 +15,12 @@ DollarQuestionMark=1 while [ "$DollarQuestionMark" != "0" ] do sleep 1 - curl imgr:15346/version -f > /dev/null -s + curl -f imgr:15346/version > /dev/null -s DollarQuestionMark=$? done -curl -v -s imgr:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" +curl -v -s -f imgr:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" + +curl -v -s -f imgr:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\"}" sleep 10000 From 606a3e5f2ea0fd858d8b58d0688a1ab4e3195a84 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 9 Aug 2021 11:03:39 -0700 Subject: [PATCH 049/258] Updated README.md to match the new path to an development environment --- README.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1bbc61aa..183ea42b 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,8 @@ Integrated File and Object Access for Swift Object Storage ## Synopsis -ProxyFS is a hierarchical file system that provides integrated file -and object API access for data stored with -[Swift object storage](http://swift.openstack.org) for the same -data. It supports SMB and NFS along with AWS S3 and Swift object -protocols. We call this "bi-modal access", and it means that filesystem -clients and mount and read/write data that is simultaneously accessible -to object API clients. +ProxyFS is a disributed read/write hierarchical POSIX file system layered on top of +[Swift object storage](http://swift.openstack.org). ## How to Contribute @@ -35,11 +30,13 @@ or the [ProxyFS Slack group](https://proxyfs.slack.com), which you can join thro ## How to get the code * git clone git@github.com:NVIDIA/proxyfs.git -* cd ProxyFS +* cd proxyfs ## How to run unit tests (in your Development Environment) -* make +* [Host shell] docker compose build +* [Host shell] docker compose up -d dev +* [dev shell] make ## License From dcdf517f7878fd7d475198720f8d91a8670ee929 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 9 Aug 2021 16:44:03 -0700 Subject: [PATCH 050/258] Added imgr/mkmount.sh to do the "mkfs" and tell imgr to mount it --- go.mod | 7 +- go.sum | 467 +++++++++++++++++++++++++++++++++++++++++++++ iclient/iclient.sh | 2 +- imgr/mkmount.sh | 28 +++ 4 files changed, 502 insertions(+), 2 deletions(-) create mode 100755 imgr/mkmount.sh diff --git a/go.mod b/go.mod index e8fdfc5c..a62d512e 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -32,16 +33,20 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kr/pretty v0.2.0 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/pborman/uuid v1.2.1 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spf13/cast v1.4.0 // indirect + github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/etcd v3.3.25+incompatible go.uber.org/zap v1.18.1 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e - golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 + golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 1b0d8831..468e9127 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,45 @@ bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d h1:DSX7RxYXf0jevVSL+NbPdaMKNr8YADNcAFYvP1L10JE= @@ -10,6 +49,7 @@ github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc/go.mod h1:WCQif2Y+7 github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d h1:x/cEPpdNatZGZlnQ0lalpfxy7bd7dy9zW/SZKMGn5Ys= github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d/go.mod h1:94gmSfIoS2wX1UFVNLFTfLaueExrrVVpVVp8QP1iGmw= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -23,39 +63,69 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -63,19 +133,34 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -85,29 +170,85 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -115,14 +256,19 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -130,10 +276,27 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -142,14 +305,30 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -159,45 +338,98 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= +github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -207,90 +439,246 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 h1:siQdpVirKtzPhKl3lZWozZraCFObP8S1v6PRp0bLrtU= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809203939-894668206c86 h1:MmHNJuaWvdCGJjEzhPN709DMlCsVJvzBr1s8b2HduQE= +golang.org/x/sys v0.0.0-20210809203939-894668206c86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -299,12 +687,76 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -327,6 +779,13 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -340,6 +799,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/iclient/iclient.sh b/iclient/iclient.sh index 8e35b122..75957931 100755 --- a/iclient/iclient.sh +++ b/iclient/iclient.sh @@ -15,7 +15,7 @@ DollarQuestionMark=1 while [ "$DollarQuestionMark" != "0" ] do sleep 1 - curl -f imgr:15346/version > /dev/null -s + curl -s -f imgr:15346/version > /dev/null DollarQuestionMark=$? done diff --git a/imgr/mkmount.sh b/imgr/mkmount.sh new file mode 100755 index 00000000..36d1a1e1 --- /dev/null +++ b/imgr/mkmount.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +AuthToken="" + +while [ "$AuthToken" == "" ] +do + sleep 1 + AuthToken=`curl -v -s -H "X-Auth-User: test:tester" -H "X-Auth-Key: testing" swift:8080/auth/v1.0 2>&1 | awk /"X-Auth-Token:"/'{print $3}'` +done + +echo +echo " AuthToken: $AuthToken" +echo + +curl -s -f -H "X-Auth-Token: $AuthToken" swift:8080/v1/AUTH_test/con -X PUT > /dev/null + +DollarQuestionMark=1 + +while [ "$DollarQuestionMark" != "0" ] +do + sleep 1 + curl -s -f dev:15346/version > /dev/null + DollarQuestionMark=$? +done + +curl -s -f dev:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" > /dev/null + +curl -s -f dev:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\"}" > /dev/null From f1887c574ee4875c9c6a1867b16e96b57086f95b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 10 Aug 2021 19:10:17 -0700 Subject: [PATCH 051/258] Added "pretty" HTML serving for some of the web pages (e.g. in text/html is accepted) --- Makefile | 30 +- docker-compose.yml | 2 +- go.mod | 1 + go.sum | 7 +- iclient/iclient.sh | 2 +- imgr/imgrpkg/.gitignore | 13 + imgr/imgrpkg/Makefile | 15 + imgr/imgrpkg/html_templates.go | 502 ++++++++++++++++ imgr/imgrpkg/http-server.go | 79 ++- imgr/imgrpkg/static-content.go | 22 + imgr/imgrpkg/static-content/bootstrap.min.css | 7 + imgr/imgrpkg/static-content/bootstrap.min.js | 7 + imgr/imgrpkg/static-content/jquery.min.js | 2 + imgr/imgrpkg/static-content/jsontree.js | 182 ++++++ .../font/css/open-iconic-bootstrap.min.css | 1 + .../open-iconic/font/fonts/open-iconic.eot | Bin 0 -> 28196 bytes .../open-iconic/font/fonts/open-iconic.otf | Bin 0 -> 20996 bytes .../open-iconic/font/fonts/open-iconic.svg | 543 ++++++++++++++++++ .../open-iconic/font/fonts/open-iconic.ttf | Bin 0 -> 28028 bytes .../open-iconic/font/fonts/open-iconic.woff | Bin 0 -> 14984 bytes imgr/imgrpkg/static-content/popper.min.js | 5 + imgr/imgrpkg/static-content/styles.css | 71 +++ make-static-content/.gitignore | 1 + make-static-content/Makefile | 3 + make-static-content/main.go | 16 +- 25 files changed, 1487 insertions(+), 24 deletions(-) create mode 100644 imgr/imgrpkg/.gitignore create mode 100644 imgr/imgrpkg/html_templates.go create mode 100644 imgr/imgrpkg/static-content.go create mode 100644 imgr/imgrpkg/static-content/bootstrap.min.css create mode 100644 imgr/imgrpkg/static-content/bootstrap.min.js create mode 100644 imgr/imgrpkg/static-content/jquery.min.js create mode 100644 imgr/imgrpkg/static-content/jsontree.js create mode 100755 imgr/imgrpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css create mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.eot create mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.otf create mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.svg create mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.ttf create mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.woff create mode 100644 imgr/imgrpkg/static-content/popper.min.js create mode 100644 imgr/imgrpkg/static-content/styles.css create mode 100644 make-static-content/.gitignore diff --git a/Makefile b/Makefile index 587a4d1e..db4675f4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 +gopregeneratedirs = \ + make-static-content + gopkgdirs = \ bucketstats \ conf \ @@ -26,13 +29,16 @@ gobindirs = \ godirsforci = $(gopkgdirs) $(goplugindirs) $(gobindirs); godirpathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(godirsforci)) -all: version fmt generate test build +generatedfiles := \ + coverage.coverprofile + +all: version fmt pre-generate generate test build -ci: version fmt generate test cover build +ci: version fmt pre-generate generate test cover build -minimal: version generate build +minimal: version pre-generate generate build -.PHONY: all bench build ci clean cover fmt generate minimal test version +.PHONY: all bench build ci clean cover fmt generate minimal pre-generate test version bench: @set -e; \ @@ -60,6 +66,9 @@ build: clean: @set -e; \ + for godir in $(gopregeneratedirs); do \ + $(MAKE) --no-print-directory -C $$godir clean; \ + done; \ for godir in $(gopkgdirs); do \ $(MAKE) --no-print-directory -C $$godir clean; \ done; \ @@ -69,6 +78,9 @@ clean: for godir in $(gobindirs); do \ $(MAKE) --no-print-directory -C $$godir clean; \ done; \ + for generatedfile in $(generatedfiles); do \ + rm -f $$generatedfile; \ + done; \ rm -f go-acc cover: @@ -79,7 +91,9 @@ cover: fmt: @set -e; \ - $(MAKE) --no-print-directory -C make-static-content fmt; \ + for godir in $(gopregeneratedirs); do \ + $(MAKE) --no-print-directory -C $$godir fmt; \ + done; \ for godir in $(gopkgdirs); do \ $(MAKE) --no-print-directory -C $$godir fmt; \ done; \ @@ -102,6 +116,12 @@ generate: $(MAKE) --no-print-directory -C $$godir generate; \ done +pre-generate: + @set -e; \ + for godir in $(gopregeneratedirs); do \ + $(MAKE) --no-print-directory -C $$godir build; \ + done + test: @set -e; \ for godir in $(gopkgdirs); do \ diff --git a/docker-compose.yml b/docker-compose.yml index f2f58ffa..4e6560d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,7 +85,7 @@ services: - type: bind source: . target: /src - command: ["sleep", "10000"] + command: ["sleep", "100000"] imgr: build: context: . diff --git a/go.mod b/go.mod index a62d512e..836b373c 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( go.uber.org/zap v1.18.1 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e + golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 468e9127..e8a71cea 100644 --- a/go.sum +++ b/go.sum @@ -606,10 +606,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 h1:siQdpVirKtzPhKl3lZWozZraCFObP8S1v6PRp0bLrtU= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809203939-894668206c86 h1:MmHNJuaWvdCGJjEzhPN709DMlCsVJvzBr1s8b2HduQE= -golang.org/x/sys v0.0.0-20210809203939-894668206c86/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -620,8 +616,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/iclient/iclient.sh b/iclient/iclient.sh index 75957931..7753a64b 100755 --- a/iclient/iclient.sh +++ b/iclient/iclient.sh @@ -23,4 +23,4 @@ curl -v -s -f imgr:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/ curl -v -s -f imgr:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\"}" -sleep 10000 +sleep 100000 diff --git a/imgr/imgrpkg/.gitignore b/imgr/imgrpkg/.gitignore new file mode 100644 index 00000000..3a694af3 --- /dev/null +++ b/imgr/imgrpkg/.gitignore @@ -0,0 +1,13 @@ +bootstrap_dot_min_dot_css_.go +bootstrap_dot_min_dot_js_.go +jquery_dot_min_dot_js_.go +jquery_underline_3_dot_2_dot_1_dot_min_dot_js_.go +jsontree_dot_js_.go +open_iconic_bootstrap_dot_css_.go +open_iconic_dot_eot_.go +open_iconic_dot_otf_.go +open_iconic_dot_svg_.go +open_iconic_dot_ttf_.go +open_iconic_dot_woff_.go +popper_dot_min_dot_js_.go +styles_dot_css_.go diff --git a/imgr/imgrpkg/Makefile b/imgr/imgrpkg/Makefile index a2b7d274..8df1c430 100644 --- a/imgr/imgrpkg/Makefile +++ b/imgr/imgrpkg/Makefile @@ -3,4 +3,19 @@ gosubdir := github.com/NVIDIA/proxyfs/imgr/imgrpkg +generatedfiles := \ + bootstrap_dot_min_dot_css_.go \ + bootstrap_dot_min_dot_js_.go \ + jquery_dot_min_dot_js_.go \ + jquery_underline_3_dot_2_dot_1_dot_min_dot_js_.go \ + jsontree_dot_js_.go \ + open_iconic_bootstrap_dot_css_.go \ + open_iconic_dot_eot_.go \ + open_iconic_dot_otf_.go \ + open_iconic_dot_svg_.go \ + open_iconic_dot_ttf_.go \ + open_iconic_dot_woff_.go \ + popper_dot_min_dot_js_.go \ + styles_dot_css_.go + include ../../GoMakefile diff --git a/imgr/imgrpkg/html_templates.go b/imgr/imgrpkg/html_templates.go new file mode 100644 index 00000000..29f31dd6 --- /dev/null +++ b/imgr/imgrpkg/html_templates.go @@ -0,0 +1,502 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +// To use: fmt.Sprintf(indexDotHTMLTemplate, proxyfsVersion) +const indexDotHTMLTemplate string = ` + + + + + + + imgr + + + +
+ +

+ ProxyFS imgr +

+
+
+
+
Configuration parameters
+

Diplays a JSON representation of the active configuration.

+
+ +
+
+
+
+
Stats
+

Displays current statistics.

+
+ +
+
+
+
+
+
+
+
+
Volumes
+

Examine volumes currently active on this imgr node.

+
+ +
+
+
+ + + + + +` + +// To use: fmt.Sprintf(configTemplate, proxyfsVersion, confMapJSONString) +const configTemplate string = ` + + + + + + + Config + + + +
+ +

+ Config +

+

+    
+ + + + + + + +` + +// To use: fmt.Sprintf(volumeListTopTemplate, proxyfsVersion) +const volumeListTopTemplate string = ` + + + + + + + Volumes + + + +
+ + +

Volumes

+ + + + + + + + + + + + +` + +// To use: fmt.Sprintf(volumeListPerVolumeTemplate, volumeName) +const volumeListPerVolumeTemplate string = ` + + + + + + + +` + +const volumeListBottom string = ` +
Volume Name     
%[1]vSnapShotsFSCK jobsSCRUB jobsLayout ReportExtent Map
+
+ + + + + +` + +// To use: fmt.Sprintf(layoutReportTopTemplate, proxyfsVersion, volumeName) +const layoutReportTopTemplate string = ` + + + + + + + Layout Report %[2]v + + + +
+ +

+ Layout Report + %[2]v +

+ Show validated version

+` + +// To use: fmt.Sprintf(layoutReportTableTopTemplate, TreeName, NumDiscrepencies, {"success"|"danger"}) +const layoutReportTableTopTemplate string = `
+
+

%[1]v

%[2]v discrepancies

+
+ + + + + + + + +` + +// To use: fmt.Sprintf(layoutReportTableRowTemplate, ObjectName, ObjectBytes) +const layoutReportTableRowTemplate string = ` + + + +` + +const layoutReportTableBottom string = ` +
ObjectNameObjectBytes
%016[1]X
%[2]v
+` + +const layoutReportBottom string = `
+ + + + + + +` + +// To use: fmt.Sprintf(extentMapTemplate, proxyfsVersion, volumeName, extentMapJSONString, pathDoubleQuotedString, serverErrorBoolString) +const extentMapTemplate string = ` + + + + + + + Extent Map %[2]v + + + +
+ + +

+ Extent Map + %[2]v +

+ + + +
+
+ +
+ +
+
+
+ +
+ + + + + + + + + + +
File OffsetContainer/ObjectObject OffsetLength
+
+ + + + + + +` diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 9a0ce625..ca10db03 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -149,10 +149,66 @@ func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.R func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { switch { + case "" == requestPath: + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) + case "/bootstrap.min.css" == requestPath: + responseWriter.Header().Set("Content-Type", bootstrapDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(bootstrapDotCSSContent)) + case "/bootstrap.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", bootstrapDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(bootstrapDotJSContent)) case "/config" == requestPath: serveHTTPGetOfConfig(responseWriter, request) + case "/index.html" == requestPath: + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) + case "/jquery.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", jqueryDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(jqueryDotJSContent)) + case "/jsontree.js" == requestPath: + responseWriter.Header().Set("Content-Type", jsontreeDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(jsontreeDotJSContent)) + case "/open-iconic/font/css/open-iconic-bootstrap.min.css" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicBootstrapDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(openIconicBootstrapDotCSSContent)) + case "/open-iconic/font/fonts/open-iconic.eot" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotEOTContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotEOTContent) + case "/open-iconic/font/fonts/open-iconic.otf" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotOTFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotOTFContent) + case "/open-iconic/font/fonts/open-iconic.svg" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotSVGContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(openIconicDotSVGContent)) + case "/open-iconic/font/fonts/open-iconic.ttf" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotTTFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotTTFContent) + case "/open-iconic/font/fonts/open-iconic.woff" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotWOFFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotWOFFContent) + case "/popper.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", popperDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(popperDotJSContent) case "/stats" == requestPath: serveHTTPGetOfStats(responseWriter, request) + case "/styles.css" == requestPath: + responseWriter.Header().Set("Content-Type", stylesDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(stylesDotCSSContent)) case "/version" == requestPath: serveHTTPGetOfVersion(responseWriter, request) case strings.HasPrefix(requestPath, "/volume"): @@ -179,13 +235,23 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ logFatalf("json.Marshal(globals.config) failed: %v", err) } - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(confMapJSON))) - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) + if strings.Contains(request.Header.Get("Accept"), "text/html") { + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) - _, err = responseWriter.Write(confMapJSON) - if nil != err { - logWarnf("responseWriter.Write(confMapJSON) failed: %v", err) + _, err = responseWriter.Write([]byte(fmt.Sprintf(configTemplate, version.ProxyFSVersion, string(confMapJSON[:])))) + if nil != err { + logWarnf("responseWriter.Write([]byte(fmt.Sprintf(configTemplate, version.ProxyFSVersion, string(confMapJSON[:])))) failed: %v", err) + } + } else { + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(confMapJSON))) + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write(confMapJSON) + if nil != err { + logWarnf("responseWriter.Write(confMapJSON) failed: %v", err) + } } } @@ -248,7 +314,6 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ switch len(pathSplit) { case 2: - startTime = time.Now() defer func() { globals.stats.GetVolumeListUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() diff --git a/imgr/imgrpkg/static-content.go b/imgr/imgrpkg/static-content.go new file mode 100644 index 00000000..1636cf7e --- /dev/null +++ b/imgr/imgrpkg/static-content.go @@ -0,0 +1,22 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +//go:generate ../../make-static-content/make-static-content imgrpkg stylesDotCSS text/css s static-content/styles.css styles_dot_css_.go + +//go:generate ../../make-static-content/make-static-content imgrpkg jsontreeDotJS application/javascript s static-content/jsontree.js jsontree_dot_js_.go + +//go:generate ../../make-static-content/make-static-content imgrpkg bootstrapDotCSS text/css s static-content/bootstrap.min.css bootstrap_dot_min_dot_css_.go + +//go:generate ../../make-static-content/make-static-content imgrpkg bootstrapDotJS application/javascript s static-content/bootstrap.min.js bootstrap_dot_min_dot_js_.go +//go:generate ../../make-static-content/make-static-content imgrpkg jqueryDotJS application/javascript s static-content/jquery.min.js jquery_dot_min_dot_js_.go +//go:generate ../../make-static-content/make-static-content imgrpkg popperDotJS application/javascript b static-content/popper.min.js popper_dot_min_dot_js_.go + +//go:generate ../../make-static-content/make-static-content imgrpkg openIconicBootstrapDotCSS text/css s static-content/open-iconic/font/css/open-iconic-bootstrap.min.css open_iconic_bootstrap_dot_css_.go + +//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotEOT application/vnd.ms-fontobject b static-content/open-iconic/font/fonts/open-iconic.eot open_iconic_dot_eot_.go +//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotOTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.otf open_iconic_dot_otf_.go +//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotSVG image/svg+xml s static-content/open-iconic/font/fonts/open-iconic.svg open_iconic_dot_svg_.go +//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotTTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.ttf open_iconic_dot_ttf_.go +//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotWOFF application/font-woff b static-content/open-iconic/font/fonts/open-iconic.woff open_iconic_dot_woff_.go diff --git a/imgr/imgrpkg/static-content/bootstrap.min.css b/imgr/imgrpkg/static-content/bootstrap.min.css new file mode 100644 index 00000000..7d2a868f --- /dev/null +++ b/imgr/imgrpkg/static-content/bootstrap.min.css @@ -0,0 +1,7 @@ +/*! + * Bootstrap v4.5.0 (https://getbootstrap.com/) + * Copyright 2011-2020 The Bootstrap Authors + * Copyright 2011-2020 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */:root{--blue:#007bff;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#dc3545;--orange:#fd7e14;--yellow:#ffc107;--green:#28a745;--teal:#20c997;--cyan:#17a2b8;--white:#fff;--gray:#6c757d;--gray-dark:#343a40;--primary:#007bff;--secondary:#6c757d;--success:#28a745;--info:#17a2b8;--warning:#ffc107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus:not(:focus-visible){outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]){color:inherit;text-decoration:none}a:not([href]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-weight:500;line-height:1.2}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#6c757d}.blockquote-footer::before{content:"\2014\00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#6c757d}code{font-size:87.5%;color:#e83e8c;word-wrap:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:87.5%;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;font-size:87.5%;color:#212529}pre code{font-size:inherit;color:inherit;word-break:normal}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container{max-width:540px}}@media (min-width:768px){.container{max-width:720px}}@media (min-width:992px){.container{max-width:960px}}@media (min-width:1200px){.container{max-width:1140px}}.container-fluid,.container-lg,.container-md,.container-sm,.container-xl{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container,.container-sm{max-width:540px}}@media (min-width:768px){.container,.container-md,.container-sm{max-width:720px}}@media (min-width:992px){.container,.container-lg,.container-md,.container-sm{max-width:960px}}@media (min-width:1200px){.container,.container-lg,.container-md,.container-sm,.container-xl{max-width:1140px}}.row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-auto,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-auto,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-auto,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-auto,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9,.col-xl-auto{position:relative;width:100%;padding-right:15px;padding-left:15px}.col{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-first{-ms-flex-order:-1;order:-1}.order-last{-ms-flex-order:13;order:13}.order-0{-ms-flex-order:0;order:0}.order-1{-ms-flex-order:1;order:1}.order-2{-ms-flex-order:2;order:2}.order-3{-ms-flex-order:3;order:3}.order-4{-ms-flex-order:4;order:4}.order-5{-ms-flex-order:5;order:5}.order-6{-ms-flex-order:6;order:6}.order-7{-ms-flex-order:7;order:7}.order-8{-ms-flex-order:8;order:8}.order-9{-ms-flex-order:9;order:9}.order-10{-ms-flex-order:10;order:10}.order-11{-ms-flex-order:11;order:11}.order-12{-ms-flex-order:12;order:12}.offset-1{margin-left:8.333333%}.offset-2{margin-left:16.666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.333333%}.offset-5{margin-left:41.666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.333333%}.offset-8{margin-left:66.666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.333333%}.offset-11{margin-left:91.666667%}@media (min-width:576px){.col-sm{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-sm-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-sm-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-sm-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-sm-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-sm-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-sm-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-sm-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-sm-first{-ms-flex-order:-1;order:-1}.order-sm-last{-ms-flex-order:13;order:13}.order-sm-0{-ms-flex-order:0;order:0}.order-sm-1{-ms-flex-order:1;order:1}.order-sm-2{-ms-flex-order:2;order:2}.order-sm-3{-ms-flex-order:3;order:3}.order-sm-4{-ms-flex-order:4;order:4}.order-sm-5{-ms-flex-order:5;order:5}.order-sm-6{-ms-flex-order:6;order:6}.order-sm-7{-ms-flex-order:7;order:7}.order-sm-8{-ms-flex-order:8;order:8}.order-sm-9{-ms-flex-order:9;order:9}.order-sm-10{-ms-flex-order:10;order:10}.order-sm-11{-ms-flex-order:11;order:11}.order-sm-12{-ms-flex-order:12;order:12}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.333333%}.offset-sm-2{margin-left:16.666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.333333%}.offset-sm-5{margin-left:41.666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.333333%}.offset-sm-8{margin-left:66.666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.333333%}.offset-sm-11{margin-left:91.666667%}}@media (min-width:768px){.col-md{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-md-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-md-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-md-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-md-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-md-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-md-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-md-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-md-first{-ms-flex-order:-1;order:-1}.order-md-last{-ms-flex-order:13;order:13}.order-md-0{-ms-flex-order:0;order:0}.order-md-1{-ms-flex-order:1;order:1}.order-md-2{-ms-flex-order:2;order:2}.order-md-3{-ms-flex-order:3;order:3}.order-md-4{-ms-flex-order:4;order:4}.order-md-5{-ms-flex-order:5;order:5}.order-md-6{-ms-flex-order:6;order:6}.order-md-7{-ms-flex-order:7;order:7}.order-md-8{-ms-flex-order:8;order:8}.order-md-9{-ms-flex-order:9;order:9}.order-md-10{-ms-flex-order:10;order:10}.order-md-11{-ms-flex-order:11;order:11}.order-md-12{-ms-flex-order:12;order:12}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.333333%}.offset-md-2{margin-left:16.666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.333333%}.offset-md-5{margin-left:41.666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.333333%}.offset-md-8{margin-left:66.666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.333333%}.offset-md-11{margin-left:91.666667%}}@media (min-width:992px){.col-lg{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-lg-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-lg-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-lg-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-lg-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-lg-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-lg-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-lg-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-lg-first{-ms-flex-order:-1;order:-1}.order-lg-last{-ms-flex-order:13;order:13}.order-lg-0{-ms-flex-order:0;order:0}.order-lg-1{-ms-flex-order:1;order:1}.order-lg-2{-ms-flex-order:2;order:2}.order-lg-3{-ms-flex-order:3;order:3}.order-lg-4{-ms-flex-order:4;order:4}.order-lg-5{-ms-flex-order:5;order:5}.order-lg-6{-ms-flex-order:6;order:6}.order-lg-7{-ms-flex-order:7;order:7}.order-lg-8{-ms-flex-order:8;order:8}.order-lg-9{-ms-flex-order:9;order:9}.order-lg-10{-ms-flex-order:10;order:10}.order-lg-11{-ms-flex-order:11;order:11}.order-lg-12{-ms-flex-order:12;order:12}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.333333%}.offset-lg-2{margin-left:16.666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.333333%}.offset-lg-5{margin-left:41.666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.333333%}.offset-lg-8{margin-left:66.666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.333333%}.offset-lg-11{margin-left:91.666667%}}@media (min-width:1200px){.col-xl{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-xl-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-xl-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-xl-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-xl-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-xl-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-xl-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-xl-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-xl-first{-ms-flex-order:-1;order:-1}.order-xl-last{-ms-flex-order:13;order:13}.order-xl-0{-ms-flex-order:0;order:0}.order-xl-1{-ms-flex-order:1;order:1}.order-xl-2{-ms-flex-order:2;order:2}.order-xl-3{-ms-flex-order:3;order:3}.order-xl-4{-ms-flex-order:4;order:4}.order-xl-5{-ms-flex-order:5;order:5}.order-xl-6{-ms-flex-order:6;order:6}.order-xl-7{-ms-flex-order:7;order:7}.order-xl-8{-ms-flex-order:8;order:8}.order-xl-9{-ms-flex-order:9;order:9}.order-xl-10{-ms-flex-order:10;order:10}.order-xl-11{-ms-flex-order:11;order:11}.order-xl-12{-ms-flex-order:12;order:12}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.333333%}.offset-xl-2{margin-left:16.666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.333333%}.offset-xl-5{margin-left:41.666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.333333%}.offset-xl-8{margin-left:66.666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.333333%}.offset-xl-11{margin-left:91.666667%}}.table{width:100%;margin-bottom:1rem;color:#212529}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #dee2e6}.table thead th{vertical-align:bottom;border-bottom:2px solid #dee2e6}.table tbody+tbody{border-top:2px solid #dee2e6}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #dee2e6}.table-bordered td,.table-bordered th{border:1px solid #dee2e6}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-borderless tbody+tbody,.table-borderless td,.table-borderless th,.table-borderless thead th{border:0}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{color:#212529;background-color:rgba(0,0,0,.075)}.table-primary,.table-primary>td,.table-primary>th{background-color:#b8daff}.table-primary tbody+tbody,.table-primary td,.table-primary th,.table-primary thead th{border-color:#7abaff}.table-hover .table-primary:hover{background-color:#9fcdff}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#9fcdff}.table-secondary,.table-secondary>td,.table-secondary>th{background-color:#d6d8db}.table-secondary tbody+tbody,.table-secondary td,.table-secondary th,.table-secondary thead th{border-color:#b3b7bb}.table-hover .table-secondary:hover{background-color:#c8cbcf}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#c8cbcf}.table-success,.table-success>td,.table-success>th{background-color:#c3e6cb}.table-success tbody+tbody,.table-success td,.table-success th,.table-success thead th{border-color:#8fd19e}.table-hover .table-success:hover{background-color:#b1dfbb}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#b1dfbb}.table-info,.table-info>td,.table-info>th{background-color:#bee5eb}.table-info tbody+tbody,.table-info td,.table-info th,.table-info thead th{border-color:#86cfda}.table-hover .table-info:hover{background-color:#abdde5}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#abdde5}.table-warning,.table-warning>td,.table-warning>th{background-color:#ffeeba}.table-warning tbody+tbody,.table-warning td,.table-warning th,.table-warning thead th{border-color:#ffdf7e}.table-hover .table-warning:hover{background-color:#ffe8a1}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#ffe8a1}.table-danger,.table-danger>td,.table-danger>th{background-color:#f5c6cb}.table-danger tbody+tbody,.table-danger td,.table-danger th,.table-danger thead th{border-color:#ed969e}.table-hover .table-danger:hover{background-color:#f1b0b7}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#f1b0b7}.table-light,.table-light>td,.table-light>th{background-color:#fdfdfe}.table-light tbody+tbody,.table-light td,.table-light th,.table-light thead th{border-color:#fbfcfc}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>td,.table-dark>th{background-color:#c6c8ca}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#95999c}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.table .thead-dark th{color:#fff;background-color:#343a40;border-color:#454d55}.table .thead-light th{color:#495057;background-color:#e9ecef;border-color:#dee2e6}.table-dark{color:#fff;background-color:#343a40}.table-dark td,.table-dark th,.table-dark thead th{border-color:#454d55}.table-dark.table-bordered{border:0}.table-dark.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,.05)}.table-dark.table-hover tbody tr:hover{color:#fff;background-color:rgba(255,255,255,.075)}@media (max-width:575.98px){.table-responsive-sm{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-sm>.table-bordered{border:0}}@media (max-width:767.98px){.table-responsive-md{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-md>.table-bordered{border:0}}@media (max-width:991.98px){.table-responsive-lg{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-lg>.table-bordered{border:0}}@media (max-width:1199.98px){.table-responsive-xl{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-xl>.table-bordered{border:0}}.table-responsive{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive>.table-bordered{border:0}.form-control{display:block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.form-control:focus{color:#495057;background-color:#fff;border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.form-control::-webkit-input-placeholder{color:#6c757d;opacity:1}.form-control::-moz-placeholder{color:#6c757d;opacity:1}.form-control:-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}input[type=date].form-control,input[type=datetime-local].form-control,input[type=month].form-control,input[type=time].form-control{-webkit-appearance:none;-moz-appearance:none;appearance:none}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block;width:100%}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem;line-height:1.5}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem;line-height:1.5}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;font-size:1rem;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.form-control-lg{height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}select.form-control[multiple],select.form-control[size]{height:auto}textarea.form-control{height:auto}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*=col-]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;padding-left:1.25rem}.form-check-input{position:absolute;margin-top:.3rem;margin-left:-1.25rem}.form-check-input:disabled~.form-check-label,.form-check-input[disabled]~.form-check-label{color:#6c757d}.form-check-label{margin-bottom:0}.form-check-inline{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-align:center;align-items:center;padding-left:0;margin-right:.75rem}.form-check-inline .form-check-input{position:static;margin-top:0;margin-right:.3125rem;margin-left:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#28a745}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(40,167,69,.9);border-radius:.25rem}.is-valid~.valid-feedback,.is-valid~.valid-tooltip,.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip{display:block}.form-control.is-valid,.was-validated .form-control:valid{border-color:#28a745;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-valid,.was-validated .custom-select:valid{border-color:#28a745;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-valid:focus,.was-validated .custom-select:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:#28a745}.form-check-input.is-valid~.valid-feedback,.form-check-input.is-valid~.valid-tooltip,.was-validated .form-check-input:valid~.valid-feedback,.was-validated .form-check-input:valid~.valid-tooltip{display:block}.custom-control-input.is-valid~.custom-control-label,.was-validated .custom-control-input:valid~.custom-control-label{color:#28a745}.custom-control-input.is-valid~.custom-control-label::before,.was-validated .custom-control-input:valid~.custom-control-label::before{border-color:#28a745}.custom-control-input.is-valid:checked~.custom-control-label::before,.was-validated .custom-control-input:valid:checked~.custom-control-label::before{border-color:#34ce57;background-color:#34ce57}.custom-control-input.is-valid:focus~.custom-control-label::before,.was-validated .custom-control-input:valid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-control-input.is-valid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:valid:focus:not(:checked)~.custom-control-label::before{border-color:#28a745}.custom-file-input.is-valid~.custom-file-label,.was-validated .custom-file-input:valid~.custom-file-label{border-color:#28a745}.custom-file-input.is-valid:focus~.custom-file-label,.was-validated .custom-file-input:valid:focus~.custom-file-label{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(220,53,69,.9);border-radius:.25rem}.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip,.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip{display:block}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:#dc3545;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-invalid,.was-validated .custom-select:invalid{border-color:#dc3545;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-invalid:focus,.was-validated .custom-select:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:#dc3545}.form-check-input.is-invalid~.invalid-feedback,.form-check-input.is-invalid~.invalid-tooltip,.was-validated .form-check-input:invalid~.invalid-feedback,.was-validated .form-check-input:invalid~.invalid-tooltip{display:block}.custom-control-input.is-invalid~.custom-control-label,.was-validated .custom-control-input:invalid~.custom-control-label{color:#dc3545}.custom-control-input.is-invalid~.custom-control-label::before,.was-validated .custom-control-input:invalid~.custom-control-label::before{border-color:#dc3545}.custom-control-input.is-invalid:checked~.custom-control-label::before,.was-validated .custom-control-input:invalid:checked~.custom-control-label::before{border-color:#e4606d;background-color:#e4606d}.custom-control-input.is-invalid:focus~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.custom-control-input.is-invalid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus:not(:checked)~.custom-control-label::before{border-color:#dc3545}.custom-file-input.is-invalid~.custom-file-label,.was-validated .custom-file-input:invalid~.custom-file-label{border-color:#dc3545}.custom-file-input.is-invalid:focus~.custom-file-label,.was-validated .custom-file-input:invalid:focus~.custom-file-label{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-inline{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-ms-flexbox;display:flex;-ms-flex:0 0 auto;flex:0 0 auto;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .custom-select,.form-inline .input-group{width:auto}.form-inline .form-check{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:auto;padding-left:0}.form-inline .form-check-input{position:relative;-ms-flex-negative:0;flex-shrink:0;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center}.form-inline .custom-control-label{margin-bottom:0}}.btn{display:inline-block;font-weight:400;color:#212529;text-align:center;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;line-height:1.5;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:#212529;text-decoration:none}.btn.focus,.btn:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.btn.disabled,.btn:disabled{opacity:.65}.btn:not(:disabled):not(.disabled){cursor:pointer}a.btn.disabled,fieldset:disabled a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:hover{color:#fff;background-color:#0069d9;border-color:#0062cc}.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#0069d9;border-color:#0062cc;box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:not(:disabled):not(.disabled).active,.btn-primary:not(:disabled):not(.disabled):active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0062cc;border-color:#005cbf}.btn-primary:not(:disabled):not(.disabled).active:focus,.btn-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-secondary{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:hover{color:#fff;background-color:#5a6268;border-color:#545b62}.btn-secondary.focus,.btn-secondary:focus{color:#fff;background-color:#5a6268;border-color:#545b62;box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:not(:disabled):not(.disabled).active,.btn-secondary:not(:disabled):not(.disabled):active,.show>.btn-secondary.dropdown-toggle{color:#fff;background-color:#545b62;border-color:#4e555b}.btn-secondary:not(:disabled):not(.disabled).active:focus,.btn-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-success{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:hover{color:#fff;background-color:#218838;border-color:#1e7e34}.btn-success.focus,.btn-success:focus{color:#fff;background-color:#218838;border-color:#1e7e34;box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:not(:disabled):not(.disabled).active,.btn-success:not(:disabled):not(.disabled):active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#1e7e34;border-color:#1c7430}.btn-success:not(:disabled):not(.disabled).active:focus,.btn-success:not(:disabled):not(.disabled):active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-info{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:hover{color:#fff;background-color:#138496;border-color:#117a8b}.btn-info.focus,.btn-info:focus{color:#fff;background-color:#138496;border-color:#117a8b;box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-info.disabled,.btn-info:disabled{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:not(:disabled):not(.disabled).active,.btn-info:not(:disabled):not(.disabled):active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#117a8b;border-color:#10707f}.btn-info:not(:disabled):not(.disabled).active:focus,.btn-info:not(:disabled):not(.disabled):active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-warning{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#212529;background-color:#e0a800;border-color:#d39e00}.btn-warning.focus,.btn-warning:focus{color:#212529;background-color:#e0a800;border-color:#d39e00;box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-warning.disabled,.btn-warning:disabled{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:not(:disabled):not(.disabled).active,.btn-warning:not(:disabled):not(.disabled):active,.show>.btn-warning.dropdown-toggle{color:#212529;background-color:#d39e00;border-color:#c69500}.btn-warning:not(:disabled):not(.disabled).active:focus,.btn-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#c82333;border-color:#bd2130}.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c82333;border-color:#bd2130;box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:not(:disabled):not(.disabled).active,.btn-danger:not(:disabled):not(.disabled):active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#bd2130;border-color:#b21f2d}.btn-danger:not(:disabled):not(.disabled).active:focus,.btn-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-light{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#212529;background-color:#e2e6ea;border-color:#dae0e5}.btn-light.focus,.btn-light:focus{color:#212529;background-color:#e2e6ea;border-color:#dae0e5;box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-light.disabled,.btn-light:disabled{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:not(:disabled):not(.disabled).active,.btn-light:not(:disabled):not(.disabled):active,.show>.btn-light.dropdown-toggle{color:#212529;background-color:#dae0e5;border-color:#d3d9df}.btn-light:not(:disabled):not(.disabled).active:focus,.btn-light:not(:disabled):not(.disabled):active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark.focus,.btn-dark:focus{color:#fff;background-color:#23272b;border-color:#1d2124;box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:not(:disabled):not(.disabled).active,.btn-dark:not(:disabled):not(.disabled):active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1d2124;border-color:#171a1d}.btn-dark:not(:disabled):not(.disabled).active:focus,.btn-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-outline-primary{color:#007bff;border-color:#007bff}.btn-outline-primary:hover{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary.focus,.btn-outline-primary:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#007bff;background-color:transparent}.btn-outline-primary:not(:disabled):not(.disabled).active,.btn-outline-primary:not(:disabled):not(.disabled):active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary:not(:disabled):not(.disabled).active:focus,.btn-outline-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-secondary{color:#6c757d;border-color:#6c757d}.btn-outline-secondary:hover{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary.focus,.btn-outline-secondary:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#6c757d;background-color:transparent}.btn-outline-secondary:not(:disabled):not(.disabled).active,.btn-outline-secondary:not(:disabled):not(.disabled):active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary:not(:disabled):not(.disabled).active:focus,.btn-outline-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-success{color:#28a745;border-color:#28a745}.btn-outline-success:hover{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success.focus,.btn-outline-success:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#28a745;background-color:transparent}.btn-outline-success:not(:disabled):not(.disabled).active,.btn-outline-success:not(:disabled):not(.disabled):active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success:not(:disabled):not(.disabled).active:focus,.btn-outline-success:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-info{color:#17a2b8;border-color:#17a2b8}.btn-outline-info:hover{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info.focus,.btn-outline-info:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#17a2b8;background-color:transparent}.btn-outline-info:not(:disabled):not(.disabled).active,.btn-outline-info:not(:disabled):not(.disabled):active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info:not(:disabled):not(.disabled).active:focus,.btn-outline-info:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-warning{color:#ffc107;border-color:#ffc107}.btn-outline-warning:hover{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning.focus,.btn-outline-warning:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-warning:not(:disabled):not(.disabled).active,.btn-outline-warning:not(:disabled):not(.disabled):active,.show>.btn-outline-warning.dropdown-toggle{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning:not(:disabled):not(.disabled).active:focus,.btn-outline-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-danger{color:#dc3545;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger.focus,.btn-outline-danger:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-danger:not(:disabled):not(.disabled).active,.btn-outline-danger:not(:disabled):not(.disabled):active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger:not(:disabled):not(.disabled).active:focus,.btn-outline-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light.focus,.btn-outline-light:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light:not(:disabled):not(.disabled).active,.btn-outline-light:not(:disabled):not(.disabled):active,.show>.btn-outline-light.dropdown-toggle{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:not(:disabled):not(.disabled).active:focus,.btn-outline-light:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-dark{color:#343a40;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark.focus,.btn-outline-dark:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark:not(:disabled):not(.disabled).active,.btn-outline-dark:not(:disabled):not(.disabled):active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:not(:disabled):not(.disabled).active:focus,.btn-outline-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-link{font-weight:400;color:#007bff;text-decoration:none}.btn-link:hover{color:#0056b3;text-decoration:underline}.btn-link.focus,.btn-link:focus{text-decoration:underline}.btn-link.disabled,.btn-link:disabled{color:#6c757d;pointer-events:none}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{position:relative;height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.dropdown,.dropleft,.dropright,.dropup{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-menu-left{right:auto;left:0}.dropdown-menu-right{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-left{right:auto;left:0}.dropdown-menu-sm-right{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-left{right:auto;left:0}.dropdown-menu-md-right{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-left{right:auto;left:0}.dropdown-menu-lg-right{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-left{right:auto;left:0}.dropdown-menu-xl-right{right:0;left:auto}}.dropup .dropdown-menu{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-menu{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropright .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropright .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-toggle::after{vertical-align:0}.dropleft .dropdown-menu{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropleft .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropleft .dropdown-toggle::after{display:none}.dropleft .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropleft .dropdown-toggle:empty::after{margin-left:0}.dropleft .dropdown-toggle::before{vertical-align:0}.dropdown-menu[x-placement^=bottom],.dropdown-menu[x-placement^=left],.dropdown-menu[x-placement^=right],.dropdown-menu[x-placement^=top]{right:auto;bottom:auto}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:.25rem 1.5rem;clear:both;font-weight:400;color:#212529;text-align:inherit;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#16181b;text-decoration:none;background-color:#f8f9fa}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#007bff}.dropdown-item.disabled,.dropdown-item:disabled{color:#6c757d;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1.5rem;color:#212529}.btn-group,.btn-group-vertical{position:relative;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-ms-flex:1 1 auto;flex:1 1 auto}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:1}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:1}.btn-toolbar{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn-group:not(:first-child),.btn-group>.btn:not(:first-child){margin-left:-1px}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropright .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropleft .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{-ms-flex-direction:column;flex-direction:column;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:center;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn:not(:first-child){border-top-left-radius:0;border-top-right-radius:0}.btn-group-toggle>.btn,.btn-group-toggle>.btn-group>.btn{margin-bottom:0}.btn-group-toggle>.btn input[type=checkbox],.btn-group-toggle>.btn input[type=radio],.btn-group-toggle>.btn-group>.btn input[type=checkbox],.btn-group-toggle>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:stretch;align-items:stretch;width:100%}.input-group>.custom-file,.input-group>.custom-select,.input-group>.form-control,.input-group>.form-control-plaintext{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;min-width:0;margin-bottom:0}.input-group>.custom-file+.custom-file,.input-group>.custom-file+.custom-select,.input-group>.custom-file+.form-control,.input-group>.custom-select+.custom-file,.input-group>.custom-select+.custom-select,.input-group>.custom-select+.form-control,.input-group>.form-control+.custom-file,.input-group>.form-control+.custom-select,.input-group>.form-control+.form-control,.input-group>.form-control-plaintext+.custom-file,.input-group>.form-control-plaintext+.custom-select,.input-group>.form-control-plaintext+.form-control{margin-left:-1px}.input-group>.custom-file .custom-file-input:focus~.custom-file-label,.input-group>.custom-select:focus,.input-group>.form-control:focus{z-index:3}.input-group>.custom-file .custom-file-input:focus{z-index:4}.input-group>.custom-select:not(:last-child),.input-group>.form-control:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-select:not(:first-child),.input-group>.form-control:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.custom-file{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center}.input-group>.custom-file:not(:last-child) .custom-file-label,.input-group>.custom-file:not(:last-child) .custom-file-label::after{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-file:not(:first-child) .custom-file-label{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-append,.input-group-prepend{display:-ms-flexbox;display:flex}.input-group-append .btn,.input-group-prepend .btn{position:relative;z-index:2}.input-group-append .btn:focus,.input-group-prepend .btn:focus{z-index:3}.input-group-append .btn+.btn,.input-group-append .btn+.input-group-text,.input-group-append .input-group-text+.btn,.input-group-append .input-group-text+.input-group-text,.input-group-prepend .btn+.btn,.input-group-prepend .btn+.input-group-text,.input-group-prepend .input-group-text+.btn,.input-group-prepend .input-group-text+.input-group-text{margin-left:-1px}.input-group-prepend{margin-right:-1px}.input-group-append{margin-left:-1px}.input-group-text{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.375rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-text input[type=checkbox],.input-group-text input[type=radio]{margin-top:0}.input-group-lg>.custom-select,.input-group-lg>.form-control:not(textarea){height:calc(1.5em + 1rem + 2px)}.input-group-lg>.custom-select,.input-group-lg>.form-control,.input-group-lg>.input-group-append>.btn,.input-group-lg>.input-group-append>.input-group-text,.input-group-lg>.input-group-prepend>.btn,.input-group-lg>.input-group-prepend>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.input-group-sm>.custom-select,.input-group-sm>.form-control:not(textarea){height:calc(1.5em + .5rem + 2px)}.input-group-sm>.custom-select,.input-group-sm>.form-control,.input-group-sm>.input-group-append>.btn,.input-group-sm>.input-group-append>.input-group-text,.input-group-sm>.input-group-prepend>.btn,.input-group-sm>.input-group-prepend>.input-group-text{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.input-group-lg>.custom-select,.input-group-sm>.custom-select{padding-right:1.75rem}.input-group>.input-group-append:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group>.input-group-append:last-child>.input-group-text:not(:last-child),.input-group>.input-group-append:not(:last-child)>.btn,.input-group>.input-group-append:not(:last-child)>.input-group-text,.input-group>.input-group-prepend>.btn,.input-group>.input-group-prepend>.input-group-text{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.input-group-append>.btn,.input-group>.input-group-append>.input-group-text,.input-group>.input-group-prepend:first-child>.btn:not(:first-child),.input-group>.input-group-prepend:first-child>.input-group-text:not(:first-child),.input-group>.input-group-prepend:not(:first-child)>.btn,.input-group>.input-group-prepend:not(:first-child)>.input-group-text{border-top-left-radius:0;border-bottom-left-radius:0}.custom-control{position:relative;display:block;min-height:1.5rem;padding-left:1.5rem}.custom-control-inline{display:-ms-inline-flexbox;display:inline-flex;margin-right:1rem}.custom-control-input{position:absolute;left:0;z-index:-1;width:1rem;height:1.25rem;opacity:0}.custom-control-input:checked~.custom-control-label::before{color:#fff;border-color:#007bff;background-color:#007bff}.custom-control-input:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-control-input:focus:not(:checked)~.custom-control-label::before{border-color:#80bdff}.custom-control-input:not(:disabled):active~.custom-control-label::before{color:#fff;background-color:#b3d7ff;border-color:#b3d7ff}.custom-control-input:disabled~.custom-control-label,.custom-control-input[disabled]~.custom-control-label{color:#6c757d}.custom-control-input:disabled~.custom-control-label::before,.custom-control-input[disabled]~.custom-control-label::before{background-color:#e9ecef}.custom-control-label{position:relative;margin-bottom:0;vertical-align:top}.custom-control-label::before{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;pointer-events:none;content:"";background-color:#fff;border:#adb5bd solid 1px}.custom-control-label::after{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;content:"";background:no-repeat 50%/50% 50%}.custom-checkbox .custom-control-label::before{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26l2.974 2.99L8 2.193z'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::before{border-color:#007bff;background-color:#007bff}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='4' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-checkbox .custom-control-input:disabled:indeterminate~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-radio .custom-control-label::before{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.custom-radio .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-switch{padding-left:2.25rem}.custom-switch .custom-control-label::before{left:-2.25rem;width:1.75rem;pointer-events:all;border-radius:.5rem}.custom-switch .custom-control-label::after{top:calc(.25rem + 2px);left:calc(-2.25rem + 2px);width:calc(1rem - 4px);height:calc(1rem - 4px);background-color:#adb5bd;border-radius:.5rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-switch .custom-control-label::after{transition:none}}.custom-switch .custom-control-input:checked~.custom-control-label::after{background-color:#fff;-webkit-transform:translateX(.75rem);transform:translateX(.75rem)}.custom-switch .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-select{display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem 1.75rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;vertical-align:middle;background:#fff url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px;border:1px solid #ced4da;border-radius:.25rem;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-select:focus{border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select[multiple],.custom-select[size]:not([size="1"]){height:auto;padding-right:.75rem;background-image:none}.custom-select:disabled{color:#6c757d;background-color:#e9ecef}.custom-select::-ms-expand{display:none}.custom-select:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.custom-select-sm{height:calc(1.5em + .5rem + 2px);padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem}.custom-select-lg{height:calc(1.5em + 1rem + 2px);padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}.custom-file{position:relative;display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);margin-bottom:0}.custom-file-input{position:relative;z-index:2;width:100%;height:calc(1.5em + .75rem + 2px);margin:0;opacity:0}.custom-file-input:focus~.custom-file-label{border-color:#80bdff;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-file-input:disabled~.custom-file-label,.custom-file-input[disabled]~.custom-file-label{background-color:#e9ecef}.custom-file-input:lang(en)~.custom-file-label::after{content:"Browse"}.custom-file-input~.custom-file-label[data-browse]::after{content:attr(data-browse)}.custom-file-label{position:absolute;top:0;right:0;left:0;z-index:1;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem}.custom-file-label::after{position:absolute;top:0;right:0;bottom:0;z-index:3;display:block;height:calc(1.5em + .75rem);padding:.375rem .75rem;line-height:1.5;color:#495057;content:"Browse";background-color:#e9ecef;border-left:inherit;border-radius:0 .25rem .25rem 0}.custom-range{width:100%;height:1.4rem;padding:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-range:focus{outline:0}.custom-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-ms-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range::-moz-focus-outer{border:0}.custom-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#007bff;border:0;border-radius:1rem;-webkit-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.custom-range::-webkit-slider-thumb:active{background-color:#b3d7ff}.custom-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#007bff;border:0;border-radius:1rem;-moz-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-moz-range-thumb{-moz-transition:none;transition:none}}.custom-range::-moz-range-thumb:active{background-color:#b3d7ff}.custom-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-ms-thumb{width:1rem;height:1rem;margin-top:0;margin-right:.2rem;margin-left:.2rem;background-color:#007bff;border:0;border-radius:1rem;-ms-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-ms-thumb{-ms-transition:none;transition:none}}.custom-range::-ms-thumb:active{background-color:#b3d7ff}.custom-range::-ms-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:transparent;border-color:transparent;border-width:.5rem}.custom-range::-ms-fill-lower{background-color:#dee2e6;border-radius:1rem}.custom-range::-ms-fill-upper{margin-right:15px;background-color:#dee2e6;border-radius:1rem}.custom-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.custom-range:disabled::-webkit-slider-runnable-track{cursor:default}.custom-range:disabled::-moz-range-thumb{background-color:#adb5bd}.custom-range:disabled::-moz-range-track{cursor:default}.custom-range:disabled::-ms-thumb{background-color:#adb5bd}.custom-control-label::before,.custom-file-label,.custom-select{transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-control-label::before,.custom-file-label,.custom-select{transition:none}}.nav{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #dee2e6}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#007bff}.nav-fill .nav-item{-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between;padding:.5rem 1rem}.navbar .container,.navbar .container-fluid,.navbar .container-lg,.navbar .container-md,.navbar .container-sm,.navbar .container-xl{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{-ms-flex-preferred-size:100%;flex-basis:100%;-ms-flex-positive:1;flex-grow:1;-ms-flex-align:center;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width:575.98px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-expand-sm{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-sm .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width:767.98px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-expand-md{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-md .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width:991.98px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-expand-lg{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-lg .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width:1199.98px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-expand-xl{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-xl .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.5);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%280, 0, 0, 0.5%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-light .navbar-text a{color:rgba(0,0,0,.9)}.navbar-light .navbar-text a:focus,.navbar-light .navbar-text a:hover{color:rgba(0,0,0,.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.5);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.5%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,.5)}.navbar-dark .navbar-text a{color:#fff}.navbar-dark .navbar-text a:focus,.navbar-dark .navbar-text a:hover{color:#fff}.card{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card>.list-group:last-child{border-bottom-width:0;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-body{-ms-flex:1 1 auto;flex:1 1 auto;min-height:1px;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-header+.list-group .list-group-item:first-child{border-top:0}.card-footer{padding:.75rem 1.25rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img,.card-img-bottom,.card-img-top{-ms-flex-negative:0;flex-shrink:0;width:100%}.card-img,.card-img-top{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-deck .card{margin-bottom:15px}@media (min-width:576px){.card-deck{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{-ms-flex:1 0 0%;flex:1 0 0%;margin-right:15px;margin-bottom:0;margin-left:15px}}.card-group>.card{margin-bottom:15px}@media (min-width:576px){.card-group{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group>.card{-ms-flex:1 0 0%;flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.card-columns .card{margin-bottom:.75rem}@media (min-width:576px){.card-columns{-webkit-column-count:3;-moz-column-count:3;column-count:3;-webkit-column-gap:1.25rem;-moz-column-gap:1.25rem;column-gap:1.25rem;orphans:1;widows:1}.card-columns .card{display:inline-block;width:100%}}.accordion>.card{overflow:hidden}.accordion>.card:not(:last-of-type){border-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.accordion>.card:not(:first-of-type){border-top-left-radius:0;border-top-right-radius:0}.accordion>.card>.card-header{border-radius:0;margin-bottom:-1px}.breadcrumb{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:.25rem}.breadcrumb-item{display:-ms-flexbox;display:flex}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;color:#6c757d;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#6c757d}.pagination{display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#007bff;background-color:#fff;border:1px solid #dee2e6}.page-link:hover{z-index:2;color:#0056b3;text-decoration:none;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:3;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.page-item.active .page-link{z-index:3;color:#fff;background-color:#007bff;border-color:#007bff}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;cursor:auto;background-color:#fff;border-color:#dee2e6}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.badge{transition:none}}a.badge:focus,a.badge:hover{text-decoration:none}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#007bff}a.badge-primary:focus,a.badge-primary:hover{color:#fff;background-color:#0062cc}a.badge-primary.focus,a.badge-primary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.badge-secondary{color:#fff;background-color:#6c757d}a.badge-secondary:focus,a.badge-secondary:hover{color:#fff;background-color:#545b62}a.badge-secondary.focus,a.badge-secondary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.badge-success{color:#fff;background-color:#28a745}a.badge-success:focus,a.badge-success:hover{color:#fff;background-color:#1e7e34}a.badge-success.focus,a.badge-success:focus{outline:0;box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.badge-info{color:#fff;background-color:#17a2b8}a.badge-info:focus,a.badge-info:hover{color:#fff;background-color:#117a8b}a.badge-info.focus,a.badge-info:focus{outline:0;box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.badge-warning{color:#212529;background-color:#ffc107}a.badge-warning:focus,a.badge-warning:hover{color:#212529;background-color:#d39e00}a.badge-warning.focus,a.badge-warning:focus{outline:0;box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.badge-danger{color:#fff;background-color:#dc3545}a.badge-danger:focus,a.badge-danger:hover{color:#fff;background-color:#bd2130}a.badge-danger.focus,a.badge-danger:focus{outline:0;box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.badge-light{color:#212529;background-color:#f8f9fa}a.badge-light:focus,a.badge-light:hover{color:#212529;background-color:#dae0e5}a.badge-light.focus,a.badge-light:focus{outline:0;box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.badge-dark{color:#fff;background-color:#343a40}a.badge-dark:focus,a.badge-dark:hover{color:#fff;background-color:#1d2124}a.badge-dark.focus,a.badge-dark:focus{outline:0;box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{position:relative;padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:4rem}.alert-dismissible .close{position:absolute;top:0;right:0;padding:.75rem 1.25rem;color:inherit}.alert-primary{color:#004085;background-color:#cce5ff;border-color:#b8daff}.alert-primary hr{border-top-color:#9fcdff}.alert-primary .alert-link{color:#002752}.alert-secondary{color:#383d41;background-color:#e2e3e5;border-color:#d6d8db}.alert-secondary hr{border-top-color:#c8cbcf}.alert-secondary .alert-link{color:#202326}.alert-success{color:#155724;background-color:#d4edda;border-color:#c3e6cb}.alert-success hr{border-top-color:#b1dfbb}.alert-success .alert-link{color:#0b2e13}.alert-info{color:#0c5460;background-color:#d1ecf1;border-color:#bee5eb}.alert-info hr{border-top-color:#abdde5}.alert-info .alert-link{color:#062c33}.alert-warning{color:#856404;background-color:#fff3cd;border-color:#ffeeba}.alert-warning hr{border-top-color:#ffe8a1}.alert-warning .alert-link{color:#533f03}.alert-danger{color:#721c24;background-color:#f8d7da;border-color:#f5c6cb}.alert-danger hr{border-top-color:#f1b0b7}.alert-danger .alert-link{color:#491217}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-ms-flexbox;display:flex;height:1rem;overflow:hidden;line-height:0;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;overflow:hidden;color:#fff;text-align:center;white-space:nowrap;background-color:#007bff;transition:width .6s ease}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}@media (prefers-reduced-motion:reduce){.progress-bar-animated{-webkit-animation:none;animation:none}}.media{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start}.media-body{-ms-flex:1;flex:1}.list-group{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;border-radius:.25rem}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.75rem 1.25rem;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.list-group-item:last-child{border-bottom-right-radius:inherit;border-bottom-left-radius:inherit}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#007bff;border-color:#007bff}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:-1px;border-top-width:1px}.list-group-horizontal{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}@media (min-width:576px){.list-group-horizontal-sm{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-sm>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:768px){.list-group-horizontal-md{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-md>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:992px){.list-group-horizontal-lg{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-lg>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1200px){.list-group-horizontal-xl{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-xl>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}.list-group-flush{border-radius:0}.list-group-flush>.list-group-item{border-width:0 0 1px}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{color:#004085;background-color:#b8daff}.list-group-item-primary.list-group-item-action:focus,.list-group-item-primary.list-group-item-action:hover{color:#004085;background-color:#9fcdff}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#004085;border-color:#004085}.list-group-item-secondary{color:#383d41;background-color:#d6d8db}.list-group-item-secondary.list-group-item-action:focus,.list-group-item-secondary.list-group-item-action:hover{color:#383d41;background-color:#c8cbcf}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#383d41;border-color:#383d41}.list-group-item-success{color:#155724;background-color:#c3e6cb}.list-group-item-success.list-group-item-action:focus,.list-group-item-success.list-group-item-action:hover{color:#155724;background-color:#b1dfbb}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#155724;border-color:#155724}.list-group-item-info{color:#0c5460;background-color:#bee5eb}.list-group-item-info.list-group-item-action:focus,.list-group-item-info.list-group-item-action:hover{color:#0c5460;background-color:#abdde5}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#0c5460;border-color:#0c5460}.list-group-item-warning{color:#856404;background-color:#ffeeba}.list-group-item-warning.list-group-item-action:focus,.list-group-item-warning.list-group-item-action:hover{color:#856404;background-color:#ffe8a1}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#856404;border-color:#856404}.list-group-item-danger{color:#721c24;background-color:#f5c6cb}.list-group-item-danger.list-group-item-action:focus,.list-group-item-danger.list-group-item-action:hover{color:#721c24;background-color:#f1b0b7}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#721c24;border-color:#721c24}.list-group-item-light{color:#818182;background-color:#fdfdfe}.list-group-item-light.list-group-item-action:focus,.list-group-item-light.list-group-item-action:hover{color:#818182;background-color:#ececf6}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}.list-group-item-dark.list-group-item-action:focus,.list-group-item-dark.list-group-item-action:hover{color:#1b1e21;background-color:#b9bbbe}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:hover{color:#000;text-decoration:none}.close:not(:disabled):not(.disabled):focus,.close:not(:disabled):not(.disabled):hover{opacity:.75}button.close{padding:0;background-color:transparent;border:0}a.close.disabled{pointer-events:none}.toast{max-width:350px;overflow:hidden;font-size:.875rem;background-color:rgba(255,255,255,.85);background-clip:padding-box;border:1px solid rgba(0,0,0,.1);box-shadow:0 .25rem .75rem rgba(0,0,0,.1);-webkit-backdrop-filter:blur(10px);backdrop-filter:blur(10px);opacity:0;border-radius:.25rem}.toast:not(:last-child){margin-bottom:.75rem}.toast.showing{opacity:1}.toast.show{display:block;opacity:1}.toast.hide{display:none}.toast-header{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.25rem .75rem;color:#6c757d;background-color:rgba(255,255,255,.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,.05)}.toast-body{padding:.75rem}.modal-open{overflow:hidden}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal{position:fixed;top:0;left:0;z-index:1050;display:none;width:100%;height:100%;overflow:hidden;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:-webkit-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out;-webkit-transform:translate(0,-50px);transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{-webkit-transform:none;transform:none}.modal.modal-static .modal-dialog{-webkit-transform:scale(1.02);transform:scale(1.02)}.modal-dialog-scrollable{display:-ms-flexbox;display:flex;max-height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 1rem);overflow:hidden}.modal-dialog-scrollable .modal-footer,.modal-dialog-scrollable .modal-header{-ms-flex-negative:0;flex-shrink:0}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;min-height:calc(100% - 1rem)}.modal-dialog-centered::before{display:block;height:calc(100vh - 1rem);height:-webkit-min-content;height:-moz-min-content;height:min-content;content:""}.modal-dialog-centered.modal-dialog-scrollable{-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;height:100%}.modal-dialog-centered.modal-dialog-scrollable .modal-content{max-height:none}.modal-dialog-centered.modal-dialog-scrollable::before{content:none}.modal-content{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:justify;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.modal-header .close{padding:1rem 1rem;margin:-1rem -1rem -1rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;padding:1rem}.modal-footer{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:end;justify-content:flex-end;padding:.75rem;border-top:1px solid #dee2e6;border-bottom-right-radius:calc(.3rem - 1px);border-bottom-left-radius:calc(.3rem - 1px)}.modal-footer>*{margin:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{max-height:calc(100% - 3.5rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-dialog-centered::before{height:calc(100vh - 3.5rem);height:-webkit-min-content;height:-moz-min-content;height:min-content}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width:1200px){.modal-xl{max-width:1140px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[x-placement^=top],.bs-tooltip-top{padding:.4rem 0}.bs-tooltip-auto[x-placement^=top] .arrow,.bs-tooltip-top .arrow{bottom:0}.bs-tooltip-auto[x-placement^=top] .arrow::before,.bs-tooltip-top .arrow::before{top:0;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-auto[x-placement^=right],.bs-tooltip-right{padding:0 .4rem}.bs-tooltip-auto[x-placement^=right] .arrow,.bs-tooltip-right .arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=right] .arrow::before,.bs-tooltip-right .arrow::before{right:0;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-auto[x-placement^=bottom],.bs-tooltip-bottom{padding:.4rem 0}.bs-tooltip-auto[x-placement^=bottom] .arrow,.bs-tooltip-bottom .arrow{top:0}.bs-tooltip-auto[x-placement^=bottom] .arrow::before,.bs-tooltip-bottom .arrow::before{bottom:0;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-auto[x-placement^=left],.bs-tooltip-left{padding:0 .4rem}.bs-tooltip-auto[x-placement^=left] .arrow,.bs-tooltip-left .arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=left] .arrow::before,.bs-tooltip-left .arrow::before{left:0;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .arrow{position:absolute;display:block;width:1rem;height:.5rem;margin:0 .3rem}.popover .arrow::after,.popover .arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-auto[x-placement^=top],.bs-popover-top{margin-bottom:.5rem}.bs-popover-auto[x-placement^=top]>.arrow,.bs-popover-top>.arrow{bottom:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=top]>.arrow::before,.bs-popover-top>.arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=top]>.arrow::after,.bs-popover-top>.arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-auto[x-placement^=right],.bs-popover-right{margin-left:.5rem}.bs-popover-auto[x-placement^=right]>.arrow,.bs-popover-right>.arrow{left:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=right]>.arrow::before,.bs-popover-right>.arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=right]>.arrow::after,.bs-popover-right>.arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-auto[x-placement^=bottom],.bs-popover-bottom{margin-top:.5rem}.bs-popover-auto[x-placement^=bottom]>.arrow,.bs-popover-bottom>.arrow{top:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=bottom]>.arrow::before,.bs-popover-bottom>.arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=bottom]>.arrow::after,.bs-popover-bottom>.arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-auto[x-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f7f7f7}.bs-popover-auto[x-placement^=left],.bs-popover-left{margin-right:.5rem}.bs-popover-auto[x-placement^=left]>.arrow,.bs-popover-left>.arrow{right:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=left]>.arrow::before,.bs-popover-left>.arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=left]>.arrow::after,.bs-popover-left>.arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:.5rem .75rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{-ms-touch-action:pan-y;touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:-webkit-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-right,.carousel-item-next:not(.carousel-item-left){-webkit-transform:translateX(100%);transform:translateX(100%)}.active.carousel-item-left,.carousel-item-prev:not(.carousel-item-right){-webkit-transform:translateX(-100%);transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;-webkit-transform:none;transform:none}.carousel-fade .carousel-item-next.carousel-item-left,.carousel-fade .carousel-item-prev.carousel-item-right,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:no-repeat 50%/100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5L4.25 4l2.5-2.5L5.25 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5L3.75 4l-2.5 2.5L2.75 8l4-4-4-4z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:15;display:-ms-flexbox;display:flex;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{box-sizing:content-box;-ms-flex:0 1 auto;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators li{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}@-webkit-keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;-webkit-animation:spinner-border .75s linear infinite;animation:spinner-border .75s linear infinite}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@-webkit-keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1;-webkit-transform:none;transform:none}}@keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1;-webkit-transform:none;transform:none}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;background-color:currentColor;border-radius:50%;opacity:0;-webkit-animation:spinner-grow .75s linear infinite;animation:spinner-grow .75s linear infinite}.spinner-grow-sm{width:1rem;height:1rem}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-primary{background-color:#007bff!important}a.bg-primary:focus,a.bg-primary:hover,button.bg-primary:focus,button.bg-primary:hover{background-color:#0062cc!important}.bg-secondary{background-color:#6c757d!important}a.bg-secondary:focus,a.bg-secondary:hover,button.bg-secondary:focus,button.bg-secondary:hover{background-color:#545b62!important}.bg-success{background-color:#28a745!important}a.bg-success:focus,a.bg-success:hover,button.bg-success:focus,button.bg-success:hover{background-color:#1e7e34!important}.bg-info{background-color:#17a2b8!important}a.bg-info:focus,a.bg-info:hover,button.bg-info:focus,button.bg-info:hover{background-color:#117a8b!important}.bg-warning{background-color:#ffc107!important}a.bg-warning:focus,a.bg-warning:hover,button.bg-warning:focus,button.bg-warning:hover{background-color:#d39e00!important}.bg-danger{background-color:#dc3545!important}a.bg-danger:focus,a.bg-danger:hover,button.bg-danger:focus,button.bg-danger:hover{background-color:#bd2130!important}.bg-light{background-color:#f8f9fa!important}a.bg-light:focus,a.bg-light:hover,button.bg-light:focus,button.bg-light:hover{background-color:#dae0e5!important}.bg-dark{background-color:#343a40!important}a.bg-dark:focus,a.bg-dark:hover,button.bg-dark:focus,button.bg-dark:hover{background-color:#1d2124!important}.bg-white{background-color:#fff!important}.bg-transparent{background-color:transparent!important}.border{border:1px solid #dee2e6!important}.border-top{border-top:1px solid #dee2e6!important}.border-right{border-right:1px solid #dee2e6!important}.border-bottom{border-bottom:1px solid #dee2e6!important}.border-left{border-left:1px solid #dee2e6!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.border-primary{border-color:#007bff!important}.border-secondary{border-color:#6c757d!important}.border-success{border-color:#28a745!important}.border-info{border-color:#17a2b8!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#343a40!important}.border-white{border-color:#fff!important}.rounded-sm{border-radius:.2rem!important}.rounded{border-radius:.25rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-right{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-left{border-top-left-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-lg{border-radius:.3rem!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:50rem!important}.rounded-0{border-radius:0!important}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media print{.d-print-none{display:none!important}.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:-ms-flexbox!important;display:flex!important}.d-print-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-sm-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-sm-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-sm-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-sm-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-sm-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-sm-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-sm-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-sm-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-md-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-md-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-md-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-md-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-md-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-md-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-md-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-md-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-lg-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-lg-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-lg-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-lg-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-lg-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-lg-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-lg-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-lg-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-xl-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-xl-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-xl-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-xl-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-xl-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-xl-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-xl-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-xl-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.user-select-all{-webkit-user-select:all!important;-moz-user-select:all!important;-ms-user-select:all!important;user-select:all!important}.user-select-auto{-webkit-user-select:auto!important;-moz-user-select:auto!important;-ms-user-select:auto!important;user-select:auto!important}.user-select-none{-webkit-user-select:none!important;-moz-user-select:none!important;-ms-user-select:none!important;user-select:none!important}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports ((position:-webkit-sticky) or (position:sticky)){.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075)!important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15)!important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175)!important}.shadow-none{box-shadow:none!important}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.min-vw-100{min-width:100vw!important}.min-vh-100{min-height:100vh!important}.vw-100{width:100vw!important}.vh-100{height:100vh!important}.m-0{margin:0!important}.mt-0,.my-0{margin-top:0!important}.mr-0,.mx-0{margin-right:0!important}.mb-0,.my-0{margin-bottom:0!important}.ml-0,.mx-0{margin-left:0!important}.m-1{margin:.25rem!important}.mt-1,.my-1{margin-top:.25rem!important}.mr-1,.mx-1{margin-right:.25rem!important}.mb-1,.my-1{margin-bottom:.25rem!important}.ml-1,.mx-1{margin-left:.25rem!important}.m-2{margin:.5rem!important}.mt-2,.my-2{margin-top:.5rem!important}.mr-2,.mx-2{margin-right:.5rem!important}.mb-2,.my-2{margin-bottom:.5rem!important}.ml-2,.mx-2{margin-left:.5rem!important}.m-3{margin:1rem!important}.mt-3,.my-3{margin-top:1rem!important}.mr-3,.mx-3{margin-right:1rem!important}.mb-3,.my-3{margin-bottom:1rem!important}.ml-3,.mx-3{margin-left:1rem!important}.m-4{margin:1.5rem!important}.mt-4,.my-4{margin-top:1.5rem!important}.mr-4,.mx-4{margin-right:1.5rem!important}.mb-4,.my-4{margin-bottom:1.5rem!important}.ml-4,.mx-4{margin-left:1.5rem!important}.m-5{margin:3rem!important}.mt-5,.my-5{margin-top:3rem!important}.mr-5,.mx-5{margin-right:3rem!important}.mb-5,.my-5{margin-bottom:3rem!important}.ml-5,.mx-5{margin-left:3rem!important}.p-0{padding:0!important}.pt-0,.py-0{padding-top:0!important}.pr-0,.px-0{padding-right:0!important}.pb-0,.py-0{padding-bottom:0!important}.pl-0,.px-0{padding-left:0!important}.p-1{padding:.25rem!important}.pt-1,.py-1{padding-top:.25rem!important}.pr-1,.px-1{padding-right:.25rem!important}.pb-1,.py-1{padding-bottom:.25rem!important}.pl-1,.px-1{padding-left:.25rem!important}.p-2{padding:.5rem!important}.pt-2,.py-2{padding-top:.5rem!important}.pr-2,.px-2{padding-right:.5rem!important}.pb-2,.py-2{padding-bottom:.5rem!important}.pl-2,.px-2{padding-left:.5rem!important}.p-3{padding:1rem!important}.pt-3,.py-3{padding-top:1rem!important}.pr-3,.px-3{padding-right:1rem!important}.pb-3,.py-3{padding-bottom:1rem!important}.pl-3,.px-3{padding-left:1rem!important}.p-4{padding:1.5rem!important}.pt-4,.py-4{padding-top:1.5rem!important}.pr-4,.px-4{padding-right:1.5rem!important}.pb-4,.py-4{padding-bottom:1.5rem!important}.pl-4,.px-4{padding-left:1.5rem!important}.p-5{padding:3rem!important}.pt-5,.py-5{padding-top:3rem!important}.pr-5,.px-5{padding-right:3rem!important}.pb-5,.py-5{padding-bottom:3rem!important}.pl-5,.px-5{padding-left:3rem!important}.m-n1{margin:-.25rem!important}.mt-n1,.my-n1{margin-top:-.25rem!important}.mr-n1,.mx-n1{margin-right:-.25rem!important}.mb-n1,.my-n1{margin-bottom:-.25rem!important}.ml-n1,.mx-n1{margin-left:-.25rem!important}.m-n2{margin:-.5rem!important}.mt-n2,.my-n2{margin-top:-.5rem!important}.mr-n2,.mx-n2{margin-right:-.5rem!important}.mb-n2,.my-n2{margin-bottom:-.5rem!important}.ml-n2,.mx-n2{margin-left:-.5rem!important}.m-n3{margin:-1rem!important}.mt-n3,.my-n3{margin-top:-1rem!important}.mr-n3,.mx-n3{margin-right:-1rem!important}.mb-n3,.my-n3{margin-bottom:-1rem!important}.ml-n3,.mx-n3{margin-left:-1rem!important}.m-n4{margin:-1.5rem!important}.mt-n4,.my-n4{margin-top:-1.5rem!important}.mr-n4,.mx-n4{margin-right:-1.5rem!important}.mb-n4,.my-n4{margin-bottom:-1.5rem!important}.ml-n4,.mx-n4{margin-left:-1.5rem!important}.m-n5{margin:-3rem!important}.mt-n5,.my-n5{margin-top:-3rem!important}.mr-n5,.mx-n5{margin-right:-3rem!important}.mb-n5,.my-n5{margin-bottom:-3rem!important}.ml-n5,.mx-n5{margin-left:-3rem!important}.m-auto{margin:auto!important}.mt-auto,.my-auto{margin-top:auto!important}.mr-auto,.mx-auto{margin-right:auto!important}.mb-auto,.my-auto{margin-bottom:auto!important}.ml-auto,.mx-auto{margin-left:auto!important}@media (min-width:576px){.m-sm-0{margin:0!important}.mt-sm-0,.my-sm-0{margin-top:0!important}.mr-sm-0,.mx-sm-0{margin-right:0!important}.mb-sm-0,.my-sm-0{margin-bottom:0!important}.ml-sm-0,.mx-sm-0{margin-left:0!important}.m-sm-1{margin:.25rem!important}.mt-sm-1,.my-sm-1{margin-top:.25rem!important}.mr-sm-1,.mx-sm-1{margin-right:.25rem!important}.mb-sm-1,.my-sm-1{margin-bottom:.25rem!important}.ml-sm-1,.mx-sm-1{margin-left:.25rem!important}.m-sm-2{margin:.5rem!important}.mt-sm-2,.my-sm-2{margin-top:.5rem!important}.mr-sm-2,.mx-sm-2{margin-right:.5rem!important}.mb-sm-2,.my-sm-2{margin-bottom:.5rem!important}.ml-sm-2,.mx-sm-2{margin-left:.5rem!important}.m-sm-3{margin:1rem!important}.mt-sm-3,.my-sm-3{margin-top:1rem!important}.mr-sm-3,.mx-sm-3{margin-right:1rem!important}.mb-sm-3,.my-sm-3{margin-bottom:1rem!important}.ml-sm-3,.mx-sm-3{margin-left:1rem!important}.m-sm-4{margin:1.5rem!important}.mt-sm-4,.my-sm-4{margin-top:1.5rem!important}.mr-sm-4,.mx-sm-4{margin-right:1.5rem!important}.mb-sm-4,.my-sm-4{margin-bottom:1.5rem!important}.ml-sm-4,.mx-sm-4{margin-left:1.5rem!important}.m-sm-5{margin:3rem!important}.mt-sm-5,.my-sm-5{margin-top:3rem!important}.mr-sm-5,.mx-sm-5{margin-right:3rem!important}.mb-sm-5,.my-sm-5{margin-bottom:3rem!important}.ml-sm-5,.mx-sm-5{margin-left:3rem!important}.p-sm-0{padding:0!important}.pt-sm-0,.py-sm-0{padding-top:0!important}.pr-sm-0,.px-sm-0{padding-right:0!important}.pb-sm-0,.py-sm-0{padding-bottom:0!important}.pl-sm-0,.px-sm-0{padding-left:0!important}.p-sm-1{padding:.25rem!important}.pt-sm-1,.py-sm-1{padding-top:.25rem!important}.pr-sm-1,.px-sm-1{padding-right:.25rem!important}.pb-sm-1,.py-sm-1{padding-bottom:.25rem!important}.pl-sm-1,.px-sm-1{padding-left:.25rem!important}.p-sm-2{padding:.5rem!important}.pt-sm-2,.py-sm-2{padding-top:.5rem!important}.pr-sm-2,.px-sm-2{padding-right:.5rem!important}.pb-sm-2,.py-sm-2{padding-bottom:.5rem!important}.pl-sm-2,.px-sm-2{padding-left:.5rem!important}.p-sm-3{padding:1rem!important}.pt-sm-3,.py-sm-3{padding-top:1rem!important}.pr-sm-3,.px-sm-3{padding-right:1rem!important}.pb-sm-3,.py-sm-3{padding-bottom:1rem!important}.pl-sm-3,.px-sm-3{padding-left:1rem!important}.p-sm-4{padding:1.5rem!important}.pt-sm-4,.py-sm-4{padding-top:1.5rem!important}.pr-sm-4,.px-sm-4{padding-right:1.5rem!important}.pb-sm-4,.py-sm-4{padding-bottom:1.5rem!important}.pl-sm-4,.px-sm-4{padding-left:1.5rem!important}.p-sm-5{padding:3rem!important}.pt-sm-5,.py-sm-5{padding-top:3rem!important}.pr-sm-5,.px-sm-5{padding-right:3rem!important}.pb-sm-5,.py-sm-5{padding-bottom:3rem!important}.pl-sm-5,.px-sm-5{padding-left:3rem!important}.m-sm-n1{margin:-.25rem!important}.mt-sm-n1,.my-sm-n1{margin-top:-.25rem!important}.mr-sm-n1,.mx-sm-n1{margin-right:-.25rem!important}.mb-sm-n1,.my-sm-n1{margin-bottom:-.25rem!important}.ml-sm-n1,.mx-sm-n1{margin-left:-.25rem!important}.m-sm-n2{margin:-.5rem!important}.mt-sm-n2,.my-sm-n2{margin-top:-.5rem!important}.mr-sm-n2,.mx-sm-n2{margin-right:-.5rem!important}.mb-sm-n2,.my-sm-n2{margin-bottom:-.5rem!important}.ml-sm-n2,.mx-sm-n2{margin-left:-.5rem!important}.m-sm-n3{margin:-1rem!important}.mt-sm-n3,.my-sm-n3{margin-top:-1rem!important}.mr-sm-n3,.mx-sm-n3{margin-right:-1rem!important}.mb-sm-n3,.my-sm-n3{margin-bottom:-1rem!important}.ml-sm-n3,.mx-sm-n3{margin-left:-1rem!important}.m-sm-n4{margin:-1.5rem!important}.mt-sm-n4,.my-sm-n4{margin-top:-1.5rem!important}.mr-sm-n4,.mx-sm-n4{margin-right:-1.5rem!important}.mb-sm-n4,.my-sm-n4{margin-bottom:-1.5rem!important}.ml-sm-n4,.mx-sm-n4{margin-left:-1.5rem!important}.m-sm-n5{margin:-3rem!important}.mt-sm-n5,.my-sm-n5{margin-top:-3rem!important}.mr-sm-n5,.mx-sm-n5{margin-right:-3rem!important}.mb-sm-n5,.my-sm-n5{margin-bottom:-3rem!important}.ml-sm-n5,.mx-sm-n5{margin-left:-3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto,.my-sm-auto{margin-top:auto!important}.mr-sm-auto,.mx-sm-auto{margin-right:auto!important}.mb-sm-auto,.my-sm-auto{margin-bottom:auto!important}.ml-sm-auto,.mx-sm-auto{margin-left:auto!important}}@media (min-width:768px){.m-md-0{margin:0!important}.mt-md-0,.my-md-0{margin-top:0!important}.mr-md-0,.mx-md-0{margin-right:0!important}.mb-md-0,.my-md-0{margin-bottom:0!important}.ml-md-0,.mx-md-0{margin-left:0!important}.m-md-1{margin:.25rem!important}.mt-md-1,.my-md-1{margin-top:.25rem!important}.mr-md-1,.mx-md-1{margin-right:.25rem!important}.mb-md-1,.my-md-1{margin-bottom:.25rem!important}.ml-md-1,.mx-md-1{margin-left:.25rem!important}.m-md-2{margin:.5rem!important}.mt-md-2,.my-md-2{margin-top:.5rem!important}.mr-md-2,.mx-md-2{margin-right:.5rem!important}.mb-md-2,.my-md-2{margin-bottom:.5rem!important}.ml-md-2,.mx-md-2{margin-left:.5rem!important}.m-md-3{margin:1rem!important}.mt-md-3,.my-md-3{margin-top:1rem!important}.mr-md-3,.mx-md-3{margin-right:1rem!important}.mb-md-3,.my-md-3{margin-bottom:1rem!important}.ml-md-3,.mx-md-3{margin-left:1rem!important}.m-md-4{margin:1.5rem!important}.mt-md-4,.my-md-4{margin-top:1.5rem!important}.mr-md-4,.mx-md-4{margin-right:1.5rem!important}.mb-md-4,.my-md-4{margin-bottom:1.5rem!important}.ml-md-4,.mx-md-4{margin-left:1.5rem!important}.m-md-5{margin:3rem!important}.mt-md-5,.my-md-5{margin-top:3rem!important}.mr-md-5,.mx-md-5{margin-right:3rem!important}.mb-md-5,.my-md-5{margin-bottom:3rem!important}.ml-md-5,.mx-md-5{margin-left:3rem!important}.p-md-0{padding:0!important}.pt-md-0,.py-md-0{padding-top:0!important}.pr-md-0,.px-md-0{padding-right:0!important}.pb-md-0,.py-md-0{padding-bottom:0!important}.pl-md-0,.px-md-0{padding-left:0!important}.p-md-1{padding:.25rem!important}.pt-md-1,.py-md-1{padding-top:.25rem!important}.pr-md-1,.px-md-1{padding-right:.25rem!important}.pb-md-1,.py-md-1{padding-bottom:.25rem!important}.pl-md-1,.px-md-1{padding-left:.25rem!important}.p-md-2{padding:.5rem!important}.pt-md-2,.py-md-2{padding-top:.5rem!important}.pr-md-2,.px-md-2{padding-right:.5rem!important}.pb-md-2,.py-md-2{padding-bottom:.5rem!important}.pl-md-2,.px-md-2{padding-left:.5rem!important}.p-md-3{padding:1rem!important}.pt-md-3,.py-md-3{padding-top:1rem!important}.pr-md-3,.px-md-3{padding-right:1rem!important}.pb-md-3,.py-md-3{padding-bottom:1rem!important}.pl-md-3,.px-md-3{padding-left:1rem!important}.p-md-4{padding:1.5rem!important}.pt-md-4,.py-md-4{padding-top:1.5rem!important}.pr-md-4,.px-md-4{padding-right:1.5rem!important}.pb-md-4,.py-md-4{padding-bottom:1.5rem!important}.pl-md-4,.px-md-4{padding-left:1.5rem!important}.p-md-5{padding:3rem!important}.pt-md-5,.py-md-5{padding-top:3rem!important}.pr-md-5,.px-md-5{padding-right:3rem!important}.pb-md-5,.py-md-5{padding-bottom:3rem!important}.pl-md-5,.px-md-5{padding-left:3rem!important}.m-md-n1{margin:-.25rem!important}.mt-md-n1,.my-md-n1{margin-top:-.25rem!important}.mr-md-n1,.mx-md-n1{margin-right:-.25rem!important}.mb-md-n1,.my-md-n1{margin-bottom:-.25rem!important}.ml-md-n1,.mx-md-n1{margin-left:-.25rem!important}.m-md-n2{margin:-.5rem!important}.mt-md-n2,.my-md-n2{margin-top:-.5rem!important}.mr-md-n2,.mx-md-n2{margin-right:-.5rem!important}.mb-md-n2,.my-md-n2{margin-bottom:-.5rem!important}.ml-md-n2,.mx-md-n2{margin-left:-.5rem!important}.m-md-n3{margin:-1rem!important}.mt-md-n3,.my-md-n3{margin-top:-1rem!important}.mr-md-n3,.mx-md-n3{margin-right:-1rem!important}.mb-md-n3,.my-md-n3{margin-bottom:-1rem!important}.ml-md-n3,.mx-md-n3{margin-left:-1rem!important}.m-md-n4{margin:-1.5rem!important}.mt-md-n4,.my-md-n4{margin-top:-1.5rem!important}.mr-md-n4,.mx-md-n4{margin-right:-1.5rem!important}.mb-md-n4,.my-md-n4{margin-bottom:-1.5rem!important}.ml-md-n4,.mx-md-n4{margin-left:-1.5rem!important}.m-md-n5{margin:-3rem!important}.mt-md-n5,.my-md-n5{margin-top:-3rem!important}.mr-md-n5,.mx-md-n5{margin-right:-3rem!important}.mb-md-n5,.my-md-n5{margin-bottom:-3rem!important}.ml-md-n5,.mx-md-n5{margin-left:-3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto,.my-md-auto{margin-top:auto!important}.mr-md-auto,.mx-md-auto{margin-right:auto!important}.mb-md-auto,.my-md-auto{margin-bottom:auto!important}.ml-md-auto,.mx-md-auto{margin-left:auto!important}}@media (min-width:992px){.m-lg-0{margin:0!important}.mt-lg-0,.my-lg-0{margin-top:0!important}.mr-lg-0,.mx-lg-0{margin-right:0!important}.mb-lg-0,.my-lg-0{margin-bottom:0!important}.ml-lg-0,.mx-lg-0{margin-left:0!important}.m-lg-1{margin:.25rem!important}.mt-lg-1,.my-lg-1{margin-top:.25rem!important}.mr-lg-1,.mx-lg-1{margin-right:.25rem!important}.mb-lg-1,.my-lg-1{margin-bottom:.25rem!important}.ml-lg-1,.mx-lg-1{margin-left:.25rem!important}.m-lg-2{margin:.5rem!important}.mt-lg-2,.my-lg-2{margin-top:.5rem!important}.mr-lg-2,.mx-lg-2{margin-right:.5rem!important}.mb-lg-2,.my-lg-2{margin-bottom:.5rem!important}.ml-lg-2,.mx-lg-2{margin-left:.5rem!important}.m-lg-3{margin:1rem!important}.mt-lg-3,.my-lg-3{margin-top:1rem!important}.mr-lg-3,.mx-lg-3{margin-right:1rem!important}.mb-lg-3,.my-lg-3{margin-bottom:1rem!important}.ml-lg-3,.mx-lg-3{margin-left:1rem!important}.m-lg-4{margin:1.5rem!important}.mt-lg-4,.my-lg-4{margin-top:1.5rem!important}.mr-lg-4,.mx-lg-4{margin-right:1.5rem!important}.mb-lg-4,.my-lg-4{margin-bottom:1.5rem!important}.ml-lg-4,.mx-lg-4{margin-left:1.5rem!important}.m-lg-5{margin:3rem!important}.mt-lg-5,.my-lg-5{margin-top:3rem!important}.mr-lg-5,.mx-lg-5{margin-right:3rem!important}.mb-lg-5,.my-lg-5{margin-bottom:3rem!important}.ml-lg-5,.mx-lg-5{margin-left:3rem!important}.p-lg-0{padding:0!important}.pt-lg-0,.py-lg-0{padding-top:0!important}.pr-lg-0,.px-lg-0{padding-right:0!important}.pb-lg-0,.py-lg-0{padding-bottom:0!important}.pl-lg-0,.px-lg-0{padding-left:0!important}.p-lg-1{padding:.25rem!important}.pt-lg-1,.py-lg-1{padding-top:.25rem!important}.pr-lg-1,.px-lg-1{padding-right:.25rem!important}.pb-lg-1,.py-lg-1{padding-bottom:.25rem!important}.pl-lg-1,.px-lg-1{padding-left:.25rem!important}.p-lg-2{padding:.5rem!important}.pt-lg-2,.py-lg-2{padding-top:.5rem!important}.pr-lg-2,.px-lg-2{padding-right:.5rem!important}.pb-lg-2,.py-lg-2{padding-bottom:.5rem!important}.pl-lg-2,.px-lg-2{padding-left:.5rem!important}.p-lg-3{padding:1rem!important}.pt-lg-3,.py-lg-3{padding-top:1rem!important}.pr-lg-3,.px-lg-3{padding-right:1rem!important}.pb-lg-3,.py-lg-3{padding-bottom:1rem!important}.pl-lg-3,.px-lg-3{padding-left:1rem!important}.p-lg-4{padding:1.5rem!important}.pt-lg-4,.py-lg-4{padding-top:1.5rem!important}.pr-lg-4,.px-lg-4{padding-right:1.5rem!important}.pb-lg-4,.py-lg-4{padding-bottom:1.5rem!important}.pl-lg-4,.px-lg-4{padding-left:1.5rem!important}.p-lg-5{padding:3rem!important}.pt-lg-5,.py-lg-5{padding-top:3rem!important}.pr-lg-5,.px-lg-5{padding-right:3rem!important}.pb-lg-5,.py-lg-5{padding-bottom:3rem!important}.pl-lg-5,.px-lg-5{padding-left:3rem!important}.m-lg-n1{margin:-.25rem!important}.mt-lg-n1,.my-lg-n1{margin-top:-.25rem!important}.mr-lg-n1,.mx-lg-n1{margin-right:-.25rem!important}.mb-lg-n1,.my-lg-n1{margin-bottom:-.25rem!important}.ml-lg-n1,.mx-lg-n1{margin-left:-.25rem!important}.m-lg-n2{margin:-.5rem!important}.mt-lg-n2,.my-lg-n2{margin-top:-.5rem!important}.mr-lg-n2,.mx-lg-n2{margin-right:-.5rem!important}.mb-lg-n2,.my-lg-n2{margin-bottom:-.5rem!important}.ml-lg-n2,.mx-lg-n2{margin-left:-.5rem!important}.m-lg-n3{margin:-1rem!important}.mt-lg-n3,.my-lg-n3{margin-top:-1rem!important}.mr-lg-n3,.mx-lg-n3{margin-right:-1rem!important}.mb-lg-n3,.my-lg-n3{margin-bottom:-1rem!important}.ml-lg-n3,.mx-lg-n3{margin-left:-1rem!important}.m-lg-n4{margin:-1.5rem!important}.mt-lg-n4,.my-lg-n4{margin-top:-1.5rem!important}.mr-lg-n4,.mx-lg-n4{margin-right:-1.5rem!important}.mb-lg-n4,.my-lg-n4{margin-bottom:-1.5rem!important}.ml-lg-n4,.mx-lg-n4{margin-left:-1.5rem!important}.m-lg-n5{margin:-3rem!important}.mt-lg-n5,.my-lg-n5{margin-top:-3rem!important}.mr-lg-n5,.mx-lg-n5{margin-right:-3rem!important}.mb-lg-n5,.my-lg-n5{margin-bottom:-3rem!important}.ml-lg-n5,.mx-lg-n5{margin-left:-3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto,.my-lg-auto{margin-top:auto!important}.mr-lg-auto,.mx-lg-auto{margin-right:auto!important}.mb-lg-auto,.my-lg-auto{margin-bottom:auto!important}.ml-lg-auto,.mx-lg-auto{margin-left:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0!important}.mt-xl-0,.my-xl-0{margin-top:0!important}.mr-xl-0,.mx-xl-0{margin-right:0!important}.mb-xl-0,.my-xl-0{margin-bottom:0!important}.ml-xl-0,.mx-xl-0{margin-left:0!important}.m-xl-1{margin:.25rem!important}.mt-xl-1,.my-xl-1{margin-top:.25rem!important}.mr-xl-1,.mx-xl-1{margin-right:.25rem!important}.mb-xl-1,.my-xl-1{margin-bottom:.25rem!important}.ml-xl-1,.mx-xl-1{margin-left:.25rem!important}.m-xl-2{margin:.5rem!important}.mt-xl-2,.my-xl-2{margin-top:.5rem!important}.mr-xl-2,.mx-xl-2{margin-right:.5rem!important}.mb-xl-2,.my-xl-2{margin-bottom:.5rem!important}.ml-xl-2,.mx-xl-2{margin-left:.5rem!important}.m-xl-3{margin:1rem!important}.mt-xl-3,.my-xl-3{margin-top:1rem!important}.mr-xl-3,.mx-xl-3{margin-right:1rem!important}.mb-xl-3,.my-xl-3{margin-bottom:1rem!important}.ml-xl-3,.mx-xl-3{margin-left:1rem!important}.m-xl-4{margin:1.5rem!important}.mt-xl-4,.my-xl-4{margin-top:1.5rem!important}.mr-xl-4,.mx-xl-4{margin-right:1.5rem!important}.mb-xl-4,.my-xl-4{margin-bottom:1.5rem!important}.ml-xl-4,.mx-xl-4{margin-left:1.5rem!important}.m-xl-5{margin:3rem!important}.mt-xl-5,.my-xl-5{margin-top:3rem!important}.mr-xl-5,.mx-xl-5{margin-right:3rem!important}.mb-xl-5,.my-xl-5{margin-bottom:3rem!important}.ml-xl-5,.mx-xl-5{margin-left:3rem!important}.p-xl-0{padding:0!important}.pt-xl-0,.py-xl-0{padding-top:0!important}.pr-xl-0,.px-xl-0{padding-right:0!important}.pb-xl-0,.py-xl-0{padding-bottom:0!important}.pl-xl-0,.px-xl-0{padding-left:0!important}.p-xl-1{padding:.25rem!important}.pt-xl-1,.py-xl-1{padding-top:.25rem!important}.pr-xl-1,.px-xl-1{padding-right:.25rem!important}.pb-xl-1,.py-xl-1{padding-bottom:.25rem!important}.pl-xl-1,.px-xl-1{padding-left:.25rem!important}.p-xl-2{padding:.5rem!important}.pt-xl-2,.py-xl-2{padding-top:.5rem!important}.pr-xl-2,.px-xl-2{padding-right:.5rem!important}.pb-xl-2,.py-xl-2{padding-bottom:.5rem!important}.pl-xl-2,.px-xl-2{padding-left:.5rem!important}.p-xl-3{padding:1rem!important}.pt-xl-3,.py-xl-3{padding-top:1rem!important}.pr-xl-3,.px-xl-3{padding-right:1rem!important}.pb-xl-3,.py-xl-3{padding-bottom:1rem!important}.pl-xl-3,.px-xl-3{padding-left:1rem!important}.p-xl-4{padding:1.5rem!important}.pt-xl-4,.py-xl-4{padding-top:1.5rem!important}.pr-xl-4,.px-xl-4{padding-right:1.5rem!important}.pb-xl-4,.py-xl-4{padding-bottom:1.5rem!important}.pl-xl-4,.px-xl-4{padding-left:1.5rem!important}.p-xl-5{padding:3rem!important}.pt-xl-5,.py-xl-5{padding-top:3rem!important}.pr-xl-5,.px-xl-5{padding-right:3rem!important}.pb-xl-5,.py-xl-5{padding-bottom:3rem!important}.pl-xl-5,.px-xl-5{padding-left:3rem!important}.m-xl-n1{margin:-.25rem!important}.mt-xl-n1,.my-xl-n1{margin-top:-.25rem!important}.mr-xl-n1,.mx-xl-n1{margin-right:-.25rem!important}.mb-xl-n1,.my-xl-n1{margin-bottom:-.25rem!important}.ml-xl-n1,.mx-xl-n1{margin-left:-.25rem!important}.m-xl-n2{margin:-.5rem!important}.mt-xl-n2,.my-xl-n2{margin-top:-.5rem!important}.mr-xl-n2,.mx-xl-n2{margin-right:-.5rem!important}.mb-xl-n2,.my-xl-n2{margin-bottom:-.5rem!important}.ml-xl-n2,.mx-xl-n2{margin-left:-.5rem!important}.m-xl-n3{margin:-1rem!important}.mt-xl-n3,.my-xl-n3{margin-top:-1rem!important}.mr-xl-n3,.mx-xl-n3{margin-right:-1rem!important}.mb-xl-n3,.my-xl-n3{margin-bottom:-1rem!important}.ml-xl-n3,.mx-xl-n3{margin-left:-1rem!important}.m-xl-n4{margin:-1.5rem!important}.mt-xl-n4,.my-xl-n4{margin-top:-1.5rem!important}.mr-xl-n4,.mx-xl-n4{margin-right:-1.5rem!important}.mb-xl-n4,.my-xl-n4{margin-bottom:-1.5rem!important}.ml-xl-n4,.mx-xl-n4{margin-left:-1.5rem!important}.m-xl-n5{margin:-3rem!important}.mt-xl-n5,.my-xl-n5{margin-top:-3rem!important}.mr-xl-n5,.mx-xl-n5{margin-right:-3rem!important}.mb-xl-n5,.my-xl-n5{margin-bottom:-3rem!important}.ml-xl-n5,.mx-xl-n5{margin-left:-3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto,.my-xl-auto{margin-top:auto!important}.mr-xl-auto,.mx-xl-auto{margin-right:auto!important}.mb-xl-auto,.my-xl-auto{margin-bottom:auto!important}.ml-xl-auto,.mx-xl-auto{margin-left:auto!important}}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;pointer-events:auto;content:"";background-color:rgba(0,0,0,0)}.text-monospace{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace!important}.text-justify{text-align:justify!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-light{font-weight:300!important}.font-weight-lighter{font-weight:lighter!important}.font-weight-normal{font-weight:400!important}.font-weight-bold{font-weight:700!important}.font-weight-bolder{font-weight:bolder!important}.font-italic{font-style:italic!important}.text-white{color:#fff!important}.text-primary{color:#007bff!important}a.text-primary:focus,a.text-primary:hover{color:#0056b3!important}.text-secondary{color:#6c757d!important}a.text-secondary:focus,a.text-secondary:hover{color:#494f54!important}.text-success{color:#28a745!important}a.text-success:focus,a.text-success:hover{color:#19692c!important}.text-info{color:#17a2b8!important}a.text-info:focus,a.text-info:hover{color:#0f6674!important}.text-warning{color:#ffc107!important}a.text-warning:focus,a.text-warning:hover{color:#ba8b00!important}.text-danger{color:#dc3545!important}a.text-danger:focus,a.text-danger:hover{color:#a71d2a!important}.text-light{color:#f8f9fa!important}a.text-light:focus,a.text-light:hover{color:#cbd3da!important}.text-dark{color:#343a40!important}a.text-dark:focus,a.text-dark:hover{color:#121416!important}.text-body{color:#212529!important}.text-muted{color:#6c757d!important}.text-black-50{color:rgba(0,0,0,.5)!important}.text-white-50{color:rgba(255,255,255,.5)!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.text-decoration-none{text-decoration:none!important}.text-break{word-wrap:break-word!important}.text-reset{color:inherit!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}@media print{*,::after,::before{text-shadow:none!important;box-shadow:none!important}a:not(.btn){text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap!important}blockquote,pre{border:1px solid #adb5bd;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}@page{size:a3}body{min-width:992px!important}.container{min-width:992px!important}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #dee2e6!important}.table-dark{color:inherit}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#dee2e6}.table .thead-dark th{color:inherit;border-color:#dee2e6}} +/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/imgr/imgrpkg/static-content/bootstrap.min.js b/imgr/imgrpkg/static-content/bootstrap.min.js new file mode 100644 index 00000000..3ecf55f2 --- /dev/null +++ b/imgr/imgrpkg/static-content/bootstrap.min.js @@ -0,0 +1,7 @@ +/*! + * Bootstrap v4.5.0 (https://getbootstrap.com/) + * Copyright 2011-2020 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports,require("jquery"),require("popper.js")):"function"==typeof define&&define.amd?define(["exports","jquery","popper.js"],e):e((t=t||self).bootstrap={},t.jQuery,t.Popper)}(this,(function(t,e,n){"use strict";function i(t,e){for(var n=0;n=4)throw new Error("Bootstrap's JavaScript requires at least jQuery v1.9.1 but less than v4.0.0")}};c.jQueryDetection(),e.fn.emulateTransitionEnd=l,e.event.special[c.TRANSITION_END]={bindType:"transitionend",delegateType:"transitionend",handle:function(t){if(e(t.target).is(this))return t.handleObj.handler.apply(this,arguments)}};var h="alert",u=e.fn[h],d=function(){function t(t){this._element=t}var n=t.prototype;return n.close=function(t){var e=this._element;t&&(e=this._getRootElement(t)),this._triggerCloseEvent(e).isDefaultPrevented()||this._removeElement(e)},n.dispose=function(){e.removeData(this._element,"bs.alert"),this._element=null},n._getRootElement=function(t){var n=c.getSelectorFromElement(t),i=!1;return n&&(i=document.querySelector(n)),i||(i=e(t).closest(".alert")[0]),i},n._triggerCloseEvent=function(t){var n=e.Event("close.bs.alert");return e(t).trigger(n),n},n._removeElement=function(t){var n=this;if(e(t).removeClass("show"),e(t).hasClass("fade")){var i=c.getTransitionDurationFromElement(t);e(t).one(c.TRANSITION_END,(function(e){return n._destroyElement(t,e)})).emulateTransitionEnd(i)}else this._destroyElement(t)},n._destroyElement=function(t){e(t).detach().trigger("closed.bs.alert").remove()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.alert");o||(o=new t(this),i.data("bs.alert",o)),"close"===n&&o[n](this)}))},t._handleDismiss=function(t){return function(e){e&&e.preventDefault(),t.close(this)}},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.alert.data-api",'[data-dismiss="alert"]',d._handleDismiss(new d)),e.fn[h]=d._jQueryInterface,e.fn[h].Constructor=d,e.fn[h].noConflict=function(){return e.fn[h]=u,d._jQueryInterface};var f=e.fn.button,g=function(){function t(t){this._element=t}var n=t.prototype;return n.toggle=function(){var t=!0,n=!0,i=e(this._element).closest('[data-toggle="buttons"]')[0];if(i){var o=this._element.querySelector('input:not([type="hidden"])');if(o){if("radio"===o.type)if(o.checked&&this._element.classList.contains("active"))t=!1;else{var s=i.querySelector(".active");s&&e(s).removeClass("active")}t&&("checkbox"!==o.type&&"radio"!==o.type||(o.checked=!this._element.classList.contains("active")),e(o).trigger("change")),o.focus(),n=!1}}this._element.hasAttribute("disabled")||this._element.classList.contains("disabled")||(n&&this._element.setAttribute("aria-pressed",!this._element.classList.contains("active")),t&&e(this._element).toggleClass("active"))},n.dispose=function(){e.removeData(this._element,"bs.button"),this._element=null},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.button");i||(i=new t(this),e(this).data("bs.button",i)),"toggle"===n&&i[n]()}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.button.data-api",'[data-toggle^="button"]',(function(t){var n=t.target,i=n;if(e(n).hasClass("btn")||(n=e(n).closest(".btn")[0]),!n||n.hasAttribute("disabled")||n.classList.contains("disabled"))t.preventDefault();else{var o=n.querySelector('input:not([type="hidden"])');if(o&&(o.hasAttribute("disabled")||o.classList.contains("disabled")))return void t.preventDefault();"LABEL"===i.tagName&&o&&"checkbox"===o.type&&t.preventDefault(),g._jQueryInterface.call(e(n),"toggle")}})).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',(function(t){var n=e(t.target).closest(".btn")[0];e(n).toggleClass("focus",/^focus(in)?$/.test(t.type))})),e(window).on("load.bs.button.data-api",(function(){for(var t=[].slice.call(document.querySelectorAll('[data-toggle="buttons"] .btn')),e=0,n=t.length;e0,this._pointerEvent=Boolean(window.PointerEvent||window.MSPointerEvent),this._addEventListeners()}var n=t.prototype;return n.next=function(){this._isSliding||this._slide("next")},n.nextWhenVisible=function(){!document.hidden&&e(this._element).is(":visible")&&"hidden"!==e(this._element).css("visibility")&&this.next()},n.prev=function(){this._isSliding||this._slide("prev")},n.pause=function(t){t||(this._isPaused=!0),this._element.querySelector(".carousel-item-next, .carousel-item-prev")&&(c.triggerTransitionEnd(this._element),this.cycle(!0)),clearInterval(this._interval),this._interval=null},n.cycle=function(t){t||(this._isPaused=!1),this._interval&&(clearInterval(this._interval),this._interval=null),this._config.interval&&!this._isPaused&&(this._interval=setInterval((document.visibilityState?this.nextWhenVisible:this.next).bind(this),this._config.interval))},n.to=function(t){var n=this;this._activeElement=this._element.querySelector(".active.carousel-item");var i=this._getItemIndex(this._activeElement);if(!(t>this._items.length-1||t<0))if(this._isSliding)e(this._element).one("slid.bs.carousel",(function(){return n.to(t)}));else{if(i===t)return this.pause(),void this.cycle();var o=t>i?"next":"prev";this._slide(o,this._items[t])}},n.dispose=function(){e(this._element).off(p),e.removeData(this._element,"bs.carousel"),this._items=null,this._config=null,this._element=null,this._interval=null,this._isPaused=null,this._isSliding=null,this._activeElement=null,this._indicatorsElement=null},n._getConfig=function(t){return t=a(a({},v),t),c.typeCheckConfig(m,t,b),t},n._handleSwipe=function(){var t=Math.abs(this.touchDeltaX);if(!(t<=40)){var e=t/this.touchDeltaX;this.touchDeltaX=0,e>0&&this.prev(),e<0&&this.next()}},n._addEventListeners=function(){var t=this;this._config.keyboard&&e(this._element).on("keydown.bs.carousel",(function(e){return t._keydown(e)})),"hover"===this._config.pause&&e(this._element).on("mouseenter.bs.carousel",(function(e){return t.pause(e)})).on("mouseleave.bs.carousel",(function(e){return t.cycle(e)})),this._config.touch&&this._addTouchEventListeners()},n._addTouchEventListeners=function(){var t=this;if(this._touchSupported){var n=function(e){t._pointerEvent&&y[e.originalEvent.pointerType.toUpperCase()]?t.touchStartX=e.originalEvent.clientX:t._pointerEvent||(t.touchStartX=e.originalEvent.touches[0].clientX)},i=function(e){t._pointerEvent&&y[e.originalEvent.pointerType.toUpperCase()]&&(t.touchDeltaX=e.originalEvent.clientX-t.touchStartX),t._handleSwipe(),"hover"===t._config.pause&&(t.pause(),t.touchTimeout&&clearTimeout(t.touchTimeout),t.touchTimeout=setTimeout((function(e){return t.cycle(e)}),500+t._config.interval))};e(this._element.querySelectorAll(".carousel-item img")).on("dragstart.bs.carousel",(function(t){return t.preventDefault()})),this._pointerEvent?(e(this._element).on("pointerdown.bs.carousel",(function(t){return n(t)})),e(this._element).on("pointerup.bs.carousel",(function(t){return i(t)})),this._element.classList.add("pointer-event")):(e(this._element).on("touchstart.bs.carousel",(function(t){return n(t)})),e(this._element).on("touchmove.bs.carousel",(function(e){return function(e){e.originalEvent.touches&&e.originalEvent.touches.length>1?t.touchDeltaX=0:t.touchDeltaX=e.originalEvent.touches[0].clientX-t.touchStartX}(e)})),e(this._element).on("touchend.bs.carousel",(function(t){return i(t)})))}},n._keydown=function(t){if(!/input|textarea/i.test(t.target.tagName))switch(t.which){case 37:t.preventDefault(),this.prev();break;case 39:t.preventDefault(),this.next()}},n._getItemIndex=function(t){return this._items=t&&t.parentNode?[].slice.call(t.parentNode.querySelectorAll(".carousel-item")):[],this._items.indexOf(t)},n._getItemByDirection=function(t,e){var n="next"===t,i="prev"===t,o=this._getItemIndex(e),s=this._items.length-1;if((i&&0===o||n&&o===s)&&!this._config.wrap)return e;var r=(o+("prev"===t?-1:1))%this._items.length;return-1===r?this._items[this._items.length-1]:this._items[r]},n._triggerSlideEvent=function(t,n){var i=this._getItemIndex(t),o=this._getItemIndex(this._element.querySelector(".active.carousel-item")),s=e.Event("slide.bs.carousel",{relatedTarget:t,direction:n,from:o,to:i});return e(this._element).trigger(s),s},n._setActiveIndicatorElement=function(t){if(this._indicatorsElement){var n=[].slice.call(this._indicatorsElement.querySelectorAll(".active"));e(n).removeClass("active");var i=this._indicatorsElement.children[this._getItemIndex(t)];i&&e(i).addClass("active")}},n._slide=function(t,n){var i,o,s,r=this,a=this._element.querySelector(".active.carousel-item"),l=this._getItemIndex(a),h=n||a&&this._getItemByDirection(t,a),u=this._getItemIndex(h),d=Boolean(this._interval);if("next"===t?(i="carousel-item-left",o="carousel-item-next",s="left"):(i="carousel-item-right",o="carousel-item-prev",s="right"),h&&e(h).hasClass("active"))this._isSliding=!1;else if(!this._triggerSlideEvent(h,s).isDefaultPrevented()&&a&&h){this._isSliding=!0,d&&this.pause(),this._setActiveIndicatorElement(h);var f=e.Event("slid.bs.carousel",{relatedTarget:h,direction:s,from:l,to:u});if(e(this._element).hasClass("slide")){e(h).addClass(o),c.reflow(h),e(a).addClass(i),e(h).addClass(i);var g=parseInt(h.getAttribute("data-interval"),10);g?(this._config.defaultInterval=this._config.defaultInterval||this._config.interval,this._config.interval=g):this._config.interval=this._config.defaultInterval||this._config.interval;var m=c.getTransitionDurationFromElement(a);e(a).one(c.TRANSITION_END,(function(){e(h).removeClass(i+" "+o).addClass("active"),e(a).removeClass("active "+o+" "+i),r._isSliding=!1,setTimeout((function(){return e(r._element).trigger(f)}),0)})).emulateTransitionEnd(m)}else e(a).removeClass("active"),e(h).addClass("active"),this._isSliding=!1,e(this._element).trigger(f);d&&this.cycle()}},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.carousel"),o=a(a({},v),e(this).data());"object"==typeof n&&(o=a(a({},o),n));var s="string"==typeof n?n:o.slide;if(i||(i=new t(this,o),e(this).data("bs.carousel",i)),"number"==typeof n)i.to(n);else if("string"==typeof s){if("undefined"==typeof i[s])throw new TypeError('No method named "'+s+'"');i[s]()}else o.interval&&o.ride&&(i.pause(),i.cycle())}))},t._dataApiClickHandler=function(n){var i=c.getSelectorFromElement(this);if(i){var o=e(i)[0];if(o&&e(o).hasClass("carousel")){var s=a(a({},e(o).data()),e(this).data()),r=this.getAttribute("data-slide-to");r&&(s.interval=!1),t._jQueryInterface.call(e(o),s),r&&e(o).data("bs.carousel").to(r),n.preventDefault()}}},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return v}}]),t}();e(document).on("click.bs.carousel.data-api","[data-slide], [data-slide-to]",E._dataApiClickHandler),e(window).on("load.bs.carousel.data-api",(function(){for(var t=[].slice.call(document.querySelectorAll('[data-ride="carousel"]')),n=0,i=t.length;n0&&(this._selector=r,this._triggerArray.push(s))}this._parent=this._config.parent?this._getParent():null,this._config.parent||this._addAriaAndCollapsedClass(this._element,this._triggerArray),this._config.toggle&&this.toggle()}var n=t.prototype;return n.toggle=function(){e(this._element).hasClass("show")?this.hide():this.show()},n.show=function(){var n,i,o=this;if(!this._isTransitioning&&!e(this._element).hasClass("show")&&(this._parent&&0===(n=[].slice.call(this._parent.querySelectorAll(".show, .collapsing")).filter((function(t){return"string"==typeof o._config.parent?t.getAttribute("data-parent")===o._config.parent:t.classList.contains("collapse")}))).length&&(n=null),!(n&&(i=e(n).not(this._selector).data("bs.collapse"))&&i._isTransitioning))){var s=e.Event("show.bs.collapse");if(e(this._element).trigger(s),!s.isDefaultPrevented()){n&&(t._jQueryInterface.call(e(n).not(this._selector),"hide"),i||e(n).data("bs.collapse",null));var r=this._getDimension();e(this._element).removeClass("collapse").addClass("collapsing"),this._element.style[r]=0,this._triggerArray.length&&e(this._triggerArray).removeClass("collapsed").attr("aria-expanded",!0),this.setTransitioning(!0);var a="scroll"+(r[0].toUpperCase()+r.slice(1)),l=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,(function(){e(o._element).removeClass("collapsing").addClass("collapse show"),o._element.style[r]="",o.setTransitioning(!1),e(o._element).trigger("shown.bs.collapse")})).emulateTransitionEnd(l),this._element.style[r]=this._element[a]+"px"}}},n.hide=function(){var t=this;if(!this._isTransitioning&&e(this._element).hasClass("show")){var n=e.Event("hide.bs.collapse");if(e(this._element).trigger(n),!n.isDefaultPrevented()){var i=this._getDimension();this._element.style[i]=this._element.getBoundingClientRect()[i]+"px",c.reflow(this._element),e(this._element).addClass("collapsing").removeClass("collapse show");var o=this._triggerArray.length;if(o>0)for(var s=0;s0},i._getOffset=function(){var t=this,e={};return"function"==typeof this._config.offset?e.fn=function(e){return e.offsets=a(a({},e.offsets),t._config.offset(e.offsets,t._element)||{}),e}:e.offset=this._config.offset,e},i._getPopperConfig=function(){var t={placement:this._getPlacement(),modifiers:{offset:this._getOffset(),flip:{enabled:this._config.flip},preventOverflow:{boundariesElement:this._config.boundary}}};return"static"===this._config.display&&(t.modifiers.applyStyle={enabled:!1}),a(a({},t),this._config.popperConfig)},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.dropdown");if(i||(i=new t(this,"object"==typeof n?n:null),e(this).data("bs.dropdown",i)),"string"==typeof n){if("undefined"==typeof i[n])throw new TypeError('No method named "'+n+'"');i[n]()}}))},t._clearMenus=function(n){if(!n||3!==n.which&&("keyup"!==n.type||9===n.which))for(var i=[].slice.call(document.querySelectorAll('[data-toggle="dropdown"]')),o=0,s=i.length;o0&&r--,40===n.which&&rdocument.documentElement.clientHeight;!this._isBodyOverflowing&&t&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!t&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},n._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},n._checkScrollbar=function(){var t=document.body.getBoundingClientRect();this._isBodyOverflowing=Math.round(t.left+t.right)
',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip",boundary:"scrollParent",sanitize:!0,sanitizeFn:null,whiteList:F,popperConfig:null},Y={HIDE:"hide.bs.tooltip",HIDDEN:"hidden.bs.tooltip",SHOW:"show.bs.tooltip",SHOWN:"shown.bs.tooltip",INSERTED:"inserted.bs.tooltip",CLICK:"click.bs.tooltip",FOCUSIN:"focusin.bs.tooltip",FOCUSOUT:"focusout.bs.tooltip",MOUSEENTER:"mouseenter.bs.tooltip",MOUSELEAVE:"mouseleave.bs.tooltip"},$=function(){function t(t,e){if("undefined"==typeof n)throw new TypeError("Bootstrap's tooltips require Popper.js (https://popper.js.org/)");this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=t,this.config=this._getConfig(e),this.tip=null,this._setListeners()}var i=t.prototype;return i.enable=function(){this._isEnabled=!0},i.disable=function(){this._isEnabled=!1},i.toggleEnabled=function(){this._isEnabled=!this._isEnabled},i.toggle=function(t){if(this._isEnabled)if(t){var n=this.constructor.DATA_KEY,i=e(t.currentTarget).data(n);i||(i=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(n,i)),i._activeTrigger.click=!i._activeTrigger.click,i._isWithActiveTrigger()?i._enter(null,i):i._leave(null,i)}else{if(e(this.getTipElement()).hasClass("show"))return void this._leave(null,this);this._enter(null,this)}},i.dispose=function(){clearTimeout(this._timeout),e.removeData(this.element,this.constructor.DATA_KEY),e(this.element).off(this.constructor.EVENT_KEY),e(this.element).closest(".modal").off("hide.bs.modal",this._hideModalHandler),this.tip&&e(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,this._activeTrigger=null,this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},i.show=function(){var t=this;if("none"===e(this.element).css("display"))throw new Error("Please use show on visible elements");var i=e.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){e(this.element).trigger(i);var o=c.findShadowRoot(this.element),s=e.contains(null!==o?o:this.element.ownerDocument.documentElement,this.element);if(i.isDefaultPrevented()||!s)return;var r=this.getTipElement(),a=c.getUID(this.constructor.NAME);r.setAttribute("id",a),this.element.setAttribute("aria-describedby",a),this.setContent(),this.config.animation&&e(r).addClass("fade");var l="function"==typeof this.config.placement?this.config.placement.call(this,r,this.element):this.config.placement,h=this._getAttachment(l);this.addAttachmentClass(h);var u=this._getContainer();e(r).data(this.constructor.DATA_KEY,this),e.contains(this.element.ownerDocument.documentElement,this.tip)||e(r).appendTo(u),e(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new n(this.element,r,this._getPopperConfig(h)),e(r).addClass("show"),"ontouchstart"in document.documentElement&&e(document.body).children().on("mouseover",null,e.noop);var d=function(){t.config.animation&&t._fixTransition();var n=t._hoverState;t._hoverState=null,e(t.element).trigger(t.constructor.Event.SHOWN),"out"===n&&t._leave(null,t)};if(e(this.tip).hasClass("fade")){var f=c.getTransitionDurationFromElement(this.tip);e(this.tip).one(c.TRANSITION_END,d).emulateTransitionEnd(f)}else d()}},i.hide=function(t){var n=this,i=this.getTipElement(),o=e.Event(this.constructor.Event.HIDE),s=function(){"show"!==n._hoverState&&i.parentNode&&i.parentNode.removeChild(i),n._cleanTipClass(),n.element.removeAttribute("aria-describedby"),e(n.element).trigger(n.constructor.Event.HIDDEN),null!==n._popper&&n._popper.destroy(),t&&t()};if(e(this.element).trigger(o),!o.isDefaultPrevented()){if(e(i).removeClass("show"),"ontouchstart"in document.documentElement&&e(document.body).children().off("mouseover",null,e.noop),this._activeTrigger.click=!1,this._activeTrigger.focus=!1,this._activeTrigger.hover=!1,e(this.tip).hasClass("fade")){var r=c.getTransitionDurationFromElement(i);e(i).one(c.TRANSITION_END,s).emulateTransitionEnd(r)}else s();this._hoverState=""}},i.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},i.isWithContent=function(){return Boolean(this.getTitle())},i.addAttachmentClass=function(t){e(this.getTipElement()).addClass("bs-tooltip-"+t)},i.getTipElement=function(){return this.tip=this.tip||e(this.config.template)[0],this.tip},i.setContent=function(){var t=this.getTipElement();this.setElementContent(e(t.querySelectorAll(".tooltip-inner")),this.getTitle()),e(t).removeClass("fade show")},i.setElementContent=function(t,n){"object"!=typeof n||!n.nodeType&&!n.jquery?this.config.html?(this.config.sanitize&&(n=H(n,this.config.whiteList,this.config.sanitizeFn)),t.html(n)):t.text(n):this.config.html?e(n).parent().is(t)||t.empty().append(n):t.text(e(n).text())},i.getTitle=function(){var t=this.element.getAttribute("data-original-title");return t||(t="function"==typeof this.config.title?this.config.title.call(this.element):this.config.title),t},i._getPopperConfig=function(t){var e=this;return a(a({},{placement:t,modifiers:{offset:this._getOffset(),flip:{behavior:this.config.fallbackPlacement},arrow:{element:".arrow"},preventOverflow:{boundariesElement:this.config.boundary}},onCreate:function(t){t.originalPlacement!==t.placement&&e._handlePopperPlacementChange(t)},onUpdate:function(t){return e._handlePopperPlacementChange(t)}}),this.config.popperConfig)},i._getOffset=function(){var t=this,e={};return"function"==typeof this.config.offset?e.fn=function(e){return e.offsets=a(a({},e.offsets),t.config.offset(e.offsets,t.element)||{}),e}:e.offset=this.config.offset,e},i._getContainer=function(){return!1===this.config.container?document.body:c.isElement(this.config.container)?e(this.config.container):e(document).find(this.config.container)},i._getAttachment=function(t){return K[t.toUpperCase()]},i._setListeners=function(){var t=this;this.config.trigger.split(" ").forEach((function(n){if("click"===n)e(t.element).on(t.constructor.Event.CLICK,t.config.selector,(function(e){return t.toggle(e)}));else if("manual"!==n){var i="hover"===n?t.constructor.Event.MOUSEENTER:t.constructor.Event.FOCUSIN,o="hover"===n?t.constructor.Event.MOUSELEAVE:t.constructor.Event.FOCUSOUT;e(t.element).on(i,t.config.selector,(function(e){return t._enter(e)})).on(o,t.config.selector,(function(e){return t._leave(e)}))}})),this._hideModalHandler=function(){t.element&&t.hide()},e(this.element).closest(".modal").on("hide.bs.modal",this._hideModalHandler),this.config.selector?this.config=a(a({},this.config),{},{trigger:"manual",selector:""}):this._fixTitle()},i._fixTitle=function(){var t=typeof this.element.getAttribute("data-original-title");(this.element.getAttribute("title")||"string"!==t)&&(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},i._enter=function(t,n){var i=this.constructor.DATA_KEY;(n=n||e(t.currentTarget).data(i))||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(i,n)),t&&(n._activeTrigger["focusin"===t.type?"focus":"hover"]=!0),e(n.getTipElement()).hasClass("show")||"show"===n._hoverState?n._hoverState="show":(clearTimeout(n._timeout),n._hoverState="show",n.config.delay&&n.config.delay.show?n._timeout=setTimeout((function(){"show"===n._hoverState&&n.show()}),n.config.delay.show):n.show())},i._leave=function(t,n){var i=this.constructor.DATA_KEY;(n=n||e(t.currentTarget).data(i))||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(i,n)),t&&(n._activeTrigger["focusout"===t.type?"focus":"hover"]=!1),n._isWithActiveTrigger()||(clearTimeout(n._timeout),n._hoverState="out",n.config.delay&&n.config.delay.hide?n._timeout=setTimeout((function(){"out"===n._hoverState&&n.hide()}),n.config.delay.hide):n.hide())},i._isWithActiveTrigger=function(){for(var t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1},i._getConfig=function(t){var n=e(this.element).data();return Object.keys(n).forEach((function(t){-1!==V.indexOf(t)&&delete n[t]})),"number"==typeof(t=a(a(a({},this.constructor.Default),n),"object"==typeof t&&t?t:{})).delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),c.typeCheckConfig(U,t,this.constructor.DefaultType),t.sanitize&&(t.template=H(t.template,t.whiteList,t.sanitizeFn)),t},i._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]!==this.config[e]&&(t[e]=this.config[e]);return t},i._cleanTipClass=function(){var t=e(this.getTipElement()),n=t.attr("class").match(W);null!==n&&n.length&&t.removeClass(n.join(""))},i._handlePopperPlacementChange=function(t){this.tip=t.instance.popper,this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(t.placement))},i._fixTransition=function(){var t=this.getTipElement(),n=this.config.animation;null===t.getAttribute("x-placement")&&(e(t).removeClass("fade"),this.config.animation=!1,this.hide(),this.show(),this.config.animation=n)},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.tooltip"),o="object"==typeof n&&n;if((i||!/dispose|hide/.test(n))&&(i||(i=new t(this,o),e(this).data("bs.tooltip",i)),"string"==typeof n)){if("undefined"==typeof i[n])throw new TypeError('No method named "'+n+'"');i[n]()}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return X}},{key:"NAME",get:function(){return U}},{key:"DATA_KEY",get:function(){return"bs.tooltip"}},{key:"Event",get:function(){return Y}},{key:"EVENT_KEY",get:function(){return".bs.tooltip"}},{key:"DefaultType",get:function(){return z}}]),t}();e.fn[U]=$._jQueryInterface,e.fn[U].Constructor=$,e.fn[U].noConflict=function(){return e.fn[U]=M,$._jQueryInterface};var J="popover",G=e.fn[J],Z=new RegExp("(^|\\s)bs-popover\\S+","g"),tt=a(a({},$.Default),{},{placement:"right",trigger:"click",content:"",template:''}),et=a(a({},$.DefaultType),{},{content:"(string|element|function)"}),nt={HIDE:"hide.bs.popover",HIDDEN:"hidden.bs.popover",SHOW:"show.bs.popover",SHOWN:"shown.bs.popover",INSERTED:"inserted.bs.popover",CLICK:"click.bs.popover",FOCUSIN:"focusin.bs.popover",FOCUSOUT:"focusout.bs.popover",MOUSEENTER:"mouseenter.bs.popover",MOUSELEAVE:"mouseleave.bs.popover"},it=function(t){var n,i;function s(){return t.apply(this,arguments)||this}i=t,(n=s).prototype=Object.create(i.prototype),n.prototype.constructor=n,n.__proto__=i;var r=s.prototype;return r.isWithContent=function(){return this.getTitle()||this._getContent()},r.addAttachmentClass=function(t){e(this.getTipElement()).addClass("bs-popover-"+t)},r.getTipElement=function(){return this.tip=this.tip||e(this.config.template)[0],this.tip},r.setContent=function(){var t=e(this.getTipElement());this.setElementContent(t.find(".popover-header"),this.getTitle());var n=this._getContent();"function"==typeof n&&(n=n.call(this.element)),this.setElementContent(t.find(".popover-body"),n),t.removeClass("fade show")},r._getContent=function(){return this.element.getAttribute("data-content")||this.config.content},r._cleanTipClass=function(){var t=e(this.getTipElement()),n=t.attr("class").match(Z);null!==n&&n.length>0&&t.removeClass(n.join(""))},s._jQueryInterface=function(t){return this.each((function(){var n=e(this).data("bs.popover"),i="object"==typeof t?t:null;if((n||!/dispose|hide/.test(t))&&(n||(n=new s(this,i),e(this).data("bs.popover",n)),"string"==typeof t)){if("undefined"==typeof n[t])throw new TypeError('No method named "'+t+'"');n[t]()}}))},o(s,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return tt}},{key:"NAME",get:function(){return J}},{key:"DATA_KEY",get:function(){return"bs.popover"}},{key:"Event",get:function(){return nt}},{key:"EVENT_KEY",get:function(){return".bs.popover"}},{key:"DefaultType",get:function(){return et}}]),s}($);e.fn[J]=it._jQueryInterface,e.fn[J].Constructor=it,e.fn[J].noConflict=function(){return e.fn[J]=G,it._jQueryInterface};var ot="scrollspy",st=e.fn[ot],rt={offset:10,method:"auto",target:""},at={offset:"number",method:"string",target:"(string|element)"},lt=function(){function t(t,n){var i=this;this._element=t,this._scrollElement="BODY"===t.tagName?window:t,this._config=this._getConfig(n),this._selector=this._config.target+" .nav-link,"+this._config.target+" .list-group-item,"+this._config.target+" .dropdown-item",this._offsets=[],this._targets=[],this._activeTarget=null,this._scrollHeight=0,e(this._scrollElement).on("scroll.bs.scrollspy",(function(t){return i._process(t)})),this.refresh(),this._process()}var n=t.prototype;return n.refresh=function(){var t=this,n=this._scrollElement===this._scrollElement.window?"offset":"position",i="auto"===this._config.method?n:this._config.method,o="position"===i?this._getScrollTop():0;this._offsets=[],this._targets=[],this._scrollHeight=this._getScrollHeight(),[].slice.call(document.querySelectorAll(this._selector)).map((function(t){var n,s=c.getSelectorFromElement(t);if(s&&(n=document.querySelector(s)),n){var r=n.getBoundingClientRect();if(r.width||r.height)return[e(n)[i]().top+o,s]}return null})).filter((function(t){return t})).sort((function(t,e){return t[0]-e[0]})).forEach((function(e){t._offsets.push(e[0]),t._targets.push(e[1])}))},n.dispose=function(){e.removeData(this._element,"bs.scrollspy"),e(this._scrollElement).off(".bs.scrollspy"),this._element=null,this._scrollElement=null,this._config=null,this._selector=null,this._offsets=null,this._targets=null,this._activeTarget=null,this._scrollHeight=null},n._getConfig=function(t){if("string"!=typeof(t=a(a({},rt),"object"==typeof t&&t?t:{})).target&&c.isElement(t.target)){var n=e(t.target).attr("id");n||(n=c.getUID(ot),e(t.target).attr("id",n)),t.target="#"+n}return c.typeCheckConfig(ot,t,at),t},n._getScrollTop=function(){return this._scrollElement===window?this._scrollElement.pageYOffset:this._scrollElement.scrollTop},n._getScrollHeight=function(){return this._scrollElement.scrollHeight||Math.max(document.body.scrollHeight,document.documentElement.scrollHeight)},n._getOffsetHeight=function(){return this._scrollElement===window?window.innerHeight:this._scrollElement.getBoundingClientRect().height},n._process=function(){var t=this._getScrollTop()+this._config.offset,e=this._getScrollHeight(),n=this._config.offset+e-this._getOffsetHeight();if(this._scrollHeight!==e&&this.refresh(),t>=n){var i=this._targets[this._targets.length-1];this._activeTarget!==i&&this._activate(i)}else{if(this._activeTarget&&t0)return this._activeTarget=null,void this._clear();for(var o=this._offsets.length;o--;){this._activeTarget!==this._targets[o]&&t>=this._offsets[o]&&("undefined"==typeof this._offsets[o+1]||t li > .active":".active";i=(i=e.makeArray(e(o).find(r)))[i.length-1]}var a=e.Event("hide.bs.tab",{relatedTarget:this._element}),l=e.Event("show.bs.tab",{relatedTarget:i});if(i&&e(i).trigger(a),e(this._element).trigger(l),!l.isDefaultPrevented()&&!a.isDefaultPrevented()){s&&(n=document.querySelector(s)),this._activate(this._element,o);var h=function(){var n=e.Event("hidden.bs.tab",{relatedTarget:t._element}),o=e.Event("shown.bs.tab",{relatedTarget:i});e(i).trigger(n),e(t._element).trigger(o)};n?this._activate(n,n.parentNode,h):h()}}},n.dispose=function(){e.removeData(this._element,"bs.tab"),this._element=null},n._activate=function(t,n,i){var o=this,s=(!n||"UL"!==n.nodeName&&"OL"!==n.nodeName?e(n).children(".active"):e(n).find("> li > .active"))[0],r=i&&s&&e(s).hasClass("fade"),a=function(){return o._transitionComplete(t,s,i)};if(s&&r){var l=c.getTransitionDurationFromElement(s);e(s).removeClass("show").one(c.TRANSITION_END,a).emulateTransitionEnd(l)}else a()},n._transitionComplete=function(t,n,i){if(n){e(n).removeClass("active");var o=e(n.parentNode).find("> .dropdown-menu .active")[0];o&&e(o).removeClass("active"),"tab"===n.getAttribute("role")&&n.setAttribute("aria-selected",!1)}if(e(t).addClass("active"),"tab"===t.getAttribute("role")&&t.setAttribute("aria-selected",!0),c.reflow(t),t.classList.contains("fade")&&t.classList.add("show"),t.parentNode&&e(t.parentNode).hasClass("dropdown-menu")){var s=e(t).closest(".dropdown")[0];if(s){var r=[].slice.call(s.querySelectorAll(".dropdown-toggle"));e(r).addClass("active")}t.setAttribute("aria-expanded",!0)}i&&i()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.tab");if(o||(o=new t(this),i.data("bs.tab",o)),"string"==typeof n){if("undefined"==typeof o[n])throw new TypeError('No method named "'+n+'"');o[n]()}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.tab.data-api",'[data-toggle="tab"], [data-toggle="pill"], [data-toggle="list"]',(function(t){t.preventDefault(),ht._jQueryInterface.call(e(this),"show")})),e.fn.tab=ht._jQueryInterface,e.fn.tab.Constructor=ht,e.fn.tab.noConflict=function(){return e.fn.tab=ct,ht._jQueryInterface};var ut=e.fn.toast,dt={animation:"boolean",autohide:"boolean",delay:"number"},ft={animation:!0,autohide:!0,delay:500},gt=function(){function t(t,e){this._element=t,this._config=this._getConfig(e),this._timeout=null,this._setListeners()}var n=t.prototype;return n.show=function(){var t=this,n=e.Event("show.bs.toast");if(e(this._element).trigger(n),!n.isDefaultPrevented()){this._config.animation&&this._element.classList.add("fade");var i=function(){t._element.classList.remove("showing"),t._element.classList.add("show"),e(t._element).trigger("shown.bs.toast"),t._config.autohide&&(t._timeout=setTimeout((function(){t.hide()}),t._config.delay))};if(this._element.classList.remove("hide"),c.reflow(this._element),this._element.classList.add("showing"),this._config.animation){var o=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,i).emulateTransitionEnd(o)}else i()}},n.hide=function(){if(this._element.classList.contains("show")){var t=e.Event("hide.bs.toast");e(this._element).trigger(t),t.isDefaultPrevented()||this._close()}},n.dispose=function(){clearTimeout(this._timeout),this._timeout=null,this._element.classList.contains("show")&&this._element.classList.remove("show"),e(this._element).off("click.dismiss.bs.toast"),e.removeData(this._element,"bs.toast"),this._element=null,this._config=null},n._getConfig=function(t){return t=a(a(a({},ft),e(this._element).data()),"object"==typeof t&&t?t:{}),c.typeCheckConfig("toast",t,this.constructor.DefaultType),t},n._setListeners=function(){var t=this;e(this._element).on("click.dismiss.bs.toast",'[data-dismiss="toast"]',(function(){return t.hide()}))},n._close=function(){var t=this,n=function(){t._element.classList.add("hide"),e(t._element).trigger("hidden.bs.toast")};if(this._element.classList.remove("show"),this._config.animation){var i=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,n).emulateTransitionEnd(i)}else n()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.toast");if(o||(o=new t(this,"object"==typeof n&&n),i.data("bs.toast",o)),"string"==typeof n){if("undefined"==typeof o[n])throw new TypeError('No method named "'+n+'"');o[n](this)}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"DefaultType",get:function(){return dt}},{key:"Default",get:function(){return ft}}]),t}();e.fn.toast=gt._jQueryInterface,e.fn.toast.Constructor=gt,e.fn.toast.noConflict=function(){return e.fn.toast=ut,gt._jQueryInterface},t.Alert=d,t.Button=g,t.Carousel=E,t.Collapse=D,t.Dropdown=j,t.Modal=R,t.Popover=it,t.Scrollspy=lt,t.Tab=ht,t.Toast=gt,t.Tooltip=$,t.Util=c,Object.defineProperty(t,"__esModule",{value:!0})})); +//# sourceMappingURL=bootstrap.min.js.map \ No newline at end of file diff --git a/imgr/imgrpkg/static-content/jquery.min.js b/imgr/imgrpkg/static-content/jquery.min.js new file mode 100644 index 00000000..36b4e1a1 --- /dev/null +++ b/imgr/imgrpkg/static-content/jquery.min.js @@ -0,0 +1,2 @@ +/*! jQuery v3.5.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-deprecated/ajax-event-alias,-effects,-effects/Tween,-effects/animatedSelector | (c) JS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(g,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,v=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,y=n.hasOwnProperty,a=y.toString,l=a.call(Object),m={},b=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},w=g.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function C(e,t,n){var r,i,o=(n=n||w).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function T(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-deprecated/ajax-event-alias,-effects,-effects/Tween,-effects/animatedSelector",E=function(e,t){return new E.fn.init(e,t)};function d(e){var t=!!e&&"length"in e&&e.length,n=T(e);return!b(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+R+")"+R+"*"),U=new RegExp(R+"|>"),V=new RegExp(W),X=new RegExp("^"+B+"$"),Q={ID:new RegExp("^#("+B+")"),CLASS:new RegExp("^\\.("+B+")"),TAG:new RegExp("^("+B+"|[*])"),ATTR:new RegExp("^"+M),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+R+"*(even|odd|(([+-]|)(\\d*)n|)"+R+"*(?:([+-]|)"+R+"*(\\d+)|))"+R+"*\\)|)","i"),bool:new RegExp("^(?:"+I+")$","i"),needsContext:new RegExp("^"+R+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+R+"*((?:-\\d)?\\d*)"+R+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,G=/^(?:input|select|textarea|button)$/i,K=/^h\d$/i,J=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+R+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){C()},ae=xe(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{O.apply(t=P.call(d.childNodes),d.childNodes),t[d.childNodes.length].nodeType}catch(e){O={apply:t.length?function(e,t){q.apply(e,P.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,d=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==d&&9!==d&&11!==d)return n;if(!r&&(C(e),e=e||T,E)){if(11!==d&&(u=Z.exec(t)))if(i=u[1]){if(9===d){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return O.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&p.getElementsByClassName&&e.getElementsByClassName)return O.apply(n,e.getElementsByClassName(i)),n}if(p.qsa&&!k[t+" "]&&(!v||!v.test(t))&&(1!==d||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===d&&(U.test(t)||_.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&p.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=A)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+be(l[o]);c=l.join(",")}try{return O.apply(n,f.querySelectorAll(c)),n}catch(e){k(t,!0)}finally{s===A&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>x.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[A]=!0,e}function ce(e){var t=T.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)x.attrHandle[n[r]]=t}function de(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function pe(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in p=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},C=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:d;return r!=T&&9===r.nodeType&&r.documentElement&&(a=(T=r).documentElement,E=!i(T),d!=T&&(n=T.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),p.scope=ce(function(e){return a.appendChild(e).appendChild(T.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),p.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),p.getElementsByTagName=ce(function(e){return e.appendChild(T.createComment("")),!e.getElementsByTagName("*").length}),p.getElementsByClassName=J.test(T.getElementsByClassName),p.getById=ce(function(e){return a.appendChild(e).id=A,!T.getElementsByName||!T.getElementsByName(A).length}),p.getById?(x.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},x.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(x.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},x.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),x.find.TAG=p.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):p.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},x.find.CLASS=p.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(p.qsa=J.test(T.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+R+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+R+"*(?:value|"+I+")"),e.querySelectorAll("[id~="+A+"-]").length||v.push("~="),(t=T.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+R+"*name"+R+"*="+R+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+A+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=T.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+R+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(p.matchesSelector=J.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){p.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",W)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=J.test(a.compareDocumentPosition),y=t||J.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!p.sortDetached&&t.compareDocumentPosition(e)===n?e==T||e.ownerDocument==d&&y(d,e)?-1:t==T||t.ownerDocument==d&&y(d,t)?1:u?H(u,e)-H(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==T?-1:t==T?1:i?-1:o?1:u?H(u,e)-H(u,t):0;if(i===o)return de(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?de(a[r],s[r]):a[r]==d?-1:s[r]==d?1:0}),T},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(C(e),p.matchesSelector&&E&&!k[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||p.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){k(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return Q.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&V.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+R+")"+e+"("+R+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return b(n)?E.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?E.grep(e,function(e){return e===n!==r}):"string"!=typeof n?E.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(E.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||L,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:j.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof E?t[0]:t,E.merge(this,E.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:w,!0)),k.test(r[1])&&E.isPlainObject(t))for(r in t)b(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=w.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):b(e)?void 0!==n.ready?n.ready(e):e(E):E.makeArray(e,this)}).prototype=E.fn,L=E(w);var q=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}E.fn.extend({has:function(e){var t=E(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,pe=/^$|^module$|\/(?:java|ecma)script/i;le=w.createDocumentFragment().appendChild(w.createElement("div")),(ce=w.createElement("input")).setAttribute("type","radio"),ce.setAttribute("checked","checked"),ce.setAttribute("name","t"),le.appendChild(ce),m.checkClone=le.cloneNode(!0).cloneNode(!0).lastChild.checked,le.innerHTML="",m.noCloneChecked=!!le.cloneNode(!0).lastChild.defaultValue,le.innerHTML="",m.option=!!le.lastChild;var he={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ge(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&S(e,t)?E.merge([e],n):n}function ve(e,t){for(var n=0,r=e.length;n",""]);var ye=/<|&#?\w+;/;function me(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),d=[],p=0,h=e.length;p\s*$/g;function Le(e,t){return S(e,"table")&&S(11!==t.nodeType?t:t.firstChild,"tr")&&E(e).children("tbody")[0]||e}function je(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n
",2===ft.childNodes.length),E.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(m.createHTMLDocument?((r=(t=w.implementation.createHTMLDocument("")).createElement("base")).href=w.location.href,t.head.appendChild(r)):t=w),o=!n&&[],(i=k.exec(e))?[t.createElement(i[1])]:(i=me([e],t,o),o&&o.length&&E(o).remove(),E.merge([],i.childNodes)));var r,i,o},E.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=E.css(e,"position"),c=E(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=E.css(e,"top"),u=E.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),b(t)&&(t=t.call(e,n,E.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},E.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){E.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===E.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===E.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=E(e).offset()).top+=E.css(e,"borderTopWidth",!0),i.left+=E.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-E.css(r,"marginTop",!0),left:t.left-i.left-E.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===E.css(e,"position"))e=e.offsetParent;return e||re})}}),E.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;E.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),E.each(["top","left"],function(e,n){E.cssHooks[n]=Fe(m.pixelPosition,function(e,t){if(t)return t=We(e,n),Ie.test(t)?E(e).position()[n]+"px":t})}),E.each({Height:"height",Width:"width"},function(a,s){E.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){E.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?E.css(e,t,i):E.style(e,t,n,i)},s,n?e:void 0,n)}})}),E.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),E.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){E.fn[n]=function(e,t){return 0': '>', + '"': '"', + '\'': ''', + '/': '/' + }; + + var defaultSettings = { + indent: 2 + }; + + var id = 0; + var instances = 0; + var current_collapse_level = null; + var ids_to_collapse = []; + + this.create = function(data, settings, collapse_depth) { + current_collapse_level = typeof collapse_depth !== 'undefined' ? collapse_depth : -1; + instances += 1; + return _span(_jsVal(data, 0, false), {class: 'jstValue'}); + }; + + this.collapse = function() { + var arrayLength = ids_to_collapse.length; + for (var i = 0; i < arrayLength; i++) { + JSONTree.toggle(ids_to_collapse[i]); + } + }; + + var _escape = function(text) { + return text.replace(/[&<>'"]/g, function(c) { + return escapeMap[c]; + }); + }; + + var _id = function() { + return instances + '_' + id++; + }; + + var _lastId = function() { + return instances + '_' + (id - 1); + }; + + var _jsVal = function(value, depth, indent) { + if (value !== null) { + var type = typeof value; + switch (type) { + case 'boolean': + return _jsBool(value, indent ? depth : 0); + case 'number': + return _jsNum(value, indent ? depth : 0); + case 'string': + return _jsStr(value, indent ? depth : 0); + default: + if (value instanceof Array) { + return _jsArr(value, depth, indent); + } else { + return _jsObj(value, depth, indent); + } + } + } else { + return _jsNull(indent ? depth : 0); + } + }; + + var _jsObj = function(object, depth, indent) { + var id = _id(); + _decrementCollapseLevel("_jsObj"); + var content = Object.keys(object).map(function(property) { + return _property(property, object[property], depth + 1, true); + }).join(_comma()); + var body = [ + _openBracket('{', indent ? depth : 0, id), + _span(content, {id: id}), + _closeBracket('}', depth) + ].join('\n'); + _incrementCollapseLevel("_jsObj"); + return _span(body, {}); + }; + + var _jsArr = function(array, depth, indent) { + var id = _id(); + _decrementCollapseLevel("_jsArr"); + var body = array.map(function(element) { + return _jsVal(element, depth + 1, true); + }).join(_comma()); + var arr = [ + _openBracket('[', indent ? depth : 0, id), + _span(body, {id: id}), + _closeBracket(']', depth) + ].join('\n'); + _incrementCollapseLevel("_jsArr"); + return arr; + }; + + var _jsStr = function(value, depth) { + var jsonString = _escape(JSON.stringify(value)); + return _span(_indent(jsonString, depth), {class: 'jstStr'}); + }; + + var _jsNum = function(value, depth) { + return _span(_indent(value, depth), {class: 'jstNum'}); + }; + + var _jsBool = function(value, depth) { + return _span(_indent(value, depth), {class: 'jstBool'}); + }; + + var _jsNull = function(depth) { + return _span(_indent('null', depth), {class: 'jstNull'}); + }; + + var _property = function(name, value, depth) { + var property = _indent(_escape(JSON.stringify(name)) + ': ', depth); + var propertyValue = _span(_jsVal(value, depth, false), {}); + return _span(property + propertyValue, {class: 'jstProperty'}); + }; + + var _comma = function() { + return _span(',\n', {class: 'jstComma'}); + }; + + var _span = function(value, attrs) { + return _tag('span', attrs, value); + }; + + var _tag = function(tag, attrs, content) { + return '<' + tag + Object.keys(attrs).map(function(attr) { + return ' ' + attr + '="' + attrs[attr] + '"'; + }).join('') + '>' + + content + + ''; + }; + + var _openBracket = function(symbol, depth, id) { + return ( + _span(_indent(symbol, depth), {class: 'jstBracket'}) + + _span('', {class: 'jstFold', onclick: 'JSONTree.toggle(\'' + id + '\')'}) + ); + }; + + this.toggle = function(id) { + var element = document.getElementById(id); + var parent = element.parentNode; + var toggleButton = element.previousElementSibling; + if (element.className === '') { + element.className = 'jstHiddenBlock'; + parent.className = 'jstFolded'; + toggleButton.className = 'jstExpand'; + } else { + element.className = ''; + parent.className = ''; + toggleButton.className = 'jstFold'; + } + }; + + var _closeBracket = function(symbol, depth) { + return _span(_indent(symbol, depth), {}); + }; + + var _indent = function(value, depth) { + return Array((depth * 2) + 1).join(' ') + value; + }; + + var _decrementCollapseLevel = function(caller) { + if (current_collapse_level <= 0) { + ids_to_collapse.push(_lastId()); + } else { + } + current_collapse_level--; + }; + + var _incrementCollapseLevel = function(caller) { + current_collapse_level++; + }; + + return this; +})(); diff --git a/imgr/imgrpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css b/imgr/imgrpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css new file mode 100755 index 00000000..4664f2e8 --- /dev/null +++ b/imgr/imgrpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css @@ -0,0 +1 @@ +@font-face{font-family:Icons;src:url(../fonts/open-iconic.eot);src:url(../fonts/open-iconic.eot?#iconic-sm) format('embedded-opentype'),url(../fonts/open-iconic.woff) format('woff'),url(../fonts/open-iconic.ttf) format('truetype'),url(../fonts/open-iconic.otf) format('opentype'),url(../fonts/open-iconic.svg#iconic-sm) format('svg');font-weight:400;font-style:normal}.oi{position:relative;top:1px;display:inline-block;speak:none;font-family:Icons;font-style:normal;font-weight:400;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.oi:empty:before{width:1em;text-align:center;box-sizing:content-box}.oi.oi-align-center:before{text-align:center}.oi.oi-align-left:before{text-align:left}.oi.oi-align-right:before{text-align:right}.oi.oi-flip-horizontal:before{-webkit-transform:scale(-1,1);-ms-transform:scale(-1,1);transform:scale(-1,1)}.oi.oi-flip-vertical:before{-webkit-transform:scale(1,-1);-ms-transform:scale(-1,1);transform:scale(1,-1)}.oi.oi-flip-horizontal-vertical:before{-webkit-transform:scale(-1,-1);-ms-transform:scale(-1,1);transform:scale(-1,-1)}.oi-account-login:before{content:'\e000'}.oi-account-logout:before{content:'\e001'}.oi-action-redo:before{content:'\e002'}.oi-action-undo:before{content:'\e003'}.oi-align-center:before{content:'\e004'}.oi-align-left:before{content:'\e005'}.oi-align-right:before{content:'\e006'}.oi-aperture:before{content:'\e007'}.oi-arrow-bottom:before{content:'\e008'}.oi-arrow-circle-bottom:before{content:'\e009'}.oi-arrow-circle-left:before{content:'\e00a'}.oi-arrow-circle-right:before{content:'\e00b'}.oi-arrow-circle-top:before{content:'\e00c'}.oi-arrow-left:before{content:'\e00d'}.oi-arrow-right:before{content:'\e00e'}.oi-arrow-thick-bottom:before{content:'\e00f'}.oi-arrow-thick-left:before{content:'\e010'}.oi-arrow-thick-right:before{content:'\e011'}.oi-arrow-thick-top:before{content:'\e012'}.oi-arrow-top:before{content:'\e013'}.oi-audio-spectrum:before{content:'\e014'}.oi-audio:before{content:'\e015'}.oi-badge:before{content:'\e016'}.oi-ban:before{content:'\e017'}.oi-bar-chart:before{content:'\e018'}.oi-basket:before{content:'\e019'}.oi-battery-empty:before{content:'\e01a'}.oi-battery-full:before{content:'\e01b'}.oi-beaker:before{content:'\e01c'}.oi-bell:before{content:'\e01d'}.oi-bluetooth:before{content:'\e01e'}.oi-bold:before{content:'\e01f'}.oi-bolt:before{content:'\e020'}.oi-book:before{content:'\e021'}.oi-bookmark:before{content:'\e022'}.oi-box:before{content:'\e023'}.oi-briefcase:before{content:'\e024'}.oi-british-pound:before{content:'\e025'}.oi-browser:before{content:'\e026'}.oi-brush:before{content:'\e027'}.oi-bug:before{content:'\e028'}.oi-bullhorn:before{content:'\e029'}.oi-calculator:before{content:'\e02a'}.oi-calendar:before{content:'\e02b'}.oi-camera-slr:before{content:'\e02c'}.oi-caret-bottom:before{content:'\e02d'}.oi-caret-left:before{content:'\e02e'}.oi-caret-right:before{content:'\e02f'}.oi-caret-top:before{content:'\e030'}.oi-cart:before{content:'\e031'}.oi-chat:before{content:'\e032'}.oi-check:before{content:'\e033'}.oi-chevron-bottom:before{content:'\e034'}.oi-chevron-left:before{content:'\e035'}.oi-chevron-right:before{content:'\e036'}.oi-chevron-top:before{content:'\e037'}.oi-circle-check:before{content:'\e038'}.oi-circle-x:before{content:'\e039'}.oi-clipboard:before{content:'\e03a'}.oi-clock:before{content:'\e03b'}.oi-cloud-download:before{content:'\e03c'}.oi-cloud-upload:before{content:'\e03d'}.oi-cloud:before{content:'\e03e'}.oi-cloudy:before{content:'\e03f'}.oi-code:before{content:'\e040'}.oi-cog:before{content:'\e041'}.oi-collapse-down:before{content:'\e042'}.oi-collapse-left:before{content:'\e043'}.oi-collapse-right:before{content:'\e044'}.oi-collapse-up:before{content:'\e045'}.oi-command:before{content:'\e046'}.oi-comment-square:before{content:'\e047'}.oi-compass:before{content:'\e048'}.oi-contrast:before{content:'\e049'}.oi-copywriting:before{content:'\e04a'}.oi-credit-card:before{content:'\e04b'}.oi-crop:before{content:'\e04c'}.oi-dashboard:before{content:'\e04d'}.oi-data-transfer-download:before{content:'\e04e'}.oi-data-transfer-upload:before{content:'\e04f'}.oi-delete:before{content:'\e050'}.oi-dial:before{content:'\e051'}.oi-document:before{content:'\e052'}.oi-dollar:before{content:'\e053'}.oi-double-quote-sans-left:before{content:'\e054'}.oi-double-quote-sans-right:before{content:'\e055'}.oi-double-quote-serif-left:before{content:'\e056'}.oi-double-quote-serif-right:before{content:'\e057'}.oi-droplet:before{content:'\e058'}.oi-eject:before{content:'\e059'}.oi-elevator:before{content:'\e05a'}.oi-ellipses:before{content:'\e05b'}.oi-envelope-closed:before{content:'\e05c'}.oi-envelope-open:before{content:'\e05d'}.oi-euro:before{content:'\e05e'}.oi-excerpt:before{content:'\e05f'}.oi-expand-down:before{content:'\e060'}.oi-expand-left:before{content:'\e061'}.oi-expand-right:before{content:'\e062'}.oi-expand-up:before{content:'\e063'}.oi-external-link:before{content:'\e064'}.oi-eye:before{content:'\e065'}.oi-eyedropper:before{content:'\e066'}.oi-file:before{content:'\e067'}.oi-fire:before{content:'\e068'}.oi-flag:before{content:'\e069'}.oi-flash:before{content:'\e06a'}.oi-folder:before{content:'\e06b'}.oi-fork:before{content:'\e06c'}.oi-fullscreen-enter:before{content:'\e06d'}.oi-fullscreen-exit:before{content:'\e06e'}.oi-globe:before{content:'\e06f'}.oi-graph:before{content:'\e070'}.oi-grid-four-up:before{content:'\e071'}.oi-grid-three-up:before{content:'\e072'}.oi-grid-two-up:before{content:'\e073'}.oi-hard-drive:before{content:'\e074'}.oi-header:before{content:'\e075'}.oi-headphones:before{content:'\e076'}.oi-heart:before{content:'\e077'}.oi-home:before{content:'\e078'}.oi-image:before{content:'\e079'}.oi-inbox:before{content:'\e07a'}.oi-infinity:before{content:'\e07b'}.oi-info:before{content:'\e07c'}.oi-italic:before{content:'\e07d'}.oi-justify-center:before{content:'\e07e'}.oi-justify-left:before{content:'\e07f'}.oi-justify-right:before{content:'\e080'}.oi-key:before{content:'\e081'}.oi-laptop:before{content:'\e082'}.oi-layers:before{content:'\e083'}.oi-lightbulb:before{content:'\e084'}.oi-link-broken:before{content:'\e085'}.oi-link-intact:before{content:'\e086'}.oi-list-rich:before{content:'\e087'}.oi-list:before{content:'\e088'}.oi-location:before{content:'\e089'}.oi-lock-locked:before{content:'\e08a'}.oi-lock-unlocked:before{content:'\e08b'}.oi-loop-circular:before{content:'\e08c'}.oi-loop-square:before{content:'\e08d'}.oi-loop:before{content:'\e08e'}.oi-magnifying-glass:before{content:'\e08f'}.oi-map-marker:before{content:'\e090'}.oi-map:before{content:'\e091'}.oi-media-pause:before{content:'\e092'}.oi-media-play:before{content:'\e093'}.oi-media-record:before{content:'\e094'}.oi-media-skip-backward:before{content:'\e095'}.oi-media-skip-forward:before{content:'\e096'}.oi-media-step-backward:before{content:'\e097'}.oi-media-step-forward:before{content:'\e098'}.oi-media-stop:before{content:'\e099'}.oi-medical-cross:before{content:'\e09a'}.oi-menu:before{content:'\e09b'}.oi-microphone:before{content:'\e09c'}.oi-minus:before{content:'\e09d'}.oi-monitor:before{content:'\e09e'}.oi-moon:before{content:'\e09f'}.oi-move:before{content:'\e0a0'}.oi-musical-note:before{content:'\e0a1'}.oi-paperclip:before{content:'\e0a2'}.oi-pencil:before{content:'\e0a3'}.oi-people:before{content:'\e0a4'}.oi-person:before{content:'\e0a5'}.oi-phone:before{content:'\e0a6'}.oi-pie-chart:before{content:'\e0a7'}.oi-pin:before{content:'\e0a8'}.oi-play-circle:before{content:'\e0a9'}.oi-plus:before{content:'\e0aa'}.oi-power-standby:before{content:'\e0ab'}.oi-print:before{content:'\e0ac'}.oi-project:before{content:'\e0ad'}.oi-pulse:before{content:'\e0ae'}.oi-puzzle-piece:before{content:'\e0af'}.oi-question-mark:before{content:'\e0b0'}.oi-rain:before{content:'\e0b1'}.oi-random:before{content:'\e0b2'}.oi-reload:before{content:'\e0b3'}.oi-resize-both:before{content:'\e0b4'}.oi-resize-height:before{content:'\e0b5'}.oi-resize-width:before{content:'\e0b6'}.oi-rss-alt:before{content:'\e0b7'}.oi-rss:before{content:'\e0b8'}.oi-script:before{content:'\e0b9'}.oi-share-boxed:before{content:'\e0ba'}.oi-share:before{content:'\e0bb'}.oi-shield:before{content:'\e0bc'}.oi-signal:before{content:'\e0bd'}.oi-signpost:before{content:'\e0be'}.oi-sort-ascending:before{content:'\e0bf'}.oi-sort-descending:before{content:'\e0c0'}.oi-spreadsheet:before{content:'\e0c1'}.oi-star:before{content:'\e0c2'}.oi-sun:before{content:'\e0c3'}.oi-tablet:before{content:'\e0c4'}.oi-tag:before{content:'\e0c5'}.oi-tags:before{content:'\e0c6'}.oi-target:before{content:'\e0c7'}.oi-task:before{content:'\e0c8'}.oi-terminal:before{content:'\e0c9'}.oi-text:before{content:'\e0ca'}.oi-thumb-down:before{content:'\e0cb'}.oi-thumb-up:before{content:'\e0cc'}.oi-timer:before{content:'\e0cd'}.oi-transfer:before{content:'\e0ce'}.oi-trash:before{content:'\e0cf'}.oi-underline:before{content:'\e0d0'}.oi-vertical-align-bottom:before{content:'\e0d1'}.oi-vertical-align-center:before{content:'\e0d2'}.oi-vertical-align-top:before{content:'\e0d3'}.oi-video:before{content:'\e0d4'}.oi-volume-high:before{content:'\e0d5'}.oi-volume-low:before{content:'\e0d6'}.oi-volume-off:before{content:'\e0d7'}.oi-warning:before{content:'\e0d8'}.oi-wifi:before{content:'\e0d9'}.oi-wrench:before{content:'\e0da'}.oi-x:before{content:'\e0db'}.oi-yen:before{content:'\e0dc'}.oi-zoom-in:before{content:'\e0dd'}.oi-zoom-out:before{content:'\e0de'} \ No newline at end of file diff --git a/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.eot b/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.eot new file mode 100755 index 0000000000000000000000000000000000000000..f98177dbf711863eff7c90f84d5d419d02d99ba8 GIT binary patch literal 28196 zcmdsfdwg8gedj&r&QluAL-W#Wq&pgEMvsv!&0Cf&+mau`20w)Dj4&8Iu59zN6=RG; z451+<)Ej~^SrrmCp$=hb!Zu?PlZ0v^rFqOYfzqruY1s`+ve{(Uv}w|M+teR4-tX_6 zJJQHDgm(Majx=-5J@?%6_?_SRz0Ykss3^zpP!y(cg+5#{t0IGvlZlxgLVa!|Pwg%0HwaAkJPsR_7CkF z{hz=5BS2$bQO4>H%uMR+@Bes%qU=0}`qqrY1!(P0t>lnf>u?>hCHF7DiD%jIRLs_gA0(b1L}rzgltYVrt?gc2Y5;9UDjQ z%B)P;{Yp$h?WOgkCosju&-Q&Abmg0GDQ~^0YA77V?+nuN;!-_LToFFdx5>D-3RhIC zNim@Y28=&kzxC#&OZZhTUDD)z++voc1{on3eJelI&j0@(PPn1`HTMH@R>gMK0^H#} z-APZ<6H9s`4L|t$XFtpR3vV~DpGXL)8ZghQI8nFC#;Gm~d%|gaTbMPC42!c1B?miM zn$?TN(kwg4=NH!N?1DZwr|Va=QM0@at3QmtSVbGuP_f*EuIqDh*>o`umty&fMPWVN zwOSy=lGa!#OKqKlS=4KL6^YiDEHv;MA!Dj|%KqdbXOLRkVPgo+>xM z`tdLxr03~jdXO4;l(4}>Kca7fS2gy1&DtubqsnG6amCcr?ZNni_*#ur)!una=lO+a z(W#N+^Oy#G-fw#XCIlD!Q7hD3IjwB$Uoy5LHCCk7M6R+q+PRlLC+2F#Og&0KX;fTm z9gRV6t=nO-P_Az=CG4l*~#0dwv=AFvG8)~&n&z! z>wcqjdUo&ccd;$(NdM=j`265c&L?J1yxG?F>}_{_wry>?^aan|yPK}R#cpg(b^$xz zf;Gl2?&aw=%jBtFht&{S}(z)fW6^mCJSIuQ@i4|p+ zx3$z#v51krkNGj$t;x!E@Z?f6a(ZZoC>r5@Ucl5$FlAy4?Q*}B&hb1!m&U%lE*Euc z#N62h7Dtl~c7f-y5Wr$VDS7_#wX$QaKmmSK`iqLyDz`g-`54&Z80Kl-ofTt{b;TI$ zT#%ThARiNAa&`dV8`oF>zV?w_b1QPe8_mRA%fyml9N}zE z_-m(6zyG|m?j+Mnf7=xbb%mHqB&x=o>~}ut(o3hDKA)2v)LFgfzUPV|zwQq${}Jm! zdvqS0#f$auxa~yCyx|1clRx73VPI)bD(DG&?EH&%UAHgnwu8I!`Kp(SFWc>Wqg^Ma zTe*j+Ez4Kzf`(q!&Qco{4bZc|i%U<6aYU6B7)Lx7;53d@W>5_ia)5Ny1_i;Fuu5e! z-gKnZ5^0T^BYvyJ8eYL}Z1AdPGrK^uOnkDgwNvdLC@Di@t#zMFFbngC*yBaZnjCxO zZVNwAs{vvUm;SyZn;h!w92-hzJ6O%btT}YL>chAEtV)iFcrVtkM#9EvCDS2-twqu&y5y= zw;q?%OgQCDn!(c|X=^MS%LcRltks{LOR&8^`AO+?V#}7fxh-2D&&;XX#mAnwc+n^T z?I3bku^;?ONNGpAEzQ9|wZK)t4otF{`3c3+*b1IhG!ph>Qy^76GG!OWj>gw*J9S{; z4GguD#dS*bxuJZ1h^DeJ+j4C4fm1qeo$MT>2@;LZAJ13vO*7V9&^G2tG7zXZ?FfUm z#SMB%w5<{KY9(%XvO$a>;P-@EExte!yNWhJc8Fzlj6qNMLkn-vTJq?^8$)^3(jB7q zK=I-s|H2zsK0QCgqux+AWHJJLC*aI54Qv=}8o8CR zZwEnEGeI;95)@8khtt_i7IdVSr-7d=zV}u=kyugRRIfhw zeDDVL_QJF74|wmnm%D6ymv^z?^V}7hzydG+3&|d1l55zYhOj3av4&o`Cs_*%Sec7K6kNmX1R1PD zYix+tfd4N`+-xrWgR9=NE#s(Rcb7VHTc13*dDZG`u2Vy5+-xoVUX3HO%~S7URi&d_ za|fSnjU2xwx0TQZaKH4&{58k8C}uC~%bS*!t{HKh8i(U_G87Y4V6Mbq6(WCwXB8|!8EMz7QHK&Z*mcFpc< z+RRN&4^&tAL+^tIcvp=oXtiyp&{<>WDx_onB*c$TJG+1&G7a-fJb(lhUsyZ?n4aYuiGF!~%5BNht zkLp&(Oy-jvTIYsHHM$C!I<(f1-`DJlUJRPI*qqTW+kTY1z~}7?FWT8-kChzvs)6UdU2dnB zx$Q4tyPa>#r3G#wn2l*V56=aR2F{ncODvttVSQ>#9gal)dghYmi{bh)=H+FHv=R)hRtN(5RM_@E0? z5kM8i9$Uerye_+vY3w_3_P#}l!_lo1O@m<2iy=ee^_*n$LO%GqY8Q0?Zgjgfu%~GcgW`lM%ck$vJ0hs4ShNL&iUr07ttjmJdpcTs@YpWWi zLeN`YSMXY|ok4QJ?b0l&5gLe$Y$tuGLVQ^KYqd>=*0HTNl+kS35%>Tm0`e`E!ED_IcN2j(%)=h7jWUMUO0+h zRRdK=F-j8tO~s;7T+L5ZJE`9#xx)%NSO@&}!yd9s-zo3*_M|@$v_@C3vckh1zbO=c zQz)I*Tce|GeeMd4hi+VZwk!ITF`O4lyst z4Y9otCo>pme1^Sp;8gd3{bk67rC&829rHZ0Sv4^W_lM?+#W|mfdf9!dfV9s|K;O|StI2k1ficm_+HH-M&Az?i*JgaZ@5^* zE(GBy_gO3&{S94&SP6KeFT!J~`_y882z_O7zCy_m6O~Qphe|_ZM`==gUbZ=u2Swa{ zc-fe%m1d0D?+|)|HxUHK2lEHO%w;$(wR`cy*WG%iYh_pcDb`1TTj~Ka=bd}qEvd|b zQ^m{sB3zJTR-u==fD1KM#C|~QSdzg!U=2oM?a81uk|lZ~xEUA=&kOD%%>%Gb(5GU} zTOiHa&bDc8$;Tnw1g$O1?*a*kxmaWcc5HS9ORvEu4`$0U9^0!Yn(iJ=IPSjNkr=(Z zDY5+W^zl3}LDjB$vt0K9RLLL5oR)B01*NRQyg(`CyrhZKYKCkpBzcJRl8dOC)PO3V zwaRCOc~t7^!d#+yVgv-}OF|o3m8R8-X8{D#>>(A*N?k%eEp2Xp{Og1~APhL#`%a==_CxDO?0Cstm3 z30%#eV0U(fut|VC7qL}fR)`ZvgHV2zC*{}rc8UrQR$o+3OBx1mZ zBw=TjS?FXCbR;9PLY)=VCY?28(R%*NYUev|5yJtCsjYSrP2lsA^AtqzGR9J<&#=SZlzmY*a6=bs1jPR3mA)Spy%lFF5 zROWpz3sBDaoT_RIIQP`UxG^?pxxq~=8DPB}F$ARVc7;st8!RO5cGmB4ZoCptXt$F* zCv5*@5{La6dkp?4(js8{AS3-dZwU(s)Cst!XwFM`ri$l@b{jSbv$P3IT0yOVSP=dS zw*x&V*WCoyCHggs=e+QPsqGa4jr6auy%nO1Ao}q)D@u%U$o8tSy3nH?Dvbl+CYu7R zr;${9Fe_A8p_~#-b)dOUM&F@rV13*8{M%o^J~;k`hJ4<8%LsADky~hvVqJxtWL9i& zd%G1Mt!u5vSyM$+o%}ek3E&T+d^?dS@rBYBXD1idLoy_TzhGTt(IHuqpa=xQPQX9) z0h)5@Nist!gP>qOtZ~ zMv}`QE9zVNwYYBcTms~PKGwK=(ESy}0lC<7k|w5-tgTAbC1>SlGFV{0;z+^k=% zP^`6tvGjFXO#;T4IOYvy2(y&V4OomZUoa&6Vs1-oEuS+>A1T9w;)~}99&%k-92Wn0 z#WQ5b|rc;Pr&qX~%&%}F#z(-avRX_b{G<+PY*7c;v8*q~hfsmb>XW+&kft>v*aLckMzT1J z?H52T$v0c|wF=q6AAu|`zT{OizHk$e;I$04CdhHNvo^$$PQGVNwOorbI=H7r;%%PvE>$cds9X%hLl`MJ6ID0UQ$ zMeHT$iSw|nEZP>KML>Fm^x}gE6TyOH{baI=g|o?MIs%(H=}Lgtd<{kFSU|8gs^G;wS0(6~;HoUQld?%1QRZPOq4L+V$^Kce3< zza;Al%6f$Xs zJ(ifhc0+%g-EIkP+x_5%O&`B;lgFbvI(tX2(;pCqr(#uYQ^?=!6x^22htq48xpO$v_M&$&HhkRZI$5SG*{TDTls&4?T2*ow$^%;=-wcMati4n z1CHQ>9wQCHD;N>p7-?idNGxoNs;bt2YwvLPeckc+x|?c4{(9F?>4DPUv%A;0{U0rT z_kOmD&oj?W>$p&VVcQqtdrO##R}$gZvxB^K55{&58Yt zJxOe?lC{aLO=P4@bLhDSp?60bYv?&Ikwm8{*lPk&G^LoJkdZLui?+rM>F(~;>w2o| zMK;_&(66yNkzdnZIw!7G&E(FlJ&^0YY17!o8++wN$M&_u>xQ?M7Ubo=DWd@UWC>?f zaBRpICMlP|)$9eavi2=$}kiDm__jweO@3rN;(HfCW16c9Drzu=v&AdeV|?K z)Hl>6;GWe_22rqia&JR(5=A5kv`TN7kZQ7Nx(gj9+tU~<`a?Zgk%=6%J-S;Vf)l z0Lt7Py8yV%l2=b$%8RSCQEe5x!D~D$o5J(-tk}HN7&Sr#rE{V&8p{&>vO=@mh5fr@ zQ*622sGaQeFjBNykn}REr5UPzt2F@U1^%tXhqD=YE_!)(NR36wpAto)W}`tTHWeJ$ z>Kc}gmd$AFZ|-gi@CbSTFbq6RJAy4%%b{gEY$%uTDdmFttp;N%I-l% z_DCo&{xE-elH$n7{aCg!AftazXDcW*!Ul!TUdgkhUm~V-!*`ujvXDvFDD7)ohgPl3 zWm1X0-gs9>w5?TZZfdBjTAsney4@_8{!`-jJF=) z!Ih4dvLfo`b6!xSXZ<1gZ}Sax-i2Gee9%xRy`{56px72K`EN^adc9{21=65bkhPMa zR}Dn3Al|?mA(VFLEopIu&Y`6UD>6tJS#HW#Rgp`MU*q7S=7Roe3s? zbg=ZL(wEq2hzDcPE1w=LJ;!!djFtF|h&6!Q0rm&jArNo?F@_L_;&0BWr8|IO@M|p5 zV^z@OMSa^7_Ik3gs==b^kpd(=UXG#yyApH&grKsGYS>(CXI*eP5|0)*5;5XqlEGv) z>GAT5Uhjg%i|r)ZqCAxW=_qVL;vCo@d{ur$1HGvFS~T1cs1i7rfLDhc3FNwt#^9_X z`3W{;p$@^_j3^24E}?yX_{*-JGFZvcEqWTGQ3FhTSQW5DIvH?aGyF zk3DtFNc2_PSEc&;QuIYu!pDfmBKavGX=2$iW)X~27!K12bis%qj}Q|O76PUUm*Ff- zh(K=yW32f=f-Gtf8ik+mT7n?g`{Fb;KX*699YJse1^RPncoAwWVN!L?8DcsO|&<8t7Kdq z`Q9J`nkB+!vSBC#S1)l1?-teTmXcyN2z!u8TG~Z)8QW1+P4O3{b27q$os{tyrP<}z zx7OA-`w?YU^oCs3PI!_{W{^hEMU?qN`~?|#F(>0GzkJ~2VzhR7p{k1)r2?m6sBWH{_0ElUbM_IgNLK-IGf3H)siHZ*NlW8BqDLfvrrdWs4Q)9dtse@ zdgUjCVS;eqtTrRor(4+x+}wGcodNd|HfhW?)@zo&Kqz^^fH7$!vL>6cBDm6s!HHpl z#=MPK9r)$MtSMq*b3{&d=aeH*<1sr~L&)!RxEiuaV}1e(iF*QComGb3c$)@#%l813 zpfU5g?P{nz=baV?-BPtdTWz*ha}(MUGZoWM{SRhCnFzkYoX}SJUdUO7!Q6JDaqr(o zLb8vfcTx_Lc_9mdGtxeS>Lq@OQ_38%N{X~2GqXscyW%7GGs(zgkD-Vgl572IYkT7z zkYbx4!@3a-Yf@}N*%Eqw7JY+R{MNh>gF=GJk+TUtTB4p;&mta7RDt|*^%O%D@{~bW zj5rfJQ`?DTU`|A(F)!2;bd*BO#H?&*-40?SRIJPwWee=&%AG603XhI~c)|FF{nSOFGh!?# z$5_gC)e2iJoat~E2P2Di)sxrX1@%rZu%q~ai52n-sVc2aS;J)k-@p zd;{Wy3fO83T!q5&L-ERaY7XE@%u(n#W=fLr#fwEffiJ}Ja(e<+LE<| zAKks(g4^Amu2r=T-DK~?6Q#RO-ipICub*04fAsAZ{tmxK*q(*0z{wFf2t!Mmg~HS< z>`uZ0#bj`lsuhmsPTqG=(;VIR-t}1S__ab%HRvO3wh`Qv~V zG&_H|9c+aQBq1r93w9*CE!)muNoGLTzeVug92sfn5XkrE$Maj-qZVJPLz8<%)fWDT zYO|`pyy$C&v*cMl#O}-w#qaIxfR$|J=B6QX#Ts!(SZYHyqH|Va4G|3|{NW@V%W!qt zet-|{BU!&P7E4MthFhYdjup5s;)wu1vE>0W{6qMs6irp&xM52#`!HY%^9b?-BDCbe zxT3yEmE)D3l9RN7s6GvaZ1A$ap@)-g-y;2CG(Ru%Kn)<@5P3$(YF{3Ys4sm1mF*`z zWJN{{f4O};u>=p;jThsI!xA9IeMQin>M|XGoeaHWV?;bj0bXenCTp2cMTEYoihVET z)k=SXLAtLHE$8)bgCWbk^CZ^uo50^ynC}X|!3)9CL!8!NHBV)%i$OWY;Q<)FNR5Mo z4G0$|PZum+RFegqHeo^SJ!b+lN01IFab2NDZcAX#&JK1aZhOSX=S_p1CPXYFPML>S z{t1QZBuJ+dieKX3Gqtx4c6JWlTKmkwgbd#yxGnlb7U3qvWdPWihk${mv|%2t;aZ_f zErt@qWwkU`(l?~sxh#bEA_&UDvxt>Oe1dPg3>+>wAcoRtAd+J3N%#cL(0DFAuU26n zES^bVhJ{)vSfFOi9XS8Yx-}iIfApF2kMsF8>z+9uIQIDYXFmEm@P_a}#%Khw&JNO3 z7{ZQ{X%IssbOJEqkCBHx!uFCK4rEXK<44fI@&%>k_5|L9(4Jeg2hEx^JvcAZChO9L zXUGK8BgJV18%zJ^ca5CMmp}G1PyqzQqs0E2t*dmW%(5p;&en#281ton$6v&pbEmcw=4n?au4S-Sy0OJ!_)R437?}-km!s`%H9AALC89lE}Q4u=a{lsF?svCed+$tOaa z7j01y!_E-)lp}n->@^&SN_b&c_#Gi1sao0GfB+13L7b4F;FcvjFxlAyXuB3Cz*OnS zLFh&Xup&LLHOAWIaWJ;Gp|13!8P;+CbFV)7;c4bB?f;u|8Jq=COLwx){kM8wdEn7k zcQE%~oIlrf&ql+pbLmMzUxg2m>^jTN?ub3@vBo@-2+8o<8-?zdFfJ=@giXjUz22DTppvsdH%LW6F|Deg9C$UdSM+ zp7x>W(CDkBH(v!RK|E#3)|M^z&|%-f{gIZfE&V6Q9)0!IN5@WzQ~pb9rV1&%>T3ZX z`D6q>&~aZGYfl21IG+XS6HKNw`!b@b?0XiT-D4M*6e4FY{oGzG+F64gv%yqkd`1Ny zq8KZR&sg-iQhbIXD9|A=I$A3-(&ZcZ!(Y^Fjs_FH{2%G9mVVYK`jKbF20-6h3|u3L3WtCZ?%+>khd2<9P#On9qR?tn zD3Q`R#3ncc!J<>KUS1s7Jz#gM>M!5}2?cAq2L`%pf+4FV@C#LS+sik_1<$|B-OC^4 zc~K&91~DqX1|25-$#%9k?h?EXv{($)X`)ya*weB@HV~>Po#eq8OdMbMCb%Whq zt->d?0gkZ?msD9O$U4ug~o53-O@Y zXY)D(L1$-uYkOUfV_X05!g^AJDrjj7EYO>jJw!`)Ub{9IZ>u7C6|__a{914>6a(r- zAdQtqM)(Y;zq%x0Tq$!HCGA(#kukJu`aN5E8$&hQ_ie8UH4b#7DV(;!5I-P$_+G5Y zv(FmA!*rt@$D7<<)0J}cuUXUYXkB@&h#z*4P$JCDMPmANCCx6lGA+BR*!x7Igsq!& zng~K&B|pbm9V?97=_G<(fuzEJJcu|49L9g*%a%Z~Sl_EX^8~_w^k+V=>UyvC#KSEs z5Zw;m{_<-o@%`vaFGcm&URL$!^UuTMWXKPK-uM^!eL^_$094|_*&whq>dvr}r|-VI zbncGvV~A$?O@8#qvtM}oZA8yf*&c}1D4`gv zO6G7O=P!87;&V8M?59KS=?E0SB7G~Uo{)jDpY!ktmHUC9gJandKaOyhDJ8*2JWXR; zqFYsXfeG=kfY(_q&NzA!ra&#WB5#Wz{F=hdkYX#IW}QF$Nb#xCUqAgCix$6p@7Pfc z;v+vS{pj@5%=eUDdgHZwzpNjH=DZ{aRDohqOagFMYYO@(FbTNpO_-?tUXFIb(H1*E zM`hE5{t_FW*KdC6zu)uF&mYv!KO+?APQyexUwY}Kd;a@VH|r1n{Gn&gOJ%!kC>3&` zSjRA6;Sq9MnD&ZP`jJv3l(dveW`K|@a{7}r4HRZ4Ni8Pn6tPJ#k9QV@o%CYqoRF@? z1&?-$bD~@TlI#PuIM0a~cyE=U8=wl{QDu`X+%lOkp)WQl+y+~I0)nr{TS`MM@i?dG z!Hu`OJ#Re$k`3kjUKFk-)zFzjPXGpqjQ0<5BRHvT`n68n1WDt$)8LXx794u=Jl9inhOTl zy4*tU3>eu#sT3Fv|_Nmk$>MddiLLcl?ftEQR)K?w&D2nwZuD7ZAh`NI%oX?s8k zMEAs_A-z8f?rCt%O1ysWHp@C9+BVuO+wo}IE^kwuTNAvv^5k5M&d#;BEuEgT8fWL0 z9aW)2tK^1}=hl|eE&K$b(ZW&u=HSjE^TXmVpU0gy%4kL=MS`L6Q%MJjmI&Jc^M!YV0ahT)5@ za9#<`svH+wRt?I;;PUeFb@@K~un?<%EPlC1B&DB=kR@r1F@m%gzFk>ER!6uB6>bv0 zWamU)Sd3)3EctQeU6GgcQ{XzSTRrG!5QiMChEIC=GQpYzT>vrtt^61r^j~-gzuVb` zAFm8Gt!h#=l(bPf|8ICxfYb;QiA3f8HDUKtEU^)LXy>qjibDbva|2t8qkJY%y!_+> zo&3h>Kcexv;0qLkSc@^b5Q8Z62^{^lvUdE$vSn);tt0S$=Tk_x-d*aFu!0Ro-Y9Op zM;sS`p0Y&W%WI9jRbE%@t+Ie$Zn?Z(pg^bE9+ zJX1I?X2i=u$_Bkf#13LZ;3nn>0eJ#+fP`L91YozIt)D|_xuBB&(Hm_1fDOI8MxOB( zGCOz#C^sFg!x=PeGCKZ1Co<gp2|!4jrbaSO6X!>?9ULbX+xTXvAmyQl}9%v~VI= z3!M8u(_J*DN5n14CUSX+?wpH_?oUJJiCINd(OXJh+ks_BR}#7t1V)I&!e15kkn~O@ot<>Ic)hij70o`d z$5cbTGh8|yZ?ffvN{0daPq(P5rQP=gIt%$7Pi?-Yg`I4&9r$qRpXgL5=4R-lEwC5Z z&PKGL;Guw-I3Xv6FR~bjNJXixr6V{?EQ}zK$$_4FBGB5oLYR=u#~x_PWUkePBgr`}zS=;U4%-t?Dj4?Q=CpUG}+675F7%!W>pkV-far zsGNdN2rIgXFUF}%kaB517sm6;&K|lz0Wlx9i0PzofhBucDgzcs`!|g>Tuce$Fc-)k zK!Nqpt_MFS-1Q(hI@u3M8X?0O+3IDm2HU%sVg<_U2YyKyZ9D6$#d$%&>K6MTM2V(V za47Nq3y5op{f}XPEUYJ0mqZ+5Rbxjf%)C+$0ZvpyN{nDm*z3`@P@M;xMetFn;L>IZ z8wblNZ?4Fbzl#nlzhLK+A}Re?Cc^K7lh&nXoMQed0&rwnBu$v~U^qVr|Ce~Aq&Fl{ zc0(%yk6aOtwY4-g7(9i}m(#l)psZmmBE>jlN=z9d8Rnlx%+s>8>a4xUr|?sHlYYdg ziWn^jq5W)?{KY6=#%omY)$MzrwCg%u(OG$<7^6WG0VjHA1-*3wa0)m1-DC^^oXB*6 zcMc$4h(@p+R+VrgF-XFSr3H|T1Q-khK^aaGJmqVG5z!q<>q&nRbO&)SkbB{)kHpAo z1eq88W)k$;6=L{^0e~qsM8N=XGo90gXe+{vmUIJpZ$KMpV;hdp3Y!M)_ZXCNyrKj& z0S4;`oiNA_(IJf}y-Idn{9nm!^>p9}5`n8g}>V zUrayz^{+gV{$l?8bb55puFaX}3@zx6u|0dn?kJrb+O=ZEu3wh*9|1d+{9F_%XFJ>6 zAZ!`*IyQe&kWexolH3mqGT90gLz3Vz%{5t^R3F>l)mM6}Dc=;rzVSX*dQr#$(5P?| z5hVt(sSYrJlWqR{?Xxg96*D6-wK{Y7L#b~VfIer zzOlAP7Mk|$iayeI{Y>M+!^!Xd6GQO!KQ+xrrT&F?_WiQxm?Z??tp^etdbtAaLlWc)xcYL#)OVvH1n*7eUFBOS(lA7c~Y z2IQT6?~!HXyAD|W6W!IHsK42@>i;O!z%+c8z28&0^cmqjR^UAl_=pNvLsh%<8D&)c z7}Zx><*HKN`22)XY&|}#it4`i7q*Ufty6iA@|D*VYWQAlm+O|(%KGK9_j;b{S3Xl& zm!5w=ZB#zQ&Z#x4Blyo$o9;7x(e%Ge z@0jD}A@g4Ilja{g{GwTJL#a3tQvK_O{*O0kr>aOb1>I2meR$p|~I<9pbbUfuaS7WJ}sJXx9$(nD~{GGGS zdDMBz`JD5I&XOzR+UnZp`k3n}*Ppp9?wotK`>6XQP) z-Rt!o^{eV9>OWfl#rhxAml{?z9BBAz!}lBBY`D7XE3jegVp>?=*qV+`US6knS)J0B4UWxp)&DplOZMN;nw(qoEY)`e{)Ba@p8&Okq zWAyRpUq(x@q1aUHSnS!@f9t60*w``K@k%EJ-V)#Zsd5032=w9NmwcF+>f1$LfnDs6 z7U}S?@}QAt@I3t&BTrEn|J%r`N*h~g=j5;%tTT#VU)}> zSRnqBk>{{x{8uBdDx=D;jJ!#yWj7mnv(m)wHS!iEz`m%A;1%36$|PR0O|RJ2lquyy z_}z|3p3V4bcq79>yq^0oUc;>^cZ-*CA3$!ScxCqyksijo!DdjFK>a?X9e~Xd{LLyW zVXIo9>@(_8D(m**rQiEd`yie>f_D}vBZp@ukId-W)Q7a~y_zD2wHmLmtW zjfV~%*?8#i{uwRN+oyFLIC5lm<%$*iP`Zywd+*%WdvN9m+NgNf_%+jq4q`=?y>I*$ zl-)9|yywVQV)R$ObX>zcG`v@-2X?m}%(4&p6dGDKu$9`bgGX*Ta{G+ludUSjd$K)= zzJAoYvN>h3qVnEvK;J!c_|97n9n|`J@uw+(-YnpC5Mx+2u|u;n2Ybr1lh~+SdI00R z+UKVz#3^9LnaWIfqmu>pDjVJySH-H8^~wf7XA>~z8s=a%piM63Mzm5b^D-avvjFTs zb*!E>uttV}2*j(kFb(lct$6=T8*67#7GoWF{c9KNhW)Gu@x&`wAKvbapb3^@X_kSM zpJM}TB~B-)0?GVe8ojwvlaOqwE^C880lpmR-lTvTbZT+rh@z^=v2G z#dfm~usj=QH?TeIMs^e1%Wh^9Y!dWyn(1tY?PL4d0d@=2t}A7qEw zo$Ls^iydWmvt#T->>l=EcAVYI?qeTe_p{$&A4R=}~ryJ;px8{wBWs(+ak*ctXb`wIIiJIh{RUt?cq-(WAYKW6jnKeCtD%j}!%PuMH$ zPuaKFx7l~tcUh7BC-!ITd+ht{RrVVDbM`v>3-E^j%+9g@!hXnp#Qu`~m2xFed4C_r zX@~v(8>f@ z^K^!%vpk*S=>eXemG|%WfGs83cc(#vc`*}9Ovq_#!@obuBGd!E+*&NRf@a!bd zPVwwC&+0ro!?XK%u8-&Xc`m_oNuEpbT$<-HJeTFU9M28#+$7IU@!T}e={z^XbNl!} zA0O!F0|`Emkm zHOZ%@_|!C?()rX3pW4T#`}lM}pHA@UB%e<4=`^3t@aZg{&hhC1K0V2&r}*?VpVs;G z44>Y|^**lmb3MWJB-c}1PjfxP^(@zOTp!>FWY?#-KFwiu)Mto(FudR2RY_h7N?a=_ zyYd^xHEqk+73YpE1TKJCP=e1W%5egj8?mFeloRAV??P{s?&NM!x< zXm4a005N+Y6@X4bOM5s*w%T8^-qJ!;x^~iM&?WzC9lcfYveKkp=s=Nir4{<3RTUKQmsl*>#sPK=L_ zHx^j;_;{qCY|qb(kM|VRxVAwnnA#^XAoIxfe8C(UE?6SN82)&HP4pB@@d(DH>1WJS z!y4U@ofoP`3d+QWg4z{E>4Y?vVhesuxa#NFn9G7tZ|J7SUocRb(1oMDj4G0iE*kj zv0e<&7JuGat&D6K?g}pg+8$pH_$t{7>&6g9Fxv@j!->cwErNiO(nydjXpIFdYa3NKRZDLrPK=)_eZU*Udc=*J`nOaMC z;c$0jE5PK#+`QdA1%Lbuqci|GQyPq)Q7Ns9pD|HdA3tNJv>|@RLTO|CjFr-+_!%3e zq4*g)rOk1rP}BV{7)T2S(u@W)4204!2102o2102B1EI7H1EI7X1EDmEflwO5Kq&3N zKq&2uYpVpFcf~P(_k=crMVO#Pn?zdZB&6z&7rMF&UDz&hVCp8I)K&LOWHJ{aI`y74 zfG<6Tp2am_fkM2i!2Epz%Dt6PS$=CpTuX~__Mr~jaOHLd6}alKs9XtrRnXe?Ly_E> z70i#B^kd!_=v5z?0M<_CdJ2hnZ*WylA^F>?0>h?JJ%y!E0_|F_wuyEoKzPlG6PqHN zKne1o*PwUUu1SVSN%Wrv2?+rE@h_?r>?7SXCwe2Aw(11h$}HX1dSx306WT;AtuR5G zdF_t;SGcBXjbFhF!5hYhiNM)FDA6B!jBLc#!YVG`C)m`iTT*d8GNDHb>d2%H8pB5> z8~6r`3`8wzXbaTZbVmBMRJYd ziuDeU8)Fc$e~xpta2BEhJE9 zQ@oHuGD=X}0Jv%!!L!P6x+YHOSQrIZH^-k>ly%5#L55N0+W7NKlw605DA`JNhH+~f z)uGIGszaF_REIKSRA&g8>!}W9c2XV6?4ml9*-drUBJ%;NLzz6)q0Bhdq09|bX9Sr& zREIJ*QXR_NM0F^$m+GuR=4PrxnF*>xnMtZcnW=aoy9nlKx+n~ySQoif$ju0RLh))` z?28w2i?#RDg{XZ%vdqYRqR@Tr+G9AMsVLf0GmB@H{k&9( z$MeMEdX%D4)$7*{jm=ME&&yC9P z5Iif6Z;~z1Ves>XqTo5s;51bGZ?#U*(Z8WluQScPTCKR04^gV`*3_0;xaw6`H2dQAVS%Dq4X|gY2a8zpT7?rYl=nrE^r*8M62n6<51-) zbynb5S0dELz_CRMSC3!?)zGWZ6^+q6Rmd)Y*8ZBUCJ<}6r;#h%J5x)=g(6r@tvg%QbyuGN*SfhP>NBf2*-2qU8YRMQ6|b} z;F$KM%Hy~<3adCsiN(GjYLsD{siZ5nVVe@DOMA2KAY~Rx2cd;R)a$P(!%7Qt%L)sk z@+zaU28|pPHEKq2X;IXiqOz$`nZ+~8GK)(eFN}&G6dToVYFXLL^xJNmg3>8eI%w9E zK{E==(8dTQUv@MLhxx@buqz6b&|WD*SrPXC?#a{f^yB2XXq?mKjKrag%Hx!QN(%nt zF~&G05e;>Du=J>LGs=p}rWY2(MWsi@4NMsr9~*~Smp7+esHiC8(M2gHqewnEbuuXM zABBsBrL&5PXGFyf!iMu=%xEE=ZeZ7e70)c3F)%nfq6_oCcYtzkr`1MTZzU9?0QF*CfW*)7K1+6`zJgVd<6P3we@&Yj6RAm~7d6y!czsZgF& zo>Jy1)yhJMn59aMvO;-UaVvGov&t%^L0PM;S2ie{lr73OrAgVTJg4k}8rZA6r0iE( zl>^Ev%3XlkfxQ4KXr?WRVk*Q!0#o@%6eoqB`XTXm>W>P>32 z+E?wT#;CWdgVb0xUQJY!)l@ZIyIlaY3g)!hB{L%Rm;@bYK8iw`jk3PtyUMRi`AuSjk-d8T6L>+>a*%9 zwLx90u2(mxo764pHnmCJslK58mwHYWaq$U>Ny#axX>qY}adGi+32}*WNpZ<>DRHTB zX>qx6d2#u11#yLOQ{rReWO4N=iyn=sX$fhGX-R3xX(?%`X=!P> zX?bb+X$5J8X;X4zbK`R3a}#nCbCYtDb5n9tbJKEjbMtcZa|?2(lt(<>luU@)VRFGVdQjl7ZR*+keSCC&&P*5m^=>NN#xgfg(Dn?P4flQWzP#8$% z84yb?u*F@_s&^~*fCcYWSAuxzK|ZTNKx;rk>p(<}Aft^Sq|G3utstiDAg3K5sAly! z^?7v{2y3^xN8PKwsJ^7`Q}?SaYODIPdO$s>zM>vd538@Luc>Y7Z`9XSkNSpsL_Mm$ zsUB0`Qr}kJQQuYHQ{PuVP>-u8)DP8@>TlKGsi)MB)ZeQgtA9}csD7e;s{Tp+O#NIv zt$v}NQU9#|Mg3C!O8r{>M*XY$t@@q%H}&soJ4pKxB9cDXsV`ZAzG-WYZlE4Bz2V*riE+Ww5zoU?HcV`t-IDkvuQmwyB4YS z(yr64*KW{m)Ou^b(j1yoi_-dNH)%I((b_FqU(KcU)B0;M+5qiVZJ;(tsnc%LVzoFe zUQ5stwInTBOVLubG%Z~ltlh3dEbSp}v^GW?tBupfYY%IWXxZAM+GARdHbI-HoFTb;Go)k{B$pqOQiQUI{pWUN>k4Jhe?yuQ9y1MILy6)TSM_%7{{hw|abi?Qy z=H2k}jrZO-{>I09NA}L>eYm&(S2zD^!LR_Y|9CP@b8P0uCiBZ3fs*P%i`a_?% zK1=)TxoO?a%cJK;ABz6*maA^L_m+jXeAxH;zLWcY?YhzRtZS#M#r37@d_Q}?n11*4 z%kHlsJ}nvp_nZLZXJ*{fZuxmt!r=nao__3rwyzhCR}d2C)`j zc8l85!WXxMv_$fce9w!IEG_;8c3(DM?9aAFFfY%cKeZ#v8`AR(_jF|0qr&{rBFFCX zN4tE{E-TOBG5Rl6Y)3_rBVsuInb#N1nAac8^ax+OSM}BKoDhB%EsAj>4%;~H;Gx(Y zv=^bm;moGyMGm^iaWU4Wb5!K0=#UNI!9slFJKcYI{Yx6Wct7)+9}FzCPuTe^Jm*d3 z?!p|ryKlZG4Equu8(^0 z?rlSuA(};~{m#1{?aPFPl|EBeJImnj@lxGq@a}dI;Sc9Cm|p)v{cg6Gotymk%u|Mc zy7<^GhKcU_5uyJpiT5ls4)XE#cSW|&uV2IUKfKRXBjVha*(#PUgy(d$+Wj>m$I4d< z4`Z7;5EM zsp7?2%zL4^P*jl{qh=Ytxrf@jykoN_o{btrMf%nwxW}tKq7JM~CNHu}0 zz8bok{tiZ;8fKh2rH^}~=nw2PJH6-B8*doC z#ivk3e`DO9VJwxU7Tq~+oN;QHe(Kc0vy5x_oAi%iprZ^CWq#m9}4 zr}WB=3wE$(*1US##*GFq`kg)VZhd3r>M~Z$iWihrRvIUV=`X&x&BKncBW15W{-O~v zXv=J0v@cp^zG!o{`-Zvv<#r}c;c;DzpVEI_J#EocHkB3CPj4_V6k>n*Z4TTO<_bN| z-k$y1RKuU*Ptm8oHv4UMobhyi1GaQ#@EXzGzW32Bqu2;0(!~wf(s4Ly%cFa#Ihsc) zr$WHZ=d(Imz2~zqhrZ}YS`lB3l~xanOr$4e8b~TIogqC_eSNS%^H$7Tys+93^TZy} zlQ9>T$*<{^ja3^RzUM3(8yhz|eVW%RdRk}h7E^iM@@J}7EvTEf!f=b8b{;K;h*qXA zK`;HnxF@n-ScDhS&f5cn#1mi%ZQrf}9WAM;S>p76YF*;4S?TDw!?M!tUg_jxthVp* z{1)4{EASMn^oQx;R2^bgI}c34*6?`!(P0# ztl9Alt9|+zX0(YumW5A>5HW2+Mpa2=5u3mY))($5*-^6Zsr}6Gt+MQ6FE;LIGTfFO zJJ#=G``Ig%d#iR#_(X*8X$vunL@#K{Y zbjIEj*Brgc@Q=3~{oy@+4P(a2)r=<-&(m0>^blHHoY0)?=7$HS-J4fb`WSoI=xDXD z*Gpf`+mrU;!{4!g8C;9|T4)Z}`7Ha`S0)}g^2#em9424KfD2-{cH+db4wvt+HK>`K%$s#4xy7*gcJA45kR1*_qsVdDy%xHSZgILS)QiRT z!|4;lQ&WczPj!kIi}~mtk_H}AQh*{oBvb<85VYbA@#1<#jb5;5`t(HwMok6tAJ$V( z3_tDg9rpSUTZ+pu{a6C0@38N%g%-k*Ej$*N*9As{00u8gKEyEC`BrmW=%Axjk04o( z;(+e*e;J^{Z6+1^z7%cIV$xag2T_m5dx44|AzSU{u*4XvBw?|{TD-Nq+0l_@kq^U{ zfd1S|9AXS6Vd5)e9W)=9P(ez>e z|D(Mp*1c_@1u+C`u;{}%N7--K{)Rmpwrtq4dG%h<_15ZjbJxvnC}#zR*TRlfy*}k7 zW6DbpH$KFS2p4fKhEEa~M=7nV-AAt!w8;O=${bg&8;w<)CKsg8Y+5B_kmY2H)wOZ8J_ zN5*a&W;Cr?zm{+Eh3oFxr)!th8j}v{{tCatKJ=kcL!GSOxWvH|_Lm=?|0-mpi-%)# z{eINjL!A*z|M4Rb)ECV#^?*H7CgD+Nh1?as~4BgDxtwR>sTAp zS=lq?wX=vkQC8CR^Y>Au}aih*=HkItHXx+ZAW&0uHgQ+9ESW*Zn?U<=ujnkCB& z(Q8EUR{fLH8GNt^XZXty8K0&bGs;D;hSJ^DO$|*A4cHk&c&6@Nx4M2kGngA=*XH0v3OCrvg+U32OFpu^X_o z$mz%eO991t?Ed*(JM+!A`r9F#E^Qv?0PtPPsddTw0z4>t!kO3R^$nzvuw~1ZFEs{= zk-F`RTLR?T$0CKB|ADUT9h}uP3+}32US|yCxXZh|ZdonvvVGxy01p~u4Ppx? zNfC$5%g;t~?Q19oQ$67OYpyv_gq_0`8WV;k4E06(fi`^6rm&OR1gwMtf1t>eeP$JW zx7+D*2lTTXpoe*T@ONmSwpV*QhjIY&Xk?0hV75F^BU)`L+M$| zI<{d=?ONkAXcF5iwQHBInTuik(VxW%PoZG(`Z;T##BAh%|4oHB2MUq@e$JmDOA*W7xUFP+GDlEWOyOfdHL#%VFtLHk0aL>oqb=3`X9YY`oNX3ayTy}Zsyu&)T zp?aO8!(mz1(6G+g;RsYDE&_zY3Y*xHyS?}$bVpVV0nCA6*)9Nv(#HAvb2FM}?0kYi zbLrMu+sd{Ze1sKC1gPdAYY6LNT9%lVt686%g%6+rwJYzzsyFxXZMQJg`i zjEA>1&&LJb%i4H&^BP<^bt;>OuW7~==EZ&Un{i>-Dco1QM#mLBTe$5(CenhV#3OHp=L5aC?6+aMr34S)3pyq!n`I|KN;uEi=E{~*l}_Y? zw|TRz!IRU&Pk`XO0qVnvl)u@oHmkhi3YDriJKK5zY+wQ+@I4jPA1vm%*N78@?CxR8cq+BKU#(3LsX4^f) zG>K-4;n-%1nH+mQ6WefXGo2h4P&5-7aA25i;}BP9To@>_pPkKrwrbTP!0L9vNd-&N`?Qt~w@PCkx#I#DJdxMt8^pU`x z@YlfjlAJ--gRCp(UU~q*8q%p@e$z#AngELs$>U5wF2LIX*)TqXM87GSr6LUJITK?> z#lV=IUQ5v053aofMZtk*i9&mN>8LwdoFRY@xE6o}?CVi~NN+N-62Nvu9}qQib}^|N z@SNvcJF=iqZ6ALbVPt^NDw_;Snu&(u8e+Y7 z^yqt?*;aP%fzijS48D4#zHZs(QudUQE%g=H$ugfUbT4xo-=Q&9w551k)wZhUCC@YC zV-U#4mJi>2^FwEwm3=t*%@K`;Sp9)Mw{}hwTMtb^TFk-SmNjfuO>K=a(Cf9bJ+qt3 z8p|4sS3bdvAztV-npz-vpoRppD-y79fgN`x4K{!awaQ!&U3>*v8(r$ziCR6G;Vc zQo%dPn7DG9HG&5wB^4Fv)zzY2tYKn?A=3Db;zpi^?M7^A4#sDQdcLN*!4UWRM@k$> zgc}q&Cg_u9CCO3~V~{6=5Zw7zDMO`iEkLtGWRR`kSsE@T09G(fgTz`=5fQP~gr@sDLbk-_3w#{RMI7`&7 zBvd7|MP|ZB-I-|OTbZxBulu_r z_4?{f3)cos-nEN1ET}gIefPm}{n#<~_lJ&+ezQLtJ=z#Ca^Sa++fUZdhscIQVTDm+ z;kqcc^IoEtIEk$%zYg+_9Ihl3f@03J9l)66a42P%NZZQumxE8sAwUIsEIAcI&+ zfBq={%|F3k63}^>gP6x|+j60z0q;f2+ijQ{lB&#UF0l!WypaTU(7F|^WkX<0qS*w| z55g)-$DCw~95w>o-T;gy*^;m?O))r5;v~o)*>(>bI5`x$$F>EYTNuMOj~C$tJdS^S zS2q*%EFJ?$K}tBnnA993lR)4~whvZqT{AcT+}2I_L#(=L*&DN7Jw3Ejhh%9)?)jhj!j`R za~D4U#NMg>9#}r1Cgm^lPBP&3-OU#ng{Z_R|cOV%&mcy#+d>77?Q#$W&f(GnMyP8Tf4RaEVX>j3uFRiR3V)hy+ysmzPK&k!bBIG|ja0!VOiJ~lMb%F6g-Mpa_JH^E3v0uo`fA7d4F7z) zIAE==U)12}h_N)(*Ecx%fuO4s-oAjV({~u_Ai=LW4ggDnzdcFQ0?JDa5AU<2yllAi zy#&$WC6VkCb9p%!(KPL_TrLy5!{JPdDOgTsCB^{0$szZqG*{H)ak2>6Z{1Rj8BJ6C~CDa}~hN7;aFXc0O;4N=;fPz08;5m@5i ziEsIL{96hgwXq}6Rk7a)q(j8U3M5BdJeKT4jE#*L2EIDjP!x?JRgK4|Z<1k9#V#-0 zBv()h9j#Doh@Zg5la6s3ErWlYB&3Tx6R>8`8rgcCm-W0muySs5YU6b z9-iPi{v*!@f*}Yi(U7#>f|gsrfWyuV zzW@6=R}8lY;_R1%+et$ZotX9t_94E*B+o8*H>wbDc*=l$J4%#9I6%^q*X`EV*EF(5 zEZK#;0n?8IquhQwp>9+Unt}WVtog;bfH(`SDq^|@2M}oj>qyR!;j(2===ysgP0%#a zk~iqmHKV6ANhFDgP{GsC#rBLa^E=|43vSC0{yD8WwT`)xuO7pX>EbCj z0bpnE+B;2-_iJaZQT{Zz4%tz|n_7`81?p9m|ifZNpOY2LQ2 z*~zw7Y@JnW{CGt#y={xwkFZ7OXrxJwG&xR}3=&W%kvyl6Ri?eoA0r+M;g4bYU~$tj zS$Rv1eN0XMoL^5fCQs7mEvlZwo-!j9>)ED;`nATvgZiF5C!cN2+h6eX$ozZ*f-vTi zdYh>pglUZa$tR3=&-kRcdD_Ou>nm&Lu*wyN{~GbObcgC08BBElB;)9q&#Hdgv~%^2 z^;@?Z2M+3M>l-$+^=1&_DOORvXr3`?l3rAlxj3)2VE>8_T3XD;>+4rGvIeu>a<**6 zat0{3h%KmI1{iTr900zh6}Lw4Re$^L9~s^rwrbyLM1joVbsZW#^5w&tH0klBCC`*R z^Hc+4W~c+`lp^&{HdL%%w0_a1xotH@Tg`7bz5DJJ#%om8&ZYrlZE{4FJ^Pt^D@Tno z=j#e1Ut7QW(otVNvdKM9EDi#{r%E;4da z3rYY@xgnv*r*jx80S&pKRZSO-vdI!|FO{y|V5S#xy^!(6$2s3($JW2L!@aC-3A`T&8#Gq! zp1X}5Wrq&oYunu2RgH$rt1qivT({J{^R*3cGQ@R*Nnrl=P~k*sLI`(ayRb)ogHzlj z6l^y+DZoLlD+~p$JE<&#PDPUa(h4N&B!?rd1Ww0vrzXydpIEiL>fqi5z<`>#~JpNFmqun z5f=~?X&jw3Bp+;5TpT$&nBm?2@BdxH!gW|N#p(ao!8fo zLXo&N#*3-4{ls^HJ0~xgI*Co9a6FtfK`R}Or5skPOV|VDwS4h%Lr~t&MID{3+s-l3 zkE_Q|yDvF7_&PAPz;&-ug=a3-DyJwz6a8zG7U(d`Gp)B*{y&pcqwc{rZ zzKb{OEiE6c*k7=}VEF@6fCSuv=?fNAvIVObtY#ZmuQr}_fBjwN$pJC?V~?@hUw!P= z$3A7RzG}dER1-u71^XY_{0N{ojC{yJf*}%jdv!mO%iyCjZ4onAO45_~%NLD|BFZd6 zU5YW|wnx~c$7eqL%DA0FSqhs`Q?jIFQ}xD0TbXhCgc;!;{xzHqCxHqf9c29bL>!_& z7q9t>#Yy|*M@CH_vD~nIw6k!-1eR@#AhBg-uTMWXX{&MG;j&LEpFRnRR3hDKTMI@_ zM?Mu@n>hZ#>6t8(J-BP42bz~2v&Q63$Oj-}Esnx|!tpiGF1gmt9NaiWFg2$rggM-2 zX>uYHis6ET#>%*o{Fgp;;~pGZkj~QC(Ea1yq2!%5ZySU?S(s2f#N==t|Lua!95k+c zd0mYwe|IDbAsq^)8js1g+kSu)BqtKZ1!GuZ!Tt9cybbUN6x*b1RVf>=nr8e=LRKt&Am7KttP~DM?F&vG2p-}FU}x!0mZE{a z0y+pCnED4ZCH0T#x0AVyBoiq#K2xfzTf#(zh_)9_*VFGC4;NmD5mcTWN)+2T2)>Yq zy=m_og}WZecxk$RY{LG#*D;U19%UCIrnHz#6Cc$r_{%5T7Ti|E-ZdhQeU zec!zF*O&fktS#nM@IZ2G~apy$t%;kLyig^3mVL6kMkbky1 z8j_tAZ=ADwmU{_Xz~&pa=R_51Raw{?xO`VG*j~9AxlV5$IPm712PThpu;R)&3ue`r zb$J!)p&DCRW7vjoU$D8dnVD559~kW{W^*cMEm%^6Rzb2=qRL85x>p*uy4Bk^%2rX$ zF?#ak(awlx;gf-98;X#k!3?vI%pA&zvzHbc-uZg%j{5DJ@Y%KTI2`;hR&B1_ zTv=bnN?GdEvg}FOlSbah#8pPAx5>&*@7mUOu+!_^JXZmQeN-eaDEtz+Nc@ai#Kxhxw(7?33w)iF4OAd_@m(VASU zPsLh+d7rat}dTRi8YyGAhNs4ca*Owf`7*4 zwYY0|iWmdLm

=q+oq7+tRRgr-9Vc(Lh=j6D4m!A>yC8%GnaP7{>EZ zX-pf@FJa{XJP#(u2LqqMU@wxK*gp@RI%Nz)Cil1@MXAUql8E#os&k%ZryhS}tU+!w z>9z16Hz-^mcBo!f4A~8e2ds3 z&cO2VMT!&rgg+8S7IJraDbK`0mQqOhIZ?*T#B+fQ(sxP4LH{J`Bc%*8f;>BtVQ{e! z?6*NAV;&_i^dFY)R`P{8C~r8&YP#5-_90GjzqEF28zgpiOJ6Iw)*QB5DSygpgG{yB zZk5V|mftjmV1|4Q4$mtp%5$Riygfy&4&Qi7>z+NWPTpM_oIu;KH$9OqtH`B%_d#Xi zu`OSI`oVV)B~VecE;QLvrv%j>=h`zIF8faA!5Dkq8bRA2Xw7wp0| zUi26%dOmDSx1!w>qVJ!gTE-uk^z!tVr?-?JVux7E)|Yp^yz9Wh7SEr4Jb@@APd9d1 zMbFnok0Zk7F)CK+=d(hWu^G=!+dgf3VawD*_npb+S1sZ_41SnL1mdRViczLztKEF3 z!Ib}`@_+&{5ft7b#Q~Tk6R%(tfJ=IS(rhouxu=P?orJU2_7X)O=+z1^A9<{4N?-DN zaSYpC5~(>AvQrsrm5OW#xf5s_i8M`jg6vbe806et>4vWU2lEDM1T$!UNMA}z^0FmF zMw(ngB#XBe?a6bT*Doel#v@(hm(K|ANF0XD7}#52DdbEM6XwW6EFlhYf!2`_IsGAr zvGa+ozam?R3$rCC!tFwC2Qrgvan%FD=*%{&x^Eb=P-5)1Ta*D|9a)jKK0^kC+42=> z!JCzHQQ5XNa5v3R4B*o!1RQRh)*&ul)~p~hEY13>QZ8uFw9K*bA{r46zR1YGilP8F_Xw6bMUB{ z4;CDs1S?3Q6;{|NA_2}?dW}b5wRPSHF;xI_I5h~`2B1DD1<8UKP{`$JzJZMTV4ClF zdxo74!5bpjhT)YM_%rYZ7~V(lV3~t%8|1dh1#d&%i4>h}cnJaTJMb8p^betuO{5zL z1o;jlv?E_qKrldh*U40Gw^d^tw}c^n3fsim%$gQ%s(^QIQ^nuJxOFA#N_NcKQNN>p z?Q@HEEZR}PuV+n0)7B=EYY4fL7H*E_2bpux#>%y`<$94cG#jQ+(IETWl3T^N3N(49 zqM~$RF*9J(pS5mb8`suvG}u{wuvtQ5yz5Y0-qhqoEVgMszaCxgnD<;sy;0%TE0$Nz zTTp@f#3sDn1S{EB)9wx~0vMMN3Z%mwvqYr8Lfm}?tb4Hfz}$UC>=eDBxNZiUei_US zx`G_fv*(vKR~vi2)645iYfEd5l`=~}7kXD>N5rI9LaEHfJoi!C%B8pj=uHj9}Wg(wmndeUV#b|UDAV)Y&Z zfRy$@;tUobDOdRinxhwthKBi)BZr3hXG3D%73QCBCPktaP@{Cg$kd|1Jw2_ql-0Ot z$udfp9|N957A(C3;!BBKy7ZDV+im`GmsvHI=OFiW*NVsS4-%vC_eJy zTTzdDBV(;_45D;|S^ACD*6fX>x}8hWbuh2E(~wM`(hKNhXc!NRyo zCB2kHNuPxO&1q73Gmx4u91RKw6Fm!rdXM2r)4zR-YcKF{#=9{dI{n*GhUar#sJ|7x z_M@5s_;x!RR{lV~@kX+K`1#j2yv^Xnee%!~hUbj_!2Ub8Wym^|tUtgMYbt+(`gv9M z6U;IGHQog*HpD^Eq8Ajf5&H`^&w*HC*y=ZLHh3#Ps5e(Xk0d7!`xe>Mv`28RX1x&u zoK5JoyBiRUV%38yvizpm2 z(`yYEB?A6Pd)Dw<1@@8ZPlS>dUZ6=L}CXP~r@~)LaVY#s)J) zo#8U3?Yby7y=LlzEGJec1TR@UoFsD4XG~Jq87{8}EK#Y!!h`-!ywnizg$~0Jm5P{Q zr-HsuJ)Au5ofDNWv)RHg7}T8y=LF!F;r7dI=pdSgO2fvhukr{I zF&schP6Qb_z)6U2Ai|0#Fgpvr1W9T~+DG!)KqOE>;pBorgdm(U5`tM-PLz^82;3`? zE_fROig4+E^3U$76@0Tz-CYxG})-B(dRFjKX-BUq$#7z9)MuHBw*zX$1g|K;fJT9{{6r9$S+^-e2tDf zpZ{-d2kQp+o$Ck7{@t@t{m%Dvu1oj-Cv9}T=l|mPN__^)g8TotAN*om=eoZ%*3NbQ zljHxbonLxRD!=R+o>7(s_E)R}`s#dN=i|=LtG(8ByuVbh^F4H|{?PS4D*I3Gy|k_W f%X4~$E_2;^J#ifP;CI~=<%5iE_!YyhznS + + + + +Created by FontForge 20120731 at Tue Jul 1 20:39:22 2014 + By P.J. Onori +Created by P.J. Onori with FontForge 2.0 (http://fontforge.sf.net) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.ttf b/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.ttf new file mode 100755 index 0000000000000000000000000000000000000000..fab604866cd5e55ef4525ea22e420c411f510b01 GIT binary patch literal 28028 zcmdtKd3;;feJ6U)xmZaM3$bwn2@oW}1>CTclqiYRLQA$5Y6)oBGM7s&UL;16CB=~) zH%=W_6UVZgVeQ0|xQgR(6Hfyvk(0O_J9V>Qn%H&oG)e0>@i>{hWS-onNvmm7eN1S+ zzjH1~P?8h3Z~l59fphM;=bq(ve&@HJt1v}T9Lj@=s?4rmzvGs>e#M?e$-DSAY}wuu zPnxz(DGIB>^~Cf&le3EBXMcd}6Zj5KA3GXEIX;t*;HP5m?7n+mKJ@c`Tz^VYD(~Jm zd1MylPFz2T)UxmH5A7ZOe}4HVio)j=WvpiZ%%sNt@lV$&%8rY;pWcrG(tJLATS5ef5?>;m=`T3U~TdOF!ucQC(+%tJ%mOWkhLq)lj+7BL_yl3W< z|K$8OuAf04Cua{GIr?|bL{U+0Z%`D&^z7l8*&pAf{=TBzgX+qM@uk@--(Pw5FDd=Y zzv;PiF*WcaJFOVej)kLlWmcx_K_#l7Hdl-))s-Jiaq+Wt?>bHS=G)5KZ>d2Pj^cL) zspv_s6cktVJbfGVdn<57wHg$I5=3giAFkhi>*`hfDp#)t<$c^@rlkfMM*)4yKjpoZ zm;e7O&j~k_zvW&)&a7B2n1DOHt25zBxS|PHxb6pE|LkYEcj28n_7e#qH3-ZzD|Xba zuyCr&LatB>-zH{GA;V(qa?!?47iYCXp*YJ<^ZA9f8oR8`&1u?oZB#99!|V;=FIv_H zHB=}yp=sKjTsBRN!=aeIVp3RFXLZmQUKG&EInIE&niKmm!2v$!20ko9;D~#VS11nc$`+=KtG~yf>$N>ebwp;yRE`v zGH}Jv)#<|c{rH;oR1LoSw#IV{&!ba4$LBE(`n=!v1WX7n_@h>+xl&r**uQ0L1!}B7 zt%+QDbF_1>eooBQh?%++pHi_R?rNvaVp0_&7C-Jcx2Da0VHnH(`yji@Q4AK*~y%C}@R$UciWpw&Fz=BN&REs|Hb5 z;$@}9KzIq9aGHV#O5h8E}wr4JV`QcE{(tKyortc-Ac zv8~hc$>PQ3trZG48duddZHX0S*S59PQlWs6zK{7a+O3K5cJSm-tA>$kafivtXzwF&by768I+`}rql(K|3%uZ`sLDML~eis`agzI^b!&%^)q#exy z{uPQ>X;RvWcC-W=e9lS}(GIuYlzx?4YHksgUImQXzoMzdf+Q*$Kg_9fyOSJZs$*<<+E(%oGdnwYpO{(HB(_-7zv zf{W|>&!PC0imz2WsU5X!4}vIr{4C;UXb`h{hi!c4o#Kn{u+t~=S@!wOPZV$8Jb5y& z2B{D?Kb}81xtV=Fdw=ovEV7czOS)@RtV$L75Hy$i0P=${%0+O6L9*X{n_ULtT`Uma zcpe2nR-kN&c4Mx7aJ`5UC-`?oL-n;aHU{{!w7-%2v5+p0DI98!q+H=t!kzY;Lk8jw z9$!4Yk|kTp^6XKUi`{*~_MqmmFZ`|Dqdj=ZUUQlSi+|q{2y_IPLnLaD+1c-X(xDa4 z*gYOQJE*Z**8?vU0$$A%qWMuB6`;a#{Ho zt(sfqBHoMjtCFy>n+Y~b9K*m+LKs3S=}r*hvY}^>Jv{vG+rtlQg~72wVC>ju4rR7% z$sGF3*uqQggM&0jfww#&+H;~s;H}GHHxf>{6Grf~aLOFbL^J-3H)Hl@=HhJ6PkvH7 z8{f2PZf?^i$TM?l@X8ZUUAdwcfOZf$EZYxWC7`sT-KIvruTtPDUw=L zK&%PU2IwJhOkYnG7;3ptY2dV;w43plfJ`Z{ovO3g_gK62-G8vEK~3AYZ{eI3GQtww z@naTIz&YGdTO;7iFb!-NY#O#Y?0Lu^g&BK5+2eYB9kt&Chy zfn`Q4M6*FP82LQSjArinLqVwK=$geu>6<*q=jB~2_&j$6Ca}PZ|3b3InB*GPsR8WC zdaR*a?n&0fd}iig5CvB;D?tY9&>S72HQ@i#6f+u&|KzB3ZAsgz*zsapcJtE*H?CND z(=BR1jTz0wKd7>$x43E@tfF{qbN1lV&EbE1ts7D9GGDu?OG5h7FYwkgf$VxLUl*#P#m;wC zHy9Wj9BCPLIK2U%W3wr4q*}&xM$b{3ll^&h&^+u5hcn=JN7hh-m1 zUgY!Eg_o@Ci6@G-`&Hk0cZbvNW=`vi*luVYA0ZEs-s1)rt%np7R@|$dpbgX{mqGDrvr8pyH$VUJ#p{eOwmGZp&nc8YPIm z*Gqe^tGyMQPwYJa8z?`>2;_3sX zzCdyw-DiScxfm(eg1j!u3zB9pwPDrk6lbXw+0Ifwq8%#>vD54{>7}xcq{~ehO9(P< zALw#-N2Ix$ldJ~$!4UT~G4MeLq#}SSf<4y5q~rirF2v3jJ*|iQU?^1886#}I!lG_d zy_LnY6<*bzuBw=0M&@l~+a$}X0^=JH6Hh1O9908c; zM24g{$zMn|S**+aX1^KBA#1BaN`;`eysqH2ZYzW2g4@MeR3kJH8QJdA7^F_c%u#cc zmXKPcMWmFrIxV;^*H-~nwrliPJmz0iUom!V^aVD&sCQ=N^)>B~OnXf`8B7acfS?sM zmz3BmqjPhm|D_g7CAdXH6XO%~$OS3Oav@MHWMv=`v3~r7K+uWp8xx>F#1a-+V=~Qv zF`Fvw#f$dJO~t?4#4h8)Ub%1#ziJRv9mOb#dp8scdT}K`RcWVwm*fsJ=wJ=-+Y5Wh zGJU7C+glS}pWhtmVI_r!+kTVJ|0Z8Nt2IYPTY8;k8V}vL`9e!*w5``x2K!p@dCP@J zqnH~wX@C(UGlzwx3v(o{l^9}fkQ-uq0ZwKx(D*cab^n>pe(Nic3yZ&MI5y^bY@=#m zChiT)6$*16H3+kob7x;&O`PP)cwb`d*sjCS9UuZw1#tWlj0FyOKb%#EBWezp zhTw;O0^xfl3+sJ9S}43FdcO5a0lN@{qts`ip!YX)1!5)OjlKwvrS4OW{UP*~#rX;) zLrhdQof|3+jUA&&@p;+iP!1Gv*WqPju2dQ^X0J`?3GTQb93RXd05g{0xYX{I58ra< zxsHL3+B2+|0JqcwWX>adoK4B}{xgMZ`yyPBV^*P;I)DpR6~ul(>sW%pJYe>Rqpbslp0X^vu63MFpo-IU6@N$SCoJNeMx8o)D97z!m@tlv(mI$ z_AG!vnmwd~S*c6Nr=`uUyzkPujZ5P;`h{gy@;nS%@0}F40_I7`LvmCU{JmdUsjOGF zD6ZA^jT?rC1_x4ou{Mulf>DEz2bSiv6fL2=39bdS7w9i&4y4JXSQw%|!el_I9Z4Q$ zDG01&A!rFgAP3Afg8NXMc4GO(m%!D$adxC5fK3AAxq__%vqFqG8iev2JRu*qp@Q62 zfsQZ1C?)F0siXs&TJQ_8rz^0}Objx#D+!&*3+C6HBEhQw1xxi?E8e|SfZ(UwmBEXM z-nk+5LH4QfkP#RTmL(%kiReXDqq~HZ*U&u@<+Kk8UVSa)6Kpn4BkiDNptUIDJ=SY@ zkBcBzYMiV{WwxV*=RsldIPBMY8zuXlUxEGF<1E?hVZYXuO{sF?wJ0zat_j%kx*L8!tfj+p%JQRk~3}w^rf?yJY zV*aWYrv`*%%l5>JXW1UopyOI`2*sdC8Wo|OnqPt!t+O9|CrR+?>x$HS#99MhC8K(2 ztxNDSC)1fhPHLFk45>^sQo2`KrV{UaMSyb7V^>v+&%V1B#*MK-)2&Wo$pGuMh#??- z+z~K1Z#9v)+g`idzW#bVq1{gMoUr|qNgVcP>@oPGNQ;2&gN*d=zAY>uP$%G?qB$?& znJS(q+O69ljM647X$7?cVnO&T+z#}dTz3P!v*_0-o^!(wrnZ&|G}6Dq_LPY(g6PNI zDl5^)A=|6O>OzmUsWc9Nn`{cOo`#dH{)|vzg>p(T)qv(28GVPgfc0(R^Y45C`{3jk z>T)^vff3@4BL`@XVqJxtWK=AQ4deCDx>mdFRTV_l$&Uk@0RAA#w-SjGUnp%cc6wng zBttUz3)V#z9g-ypia;Rj1pHGUpea|MCNrcm2%6F;>`Bn~;(lO%I2D0PEi9;hV_O|{aD zG1j=HZ0Bz@2u7Al4yhUFui#VCE=icjV$D@;{Qkf@_DBwYjSE z@S!s+2@6-AIdr(Qs<<)W9Xp22I@sW81Nda{lRBinMQvcmvc4D} zLItj=PwpZ>n%0P559kRR$zm|JUk0@#-)zO#%47#`7_zwdl2=Xt!c9Pe*D}}|AjerQ zSP+{a>434-Yiz}?7I-fQ38W)|0rEo`T{eJzko;$_w15_n{Aa|Ner3bK;auwcn7 zxeVbVCyG*_N#y3{=jP@k*ikeVv6rAH&cn8{Xj_C90qGUeiw7c17z>i|lF2F>$|NGG zFl^?G=caFSZhrNtCbr30Jnv@h&bMy;*x_A!?!5cO^i{?EZD*nOm1baR{Lbv5ag7`~ zoA1lsvs+u;qCND-)US|#M873|N!As}KR)pK63>MEvy5i~s2TlB_7w8{(;Aj&1IcNN zAM~-r$Nn{PC0fHWl|TF5vZ0hKf0u0d-g2pwEq|L_`u^ogj2cV2#AB?2SJ*2o0=ED* zL{5Nvli2|hJ;Dug8es@&;u^Geaw7soNFmp*NZ3jGRS(Qa0oVHAJ**PA7H>2(F}oq$ zOy-CoQ%U@a#>sm~*h2PD$fRlZM11<@b$u;XtI5A**Td^JeEhZzE|+R+?;gEHdq^0b z3Ki820dJ#Sa9chfO08aR_L^Y{2RpcEEkB)iT#W{No=m1waKkbWTZrM=(#$fcZch%=s7o$M7zP?Z2(a; zB$=R);Sl8umil$6&d!xy{U7 zTUQUS8Qxr6ke7R>^aAXYC7e;gu_0d=q+9}5vm3<^{F*cC(ti4K+YnD2cX6hz4P z!uKNNd&!H<2{pmgL?(!72E_9eo zSG~XB4RmEhJ~vdTc1F5Iz6)NG+)&>wj$`oJ3_5Pd}~f^(Nh*@hrj7 z1gjn9B;`XFAPDnS$e(eAGO&FCD06e{GT<^xUOjOsFK*CArCIO>xBjqf3eVHCV)IgC z)Cd(6FN(%!EKBsu49#*U_V2b0(dBldRNYQLU(#_1KMyUGDW*?jv_%{gXX~s6RWmv zu4+v?2YNR>)Xx2Z#@@bq#+n*kRaHjMTE^5$lUwb7HQaAh(-zfgc3OR~RF&doVs1y+ zYOwn~7HDPFBkNgnMPpjER{0JDeIo;&8ne5-(Gd%^RaRHkR(Sm;V`Y`On!E3*XtG(D zN%d5jDt&6Cd~JwZQ#_fJ-TjR0kx*c~A^yrF#gUQwv1DUFM*E(|dMFi}xyUNZGLT0Id4ixx*U!xSYmhON8Q9@Isb_MOI zQfk3JD!$fO=e3)Nzajpi%y{b(9$e{YDJi0EKIaBSdfpp=|29`w<6gMa%?EXb(p|hj z1d45PlmE8(mfL+nS0HtI1^h{XUeyu3f_MXOgizX{x1_`sI)|1btjHi?WVtC_kpmw- zwit{nag?!sX^y-0lUF8{0{=MR_U%(oxug#5u4*_^P~05cHzr zYmrc$uR`El99|uAB#`Sm5{0vh#o}=cSo9X ziN3x>U{y!QDt1I90Tl4u>VbjPC!RT>C)$dwE0VpvN%|ry;iJc6k^JP7G_m9uGYQ5i z42LNMx?n_*M~Dds3jtGw%WxJZM4&fb^Xc-Z&@90ZE#n}xH|H^K?F2PgiU8cPzG*X;t<{~s@Ewc#f%^JAcM5Di|8`8 zt)i0RFNzmsgatb-<1vb}%dhXOu5I)p%B$7pyVM&>MF{e|PB~fa2F@KDSj3l;*s{#GqTM7HF%D=1OirTVkeS`pN&nEGQGf zH<%OJD%}g%OE8$*N;K~M+ek?Ek@QZ=K{797A#g_8M^L@QFL6qlBUVX~c4TH2DRftS z1b-$Ond~tXaYJ&gcXf4ltPN6Z17uhyqG1h+MJQWB&(EN5FpJ-r7h+IAP&slo!ADEf z^Tt`kgNZ7TUv8XYs6w97>53j_Vr6P8kqpd!*b?5bt9S~%0;F7}5P?W(7@-wX9l%d=znfr%CJ4UDvf z0&J@Ey?1+whJ!}P_Nt|w7QO*-LIrHK39dq6`Js5_95n~<#OEk<95W@!_{x=n7RMK2 zd8s`CD?jlZ8z-IvKWGYV0Z@q$6U`BC@J7k43WpDZLn-k5GBQOQAcsyg#4r*Ipio9c zP+$$N7F9%~gOi2PZd0A$HRN;fm=U9+Z&pMvM508voY3C|NIgC}UlXe^X}0PW9j;EB zW;EY2{`hNb&z+~i*UqTH*B;-s)r8xfu8tMeHqBsd#}mbSPv42dG;f?)T7UHI6#fpc zOW2-;t-#I^I0!>aiG{+{EbLCg0>xx-lp4&R%$|PWU@&Owy#L-OvL|mAf~roRAr4^Y z_z~mXO}wZx+En9mn8_apw4m8}L#<#dTp$Ta(Oj@2*=@;o21_yny8b=XdlV?<*`^&veDfVWp&KJeGyLt_=znKkl`P~Kc#4@ z499g_ddY_YQ55{%%4XPZk^pu>Y4Mg>6C}e||^>sa*Z2KnZ52N|HnG0$F z`G&|dLRS0Ictm~a3n*_t;UX(CV)#q#-_~f>Ap_1oY%e$hAj8a(^$`M0)JOvzCB)@7lNe+IIY1- zo=lq;gL3r412BA%8V3g(5H3WXE?B&%CiB@X!h+g;(Ew(SARSWTIs%W~6~~^P9c+)^ z^_Yjx8wT4Ah*(CPG7k;>8HMV^Nv9KvU;N;6)priIw-4S~{oKL04BsKRE&4jp z09c=gfI(1c!91En)k2qA3?+ukYH6&bZ%DawSqSkJ5R`@I5i5=O1kY9(I9#+r45iUP zB*og3@Clru@mxKxR$w12o=IT3g<2?Bpk~bJyY$?eRc&v4^tnq<^7&P3p1b5b@#LlF zKKcgmhVVezd;C~u8|f(wVMmD+h#?X>0T}j1$-^FId&mw4vM2uWBWPghg3?lZ0&fCn z&neo2W=)zNoR=wsdFjG6WPs_B;xzpA#sBsDdd}d?wo2 zxy~oXeDy!@moVoT`iN2=iZp{$KdYD@q7d+772=l>3u#7Jq#sw@4>KUdK*s*)*};K< zD=qs*TPD`sYBt+z%vTy%Ah5Hscqz^j$umjo(RKH4{n;~HnGa{`Ag*0*8Qs@1xo!{K z>rTr*H*RZ0%vka7lBW~Nr0s*K`pnO^GN+^oa?hy3My}H&3Nk`qUpOUBgK5&b3{E6+ z1b$sN1C6!8lia9u5RHvA)p}i3A|8Yh5rQ&ArxZ2i&@$Pmg~)GS)XhrwQ{d@{8!^!554>LAvO5K>rXuKdhv6bW;n7<)3zPK z9EB}PoDri~XFAj55uweCwy3afX9&4U5x#ErIu1m|-LNbCo{*2!V9DHo01S3noRFa4 zmL)qd+1Y()yBa6JRO!b-=tdf_B0aA;%39@dFt(?zrud^7*7o2FuRZ?ZY33~M`@4&2 zoCQ&fM_Bv5JKe87^!RJrnDehLUF^7Ty>8dJ`m~_0!iPw9on>ct#GZDUqb^B=WcclE zLQ5i36wFmZR>(p~#lDuOb@Vej1qc+vdV-@T(1@19Uc_KX*q1^@T3xM+_Gpm*MLTjc z2(jGH%jq^$TTovd-6P$T4r}T*LK2IFu@GcS@Ed6>R7H$mjpV0v3QWbukrt99M3;=z zIfCS4%8*R`;85Eh$RNqC)}hGI=xfEdUIQvYJY~w}rcL+JVc)@h;ik<^eW%ABf9X5yRtP?g%n=#HJ^ukG6EmyxUY=0CxJ|y&w}&`CR3b!1<_R2-3!m}wu(y%k+T+m zZY>n7tj>zrP}_RkjV>F=*m{c3SoFD4e1=87T0&n67J{Z=6Q)_163G85zB0H_ z(Au8}+P-+khxyz%%_9z{L=g$8nz%U7zo^<6@lATSdmFMx z=dG$^7oYz?@vE($YK=UsHGF;dO)NW7{HKxJpJ>gdK2|UKk!QvFLEoBmTqB7Jhkz08 z;EiX7I1r9d8V5om&}x$?k_S_^Uem`#Y=r0kg^X z3srSmOE<*@&%MXpYait~Q35z~@=dZ|1J0yBSuS+P9D>(@7K@?U4HT;ads=450zws` zlRP+siGytb_CG(cX0WrP*tznTr1iQwGKO|lpKDWheV}UV-mO)E z`u?^Qh11sQ;s<08&r4-__E|l6m~NEfcoSQzI+C`&Rjc}J%>y@!_+c9fCBocXAf``O z((HmO!?LTgy-zes*t$ul2_w{1@^hTkF~i86N+8%3NGkltgNSp$Vf?4QZ1NQfwcWwz zoJS=im`4^#ef% z$Fjp-9N{ieN`jAgn#Q)oYbum#!N+`Vd!;zz=!zSB)!2%>C5-TE3Nu5Bt$3ET|L`M) zXNrIO?CUI2`11W@$1sSG{IK|=v(GZmGg|S@*YE$bb_|;Hk{nP0nn*DTz};Yj-$Q{( zz+HFTK<#&Pvt}$20%^zDIukuy*M=p+L9mCer!h%P-&e-=Dcd zd-&&%Ja*|rBpHlgj|u+pQLG^Fgs0ZF-fP0 zO@ev6y&&wQSBe*fbS*A;q+Og71>FE3$v#kx^PGr*cUK6y0jdBVRWixKEt3ur`eK8^ zZLsMlAoyCWsW{XWi*bq`Tz|LI_4ZRB*-*~!M`06>G@)GEH8S_T(q2FxHq1xZ-*MKR z+Dd|UN{^ZLE``^G0$t{$BoUA^*&jm(}czG*v{jdvpQ*XlUZ*!1?F zZ|g~=dbWN0t)|8!3%Btt_g#2mV@s1UYkEa`}7TW_;u$D?h#yiIX# zP2f=Z$+;+Ci{KMi885SW&_!riG61xao5WJRr(K1GuPAc@k!@df< z3%=;Jt5;-`y)a9{Dk)=z;fpSFUJ1>r6c=1l4NAn|+VawM=|20g5UYPIez{8|#h;6i zC25S&gR~dEU0y?0N4N?VZVr2W9e@7{jA2)adP41?rJgqjDNB!`AOM`^3=%+y;A7fL%L+^HAY0{O1?gW7mBC+sS zg;MolS0cwW+7k1NNA#tF?!UXJZYP>`?JAVE^eRRW-GGoGzksjj8MI7=*yAdty{o?6`3 z+}LcNSuA^;WQ5+|)84wapH#SqzEiC_i_dx- zjS+`+ZbKP<$(S&knbTN=Jsm2i;1j}%F5-)EDifq!+RugY{F<|e4p2bM$0=euDO_O5 zUY1OQ1=9XaVGS2k!Z^$YvIkILEwt;w&k1)u2#!Yf1CmC_a7MOz8LYwfET&k2()xj4 z5=L7tc&c$;P_VkiJ_u1FDHR+_y#E5?T72IV*dGgPN!2A0hgj9vF$yy;*F&)9Dj_9? zF(>TxNK2r`h0P-Ps8n!ivxM}6<&-y;<;mYghm~Kn@=1{te=HN>_rXc)Vk1s5{}cf@ zGA)oMOnNY!AB6u)JW|pdk|;Z&6@f?g#G)-t4RtzCq4VYRZU-o97>h_T4w({DhDe6_ zrx5eBEUma;E$}J)6yKsBF{%Pa3qokUP$7RY%2)6j6?`@8ZYb@VMptxJ9x2AC(?r0D z-dRC!odBFd4PGZ10{|y7UErMqh!>&}EQeJ&+(-^8dK4Ji1iVaXO0NhL$H6hxHaHA#NfZiL> z0@~PuBecS%LHj)lr5vv)0Zo9xI!q@FGDCDoBSNoIAmYF_4-Y>~azSfk>LVYSQkx@n zHEVY6TvJn58|vr`*3ukF2(GC8qc_ghS~ZjFu20P^kE00*-yN+t;&?1_ zAL@M@ukB`etEERI*cM*gv-V3slWmsB; z*hOEK8nYN!M5Px6s4QY&04kWm!Y=nVt96?jFEJqLh)Ba?`@hECw1N}Yp?$x*s-k4u z6PkN8U5%Hfkq#gA>FyeK{EaWB9{u`P9!q^OcWF8`x_jrw^b5KcbkErC-DCF@FAnYO z>Dl?qlKvxLr;?wGBIPU>8ta5DgI>qxO$ZW7=0lSEVL>Kafuc(iJQ{RN7ADmv_I30Y z-)_h?1h8-1PZVDgasV_c+(bmm88%cvxwm2AvEJ{#OL$FRY15;&?SiL5a(5$gS(n{$yiNQiv|mJiq2XmbB6LtV%ZnFb z>e8>l6tQsyO~HCE`Z%MYC3qJ>TO<6Ou-m=2pHm1lh?%FL47`gAx(K)w!rD>^;rFx{ z_bvK84O?!7-}5`fZ*JRQcd04CA_RuK_IPd^Vor1)=su$*hNlmJHLdVl)RFQ1-KbT< znX)lb3|hy(c8qiw_kD~_gd31|_P38LE#Gy(YM<(?_)+Q($BO@@R07lRS@wQUc^A=0St)(r{b2RV>%P}q%j>+K{O@Y# zy~au9*WJSyMVX%7unzF6{JHXc`FO$4m(BOR>Xko3d7L#{_8gVH-)FCF>;L36jbRzA z%hwZm{o{l8$){wMTa^>algc-hpTqZfGn-lxVE@EzyqRbDX0Gx3_$T>`U}Med z4)vH?P=9H#8Fm>SFnrPQKMn61W5yxl9^=!-ADV)uoav`#pE+m#l=)}o%NCQR#?oOq zVVSeMX!*Y7rqtF@l3^cDs7b=m7|sWD<7`BVym{@Y&&Rs z#&)sFR5elcVAa!A->UitdyD;;{fzwu`w#6!N7}L3vDfi2$1{$-f2db8eJy$^Z|K7%jf zyV-Zx_oT1jd)MFWf3n6`^JL8%wQaR4YA0$xTKmP?AJi7>R@CjU`)b|y>)xunTyLvy zsb5jQqh70jp#JIlUo|KVS#Zz?8_qWr19br{@QJ`nfxm5RZd~1XTjQr1Uv2zlQ*+a? zrf&v^f+vD!gD(ev82nYJF?3t#Oz2yopElPu4>wOVpKAVU^Sj}i@agcY;h(nHTQ;`L zwmjYPot7)D$=3T?pKg6KVu-AdJQ?}xNHIDTor<1_J|F#WZ8dG{+h*HdZKuFn;+sEJ z_9GI3K3x2g4>MhPx5z87i~Y$W9UfL5*7FRWr~j(wDGKBN)$^*-!Ups_PD8RIdfuqm z*=O`T-k!r=g*3$sBoz}z$vlGv;=ky54r|8$t>;x`RQZ*jHz?KY4n1#F8rc1M-lX{0 z7nKp^Fy8h&sT{?xrUaEK)H#6sar_>|%!4>ja|q=}MS2+T z2Ae@y9QAvVwxPyR{LLx@uvPUad-b}M%DUak5tMeLg&EX?GCp#6X7cEa7M%J}aBKI* z?%4w(UQ9batSpXD>?kQfc>*z1;_Aj-rj5 zlxfismg1)ALkE!@&`T&)4xsD+(%&}n0gQg9m>13SZUK=#lu>z~(gnL)7iQUud=d>U z8`wZ_=fR@~j@~_^^#uoleO;NZcyAwSUEiFtSW!`Sp^L)+#sM*M>ZDu$261!d@R0+D z4hH+W@rUa}fanZH*R_0Nhh}FEc9mu)u~E7D5XO0<&reZ^Q^1Tfl^O6xCll;d7Q8X8 zf>kPOm34s524K!j%*Lufn;guEXr*fAW*+8cKG=b3SS_n#^$Y>PA9Iw!Sf-uimhgA*f1Mm zYuP%so^4>G>?XDmFD$;9-NH7rEo>{>#>Uuowu9|tyVwU{IODvpM#M>`C?% z`!xFudz$?R_F48h_6++Yc9wmfJUnc=!^5d1n*1oz7+3E^S%u4%ksW{ z-Z#nnrg+~p@6&kS4DZ{^$5T9>=J5=VXL-Dz$0vDwipQsUT;uT> z9^cCoy*$weuQE?0cp}LYDV|94M207_Jkie+lRPoS6Vp7Q@x%;I?B&T`p6uhvI8P>c zGRc!E1YPlDh9|Q;+0T=cJUPXa(>$s1f@<6PbJ`~=BX4XgXW~4Q;F%=PqgQ9Fd}@kMP4g*@PtEYDy?nZtPxtZZ zIG;}N=_H>{@#!?5&hY6hpYG?=lYDxLPfzn{jZe?;>AhU*w`~4l|1WJN*uYz)E%B3gjC&tIe>+`I0d_0_2w&rHW$Gh@sEVwS1 zH?&S-K*o`+xx6tvoHvDsG5qm7o9N0LVquIcsGT!T4F~Ct>^xsFl2<0y<<*W5N=JgH zf~U~(xn5)IscpH5t@V>*@|#un=G|;W9iN26)56 zlXFPd2MoSSKc1O1cJf5ZDb?O3z_inc)p6R#&A`I ztFF8Q%{T=}f`Gs@hMl*MOaxC&1oL(Ptt;=0ZQ7ALXVBJ;x8$p4!Y8`&uGpq+xlP+; zVSNbYZc$zxJEu5CcIM7G93y!)Ih=QN5`qG4htJvQrwTuL=EF*;ty^>F2x|eX;Zs;# z>b4^k#$%;?y}VD40PpGUIA*c|aRt$vF2nIrF6a%5O4FjRHJr-Oc@Vq02`8y|qBUpq9 zTC_=|`F298&RD*qGv9&j5(B1g07~6(zl0~VVWLyNwFdB|E8n%a2F#a_b>x}1S3tSD z94gCi^~8cHG0tApVe78nuAl-p92S);zOM>eyLKp?J=ep$m`NYzje*|qkqKb!WVS0G zk9GT3bmbGjt12*T8r73n3dPqN><(_Aoe2=$bn4WG@CHzV9OyOZ9ky$NAyN|kr$9n{ zz<&ITDtYTj=gg_@a4@*y6xvEJ-41rkHu46viCV$@1a0Qk+j3vwK{Z(a6}%9?P=mY~HN@&3D2JDSMB;$3hqQyx(+$sivU$77&VM~1hOELt5AbK}O zbQpwJ05n-qoVQ^227~Lv8>ll{t$qPAnt%>bWk;?%xB^U%Mywa2u_ch3T5)v~ZY{D^ zxlq?5*F;!f8H}+jKcJ6bq_i{>#CNX+Txlr>W8q*oL2W&#?uzm5bDhkCjkjX47^}Hd zymGNv)Gj@`tjPYLas1& zMK?By9OD`g3lQiEz|xCYmQXO-Y| zQ;g6tKMJsJjGb4MHOOp2hEe9`*m)*OZb3$rY^FNHxV44qP-ZLDq0Ba_LzywEGla}` zszaF_REIJ3CWBKf2?R|71YVQ|0s(nD@ zsOp`ueE(wAyXZnxy<6m{>OCSyRS(AU1B+D;(S@iwD{@rzgCa*&568X&|7J-t8t%+n zX7Xyw))T~Px)cc5g)s;q?2{nMQly?erx=GJFm%Y&vMl`uxQA7g=s8tcd#;5&vJJxG tBe`>`w)R|vu3oY{2>a6NN2Vb$p$g>T@pFo;#)kMsZl literal 0 HcmV?d00001 diff --git a/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.woff b/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.woff new file mode 100755 index 0000000000000000000000000000000000000000..f9309988aeab3868040d3b322658902098eba27f GIT binary patch literal 14984 zcmZ8|b8seK(C!=Cwr#($lZ~BhY}>Y-jcwc5*vZBlYh&9^ZhqhW{ZvpRobEY2 zRim2jc2|&)0Du6#g(m`l^xtUf0|3Fv_;2t37YPYfIRF6U=Qof04SefskYWWDCf0Ax zvBgA?Sg zQ{3X4{N{ANb;56uL&kuESlGIFd~-hEx-kF%7M7U{z_qbA{?BgvJGPPkQ1m-q%+}E3 zdtHw2HU7t!7$h5R$XB`1U|?VZ2x4oEo(?{~<9cW^U`%1|L<`O49o%ya3Cchk?TQjvHN{6At8vTKtqH+gT24Lz@);yzA(}YXmPMtu?=J) zB`AsehXP=+al-fk06b49&+lmeAMwbpQMYtnkU%E5*g+%ehk}td81f)!!euyQg~T*2 z)@9npKco9a9KNs1`!r1D7wjizEmb+j<)@`LL%3o_S^DOxFhSl--hj14 zM#H5aHC`i!yXJ}d7a=RP@L93co8&-xe2dITtXa!y%MBkDB~oaSX8=|B+}p%5@uonM zn_)dskE5dgxwy$B7UDtO_s#N{dQ@IiYRc?**2_dj%d{C+ob@a*k&~f+QCmvu@MvPv zXAzzv=m(mV@f35IWRg%#BWNS#Yb*+XqhW64orn;jVCARAp6(CT+dJl6*AU;? zM*P*yjc8Zknkp&+s)x#G((ur2&&kDr+QHf9@3~dEGc~r>L7*Gzy1Zi26w8WWema4O9nUHF1Ay`VkG|KN;jIkW!y|Iqm z_{%A18!12g;hLL=>v$cmr4i55J7qcYXU=B~yAkp<@s~C6tv|V{8@vThN7>Ar*+kUT zG#R!Mo!W$4Nb=yBdJDs4I&6_7L__a`awb5B)C3Ey=!p>9V1OES1_-UBB15l>gAY6! zgAcgD1lD&~n=am~Xzs0?{DhP>B#)UnBu6*&eKAo@JpMbD(YyVmvxqj z&@&kK=UwrH$rMA@KCPr0_vdj`DwkaL#P-jJHm=bJ?i!1 z8}!q?ktnS3m!tlo1#^A;Kj@_YSVeWK>j|c&ToS7G_GF@PG48OmO z9f5EK30J^t+iqJy*#ApP50`b1Itps9p(Y}?<(r0xM8Llb@Vv_bC)p7#QQo3mf&A%)o+*0URgNCG za4$QHzx$SKgZ`gRt#R0@*1!twSlSHhsoh;QsLMm8r|!LTG;ZrmyWdoHUi$My zm|}07P^J|LaHp^NgRiGf&NR(l5NXAon_%#8@W<{J!y{jdzW4$&DU}1qKxKQX)8XSL z?2mV_=`AIG5HC-7@$7A6{NO&-ydr#n74Uj&pF-Z$8y{E$zC4yusOM~M_{>Se`eA&?^+`>z6+^^e z-9zRTW5i&l^d`h>3TNz)Nke3o@P4#IaDYO_;5OYM^K&LQe2?L@Z-9NqAh8)@a0oa2 zBgZE0*v2lzCWIB9Dg+PnN60WgJt9X9;>y;|Kz%P)#Ht|n&;k+1CZVGLZfL=$4YG(l)XI zh)7x3yd;LHCXIWu%}triolkzfz}&Mv;H7!jBuw@gw*s$C$eu=Qa`1sc z5B}ui$H!Ce4T7GYUs-(D)QtlbRq-=L`#jXs?`*z*GJpGBAOxgH)eXYY$Hg~AG4DOq z=I=cl`sYCiMJzXE)U-~?69#ZqtZ&+AQf<3#MTmlm%g{%Umm_j2vh91ay zqv1Eg^xKZrziV{;&zZQAcXh9BJ$2;6V~=dAB!U$EAp{B=FqE%)N^YkP%oiRBdy5yc}^m({p@zFIc>%w~m)m9mf}!-OfW5B#m6e+P`6X=P7dmh0oT$%qeiyr_JA?e>=;4&-SO=&B8d&53>ph7P{!2UjA~-<}+y zPd{`k0wz%CSu^`360$||g)I7cO(uA+j+wedG2^l`$+y$zR;9Uh)P|Z7YDCGkDr?Emz*2pk z=&{N3d}iyDCb5)=dbZCriD^F425+7nvY$^RexMM&Y@~fu_8dox`Rv=J+(Qc9 zWn-qPasT@eA02E~FvN~G5E{6FE|YOYXW<6Lr~;=-HsGPY*-BMa)A~nN0YuSZvNR`; z?3GZSJ9gTT=B1hQ>?q8Z$4Lc+-+cJDeA2{i2Y;$GDd|}~D%QeStOPVz3q!BG*3_3< zsN9j}+#54rC}E;sx!5Odt+_wQl@-R;EOL%rm7PhG84}(HzEmEj=aMrK zIbG|+mgHB(oqX}A(s99tu1a)pigk_tAoUw~m?aQ&b3GAeI>XD0@EuIa$5l*WS1n*g zVJzBC98rNH+I+s$#v@W|d9@)RcYCycT4=Se+q`R8J-~u{;9-d3WS5+P6N)5m6Yiaf zW5r-x?=Ll_GwMmLqv7bF{L`WyIobWu>Q~t8YF*XhO1GVnn(*7@JyIqu1`U@KGOlS7 zDkIuCSkaEPKx|W0eg3B=i?9iL1FUT5wishps-be9I&>pL2hh8|-SBPq^WaW#5tOE~ zT}eCEtSL~gqcqjWVd7I9gOLIKbVX?4W{OO%%C0HvcP#h>_@M-fc}T%}R9KJL<`U9V zXu1u!HS7X0Ez~@YB)L|YW@u9W5-|tHX@2Vd^Q|Yoj6j=D&m1~FnIk%im7$;J?kgN=T59<}6@^cfW2XSeDIy;+ z;ETOlaWdwo5OPoV_ct=W{O6{#XMgMJ$9oeE-~m`CjpUZsw{hJ#0gvO&c?Cy}%w9Ms zF1qLs5n#X6OVn!u32_b_qY`#EKw4CB&te~7XZY(jWdCXUQ92kuUn~8)qF)SI2<%X% z$*37c99~#|tO)1lveW3!TBbb0&BE?sJ2VN2b`;e?d02KJA-GD}T=1K%plNHtYUYXp zgJD%O29qwCKm_~M0K>`K8^SP{D*2gCTZu`SM9S}-Ykw9zDoswD2oi?2TS?0j|YT&|8hjXaQoPL@9w`)i%-M<8&28g z`*F!&y{zlqjf@rLrt~FRSN5BK<&28)W4m>{vp08~u*1zMt6=`$Tiv_$EYw^6mW-W< zt8zy&d5h9t;u3Jj2lY=`hj8Cq$z7Jwz83FVg8EUT_;y_|+qcUF=C!0ITJ*U22Lx;V! zcKoPS=n8#~`Z=P6J*6*B$?-V%RjyUCCvVVwdl4E(WA=YtevNLvY$%)5Bc}Fw#;j-I z0#n6dHjW;Da&pE??)2+d3EbXdopfMeK@6A7^s%KeI88UNE8A_UQz9pRg$VLmUKJVl z4I&pPU<9*3OS$nt9-xj5K$8UbcV(lbl*jMiig1b^fo^TkNqIjEk~>Q^*t@Y56IUj>ezm7Kz-yTs!n(QG%R6u)`W@o3~fE4rr$BH|lu!66Zt>E+mol2P_*O ziCJ0f=UY}ApdzPxn7#+JwBo&4_`u(lc$Y5=bBVwn<&r;>yAaRJ-31VEoTj>*61yyd zp3YVTLPv?QW5862ulNZ1OgO37-b6gtqu(;CiQAmQ# zCr+Ycyg+WEcZ!?X&fSUptp-8 zOKi8O!M8Q-*Qu1ps0AggluG*V^1Nk{%4)ki%nw(VY+snRW|#=(2QwJB9_$3%HZg&v zGierEtLuJ=$|~f4f4fwK5=?TPAjUyj8Yew=i=kkkgavOh6g$X3)xPOz)zymuI+`8M zw>dd|>IZAe!R{&|(y{JJk1V~blgfVPyc@hkWl%sl(2&%1_ zBayVylj>~>f=ABwi~c<+Iw4?r-Y>*Ha5S^04!G0F`%{@_*=~3GPH#N7wy(VW#9K~% z^A}g?O}_Q?lKt*@WTk_H-hSSv3-$^pR130pW(KZ(yEogRXYxqJ=3(mI^u9}QZvQ-a z((-M|R_NJHj9Leb)GgW74j^HIe+xHZ9kE0~@bpOQ{p$rbO7MWSD}JS|^sjCkYlGuC zUORP_Sk^=&Xl>}jo)cc3(U8>A$EKMhU3Op5&q?!5bIRWKQy#{mHJe~z zpD_@@wKexPN7*mrUJtXFETM6Et`^w$d}C!Oti(ItQxZ<}ac+wqpcwP31>V3Xy^R=>z5USMBZKK+o&=70h3Nk7J|rhq`+&2=kGz zbKt(1>sMjxt*%JtH0X1QUjjrO+!WGqJ~>^oI7Jo_J)Kc&*z0~air!w9jp!g4?wfgq zJL+up-MtWP-#IVzI~_ZIvZ7?AAS3Z;mPEnwP_cT! z*JJkw8oBTf-J3$s=O1WSr-_ar>?Lq(5SfWB(V-~fojAhaKW3_-Gv)6Cs%N6kHOpSA zcS_*;`P_me1{t2on+Vr1a$ReDFnK`uz3Z3nG7l^pUjIFTxC`QjIs zw*4v<4CwC+ww4{v+O69!bR4?vCk|s{UsX-Jfap8;>_AXh$l|f<;E74Cz!jC7G9IXy zRd53A1wnR`fLa1lq+bZjJc+3|#A70PRV!DqsMBI+{Y`^Fjxpas$8>UHzBCi7^C*i6 zK(hW0jN5kPJk|E<^L0~z;qgZas_$AoR&%@#wjhOvWDm=21DL3NucshN z&4&0NC>nxBdAUC#X!+LbzQ^kjjbhE1k1OVX7~$`<-c{$9+pA7>tr~|B)r7k3PQii)1bP3cLR~PA43g zv4&593)87tEg~Q62W|9|3QnF4m?e!IAcZS5Ibl^1YcsARB`ADY4@045znu~7a01Rh z>+l$JuFC|4z7hK3+kCD|DCv!`W2+C<_BhK-N=Y> zl~TeiuMqwCt^g2?J(W(R_x%hzZ2vT01(hBOkf{W6GNbOatvp{|VWfZ@Gaj%s85B1e z{1-eVWEKKhhEWhGjoh&iS!ze1fT3o7ow#1s4uhlLS<=;VminN4iuf0PSxB_tM4{Q*zUBpS#fqtC8M||{+PW- z5(wRsj(WEBgf#w`o)_kNV2gkk)eH-#tUQ@!r1^IZh&ZD0`?tbafwU1|CVhznf zNcNSz+~+>zhi)M#9b%<-D2l7HP?UKitR+ZD(RSuH;DtL1{iZh<2ucun!sawL z`=q-fJdKD;G+Bv51liqQ+tU(A>7MJhhOnA&5qu5Rl=-K7=a^Bc5AfVym}bjN8}a31 zSC+FQ2;YpbwsQh&KyheTK+B>WMu-W!SdTKbq+HdKtis?NxkRxZ$qSeOCGaBhz|Z(DEp*18 z1VY0=kluAfiGjwwj;QdjMMGCGU*OjKSx<7Ei}Qj)i@i@!ss5pK%B8wKW43@}FZc$1 z-YoNXL5^b2WSlRy4ve@Z5jq~L&dXc<&fA`H7{ix;`+e}9bh&Hz9biU!LH$`ro>n{E z60{dR1cz+zB{R$pgoATCvTD1<7#BtK@y^5If#X$}l~ytQCQx-!#mp8tbkW2!!BzcyD)40=2|*Yu0mzK2QhCp1h#(R@$2;3wHfiXgEyLjy>&XZ{&M zX|0LbwAC69Uagm>U>z2#~Po-F%98OE1a8pWC?$^=_E$3P3gIXP#XRT!S%HmE3Nof?Q8}oXNel$6zZ6o5zeox?V*DP z#;gc)w7}{?5S6x8>d);zSK@Bkb2cjyb4fpGEQY8yvG{d=<)f#aeV&c7cz}dINU$Mi z(%?!S-H5nn;V;BHL`q}2RFUQG#`yzUbSbPC|xe%Okxc%);L zG_IfQ50^C{^A+S3h12axEIV`>eqL^5>t|45rId@hnBdprP!y7Z)cQ%p(8ARJ5fkIp zsXBB>UB(p=2!Bb&w+Ydbzv(Zoq=hleRCOX?9E-CqQnFv*KyBvL5g10fl#6st3l1r^ z{nu}0VD+#h3EPFLP)&G6MVtXL zojBMIJEED*owWecK9Axcvs^)EyxTG6kCj#khg~RI92J@%q-I~YswpGSNItHCSVz-Z z$aI%XJe@qt>YU7K`DFEY%(uxUQNk=Y1!MdKB!^j3lDhl& zB*r^qUR%{ANk;qd1q6@ttEMdwk?leq$2=`&Sl6|!Y!1R}KfWg7%;x6J6}JEmGNXFm zg|_y^m62>BRdyx`Y%_8b#P`(XCq2~>tsGTcLL!`UA*V>h`1J*&%T zdIHFYXJMi^OA7M~hfB<*ZueY+JM&>+Qfs#=kiLtfx0Ft)66%I_u?evJL21EhB1K~o z`y+e<;GfX>bBQsII2~e7232`QBzVq9t<1BI9gB&3v^Ec(tsL>=LHPD(3RZhi>+eHu zd|8z;=K=UNDEvmBsN1(=_6jNRl;dDjM9kO}*MC(c^F3lY{V&6y`f`AQZw?~-MqNy@ zTjAUYNJv+3iVw0y+J$1+cV)GLRf00|eV_EtDGG}ZM`MgKy1E3@Y68%4IWb*yvmw;1 zW4+u|$L@h*3@+;&b&FewrGx#rG#a-Y6k`B#0lUWXJ{=|geA4hq+^u1speQWAISOkxN6G2HT#(@9Tx^dB9XN_J?3OOn|~ zl$aAWj7%vg4nFC>fH5@o+O&Bq=Yw0FizVKxE{rDu<>BtzXAf=xem*|A%c3k`_IB1; zS?QAC^M3G%gl?zt#n9;@+H;`p^q*0YcXU&pIoTNQ@}1(qL22#*r= zZZi_}Yy%6t5zSkDn-$(McjvFXR9jx!dN;Or+L1<0IbO;R%_-O(w+5pxh#!$=qJ4Y4 zYD|XROqif~U`MF-?cxEZyv;j173tj z-YY(e%y5_KiS|+MCa32c^uh!YtRyu#U+7JX-2>9+vtNsXrX)PoX~9gbOv0o7fgfj} zB`?g8I*)BLm-MV-8F|9RS6zfd%mWs5oU49T_0Hc?R!?L211om!o0F5?OCs*R=6-{c#%b^7GQ}uK~jPH z!qWw1S0j(t4IW+yW|v#OYAN)jCMFo4AluBz$FX=j+Sk*9N}jv6sek`8*blveRYyK6 z@$$QlJR0o@v$S+f-zsLw0nh#kUV&fD{$c1Ky*FirKmqzg+)FWg)*qYr#!&xh)r5FM zyIhdtLDGe=z-F!B!f`gKQ;5@DmkA~JFJ)}&q2vWU*3SVpi6R6uxf)tZkEGzFa5#xh zgxWZZW?URJ?Z)bcPP-?uZsE@O`(e|((Jc)+yo;i4MIL;)hlm(2w741^jymCajG}`Y z0+9`yJ4PswEoFzGwoK&Bt{R)>WKNgeyhyZZrCWq%%VuYWOSZTCmc7B@AINXaIYw>g zD(_7~W$3#FFPFybE@REcF<7d=>Bl!Qs|)m~SLEeCXQD;JBti`=eSRQFLEkCdcI{wy zZh^j@{zDOlr}L}zgS3@RiQBzf2Jwro|}z zp(8`DShFcww4*$ph=`Zv&Qf;2lWqEvw#uf03PUx5*6Zt_ixy%t9Lsse#_!)n3$--l zOf$;2nUJKM8%rIVj%qU1>XT_ym2MR4aaD{P*8oOSZgIqcWfWlkoR%D~ll0=66q}CTgR^m^OW6AzkH7eH)iozB+LoEQPHk( z#`+MS)QEj`X~>v7ZPYe^*p)Xt3}Ja0T^Df?O^X*F|EApS<~55@Q05SkK0sF+UD=#y zt7#A&M)vf*n^sI0F~cOr_VJvOH0Xd?%4c zS9%8jMQZ#au03wIpvh_4m~jGGx}6aI{d!htmWrf+Ec501JY=~N`(k@SGWn!aRsfxN){B8UN2djrCZY-c;VfAmwKt~0mYbZs}* zN)bzhWb*t}1j2|hWp6O^-@hIy=snZ+vUl(7haLy(cRSqP)j6yC>k9j)-0U_2f`oC* zDq6$j2-(gxSw{;!Dp96XDiCcn<=s}RfXP?}T|Y2spwLwsB6ETb1}TfF=R{7Hzpnh5 zA8mde1`9$mIOIAp6)$HGzWUmv@fqHkz82Ew-Q~St6-GJ%T zoE#?-c3l0~iaA9*ZHhlS4{FA<9Xf40OlkBmvD;}@=7o63Ay)&<*d*Y$1s;!ljpE;>z#T%*x>L7ZnjI45Ij{?bC*!?k!+qG ztdZ3sm+s_sl6t;4RC2XWn51!HZA6K~SFd{_-)wmP_l?z2qE~E~<2OIQ+O+`I`?nv4 zTY=XT@qB)6R50(?106eq%h-+tvkEe1h`*@lmM&+x3DEC^osEhDdqcgXu%ke2MH&Xk z1C-O3ZCc_QBqYIvgg?eabiv}wJFj##c2D8mmh`lixXcu@YxCQrG8!B!t|Fs3VzCQ; z9hr_t$>&PsMb)7~T9Gy2%f@h*+#5)SQ1_;4J^h9y10)bshZ z;l2nhm_6Q$h;b}ZWEkFj``_4Ccc@<0bZ^yIU;nEXlUv%4ty-&3ERH>Fs*hBk2V4(@zX=>s`_S;> znv9FMT_}=x6fgK5Eocs51k=oLfx-1*kl`Xt-`Wy>}^8>`FDC3BHmx0tiP7SUAm<*Y2o55|>ORCS?h9s0JBXbw;#Cph$cb&794ji= z+q>GiW^0_In6F@|`Go$PG?<~CdAy08(5Tw{%|4#eF}0z$P|{heEvSj_fb)BSxH5<| z05&!eJ_hd`J6pRTn3-`De*kX~6ob6;5$76=(raIQ zLf|D#m~aFvX;k~)4ngj9jDkYEH>=9Bl0Y4lFbo2hwZ;8SM5yle*pjPB#+xSFQmlZS zx-6>M44W~rAali^78Y#mRKbxFx=eMiUEa9z(ucTGd4XT}DvL>5sH(2)4?_+6KO;-8 zrn@NfBWJqrmF0aeV)74j{RNieoN=x1WWDtZBl&cYz_p4>6*bDFG3D`jit{?pN}=Kb zA$HRnUz77!U1Y__9o>Mc9eAhu-xJAe)|vDDd>|D0$V1~)51#MF`!ucYiH0PDBh7hd zP@~9L9U6_>0ITN)i|*;n^J#Cuv4^nl9;%&+iqY3>S?5D)G#pDe#$!hX0bHuh9I~vq zA2D4T@VATH2!##Rj~ya`D*lSE^NQsk@^8~~tHFwqGoQhqMQ94Y#*!-iK3j^ml#r&i zOqazq3pA5ARb?ZISzwF}DezJS|A=-F4_sjNEx`+yGyRH{IhD+PA05?2fF70oRRvbTyn=GafV{2>-SOR5)yp}dOVJQnupdB__2H{ zi%Re7Q-_+nW%M@Y$ImbA3k6IhfhQs^_th%;8QPSFoVu@2dYLVA7&B7wEV3z3DWY|4`dJ^1W>(H5b9w2ewH26TeK*KTVdYH@0yhXow`Vt zEiQb%wNti%zh@KY^!l}LTgdz&+oC$>Osld`vBzQUXWP=M-9c}NQL_(n4;71kn5XGo zmVOZ3ksQkzy(!yLlj|9MYY%lc=Ah@ZOz?K%F2w`tdy65K9JF()4*MSTo^&Wn?TB3P zh4PYQtzNI2laZ^V1u@2%VYXofo#$f9?} z{g5ky{arkjo0YZngdjFBkKC`Vo`@ZkWNC`C_ZF7g_;LQ^=gJK60isc0nfD||;QbLh zqm?XPW>-Ds0dZJbpO zb}am_%z^ldSG0U6@a*@mqlI3hkR}r6(>VCjfiSOI46I~*s;(97Ro)8+>zQ@jlv$49PArKvxkxgwBdB;#)2(4-!CdDVF!4L+<>%U)0rggTDio~bmuS8 z*DD7#>a9n~qz&fVQ)Srb$Y8w@3@3OW!=V6HjEqk8@ilHta1dF<-HO!0i~(!}5~#<= z!n4PX!FG>le~I^w5dGJxZstqGGH1pB;o}eE(Eh6Be7L8vtB>x7O+Oo_hROX4XeF%iNrNuDbMF%%Fj5&tjH zZ7s_!M;$vi4iUxIB2MrA(l$%5jD^&&(JiBh?Iq~B=emhrk`8_i{Ffx(xx%$@JBb4$SlNt~?WQ(N zrbFis>F-n+Ewf$L%LDR}95)U!ev7AlHLtPc>%(EeK6Xt72Nfmhq@VH#)l!BvMwO(w<36$uo$fW(#UmwvEP`o}J zPq{_b+bON@JG)PrK_|W_HmDM^PA|s$o1Y4khOl?^I?z#%nE! z{XC7pZ{9)DmQ?j7%D20V@pyT&Qdj#Tq9{+FAHx6pAWx)0Eu9L z5P*=4FobZ6NRH@+n21=7xPVTSv+KMKCW`On=9T!~!Jpg?S1Asw@0mRV42*4P_1jnSrl*M$yOvfC< ze8(ciO2@{;PRE|bp~m6EF~AAJsl@q<^NGucYk}L0JBj-b_Z|-(j~tH=PZiGu&krvf z?;0O~55)h8AAsM8|4D#LU_uZ>@SEVAkd#n}P=_#?aDecVh?K~UsE=5H*n_x`xQBR& z_?m=}M294iWQb&!6qi(l)POXKw3+ms44W*0Y=CT+9Fbg_+<`ose1!a!f}O&PBAa53 z5}Zw{%81H?s+?+r8k<^z+JSn2=DS1cf3GEvp@e?oJ^-k!K_hm=RJ*f~ zEPy^8)bGD}--KRiQ5NiBg;%7?zy1B=B*CHtc5B`!uGQRYFqnRBRXcLS z5pE{wla8bepSRui&#pNdE4gXH30(*{{GCl_2&(6MoneF?{$&T+Oa5g?MnXO=2THwJ zNyu0l{80#UvlT~tQNytW?0(Xc(S$a90`+1L4jIB^YnjWGh~q2PwiAbQyrJWIs()GM z-LTx|QI(~BF!yZyu3jYOyxi)d6q1}%F&nsTiNOoMg)@>4DswO zd7&f@=3|L%Ce-$h8rp+jmYY_uB#UFDQ4=Lb^GwKDnU=3`E4&nCwr*b=o=B|s^hs1R#V!agd6;mD@GGo*1m^2txCCYJ=jET}Lb#)NzldN#7*)#TZtJX7)bZh()DN<&DULB-z4J%ASOCDOS zi0&0yIg1V%+Atv2pu!%dK1bsWTZ|X)or9^6BWGs)3I=Y28W_*KeR-jvY4B^gK*h{y^sAn)+SUTnDOF`orBX|!{9+a4 zVtJ-&laFDBi^D=mo7d6d<;Dz!8i#DF~u*T d`d@*P)=+z2O9=Gccp2C_0H}G=_V0V@{{Zm~b;kez literal 0 HcmV?d00001 diff --git a/imgr/imgrpkg/static-content/popper.min.js b/imgr/imgrpkg/static-content/popper.min.js new file mode 100644 index 00000000..8a17212f --- /dev/null +++ b/imgr/imgrpkg/static-content/popper.min.js @@ -0,0 +1,5 @@ +/* + Copyright (C) Federico Zivolo 2019 + Distributed under the MIT License (license terms are at http://opensource.org/licenses/MIT). + */(function(e,t){'object'==typeof exports&&'undefined'!=typeof module?module.exports=t():'function'==typeof define&&define.amd?define(t):e.Popper=t()})(this,function(){'use strict';function e(e){return e&&'[object Function]'==={}.toString.call(e)}function t(e,t){if(1!==e.nodeType)return[];var o=e.ownerDocument.defaultView,n=o.getComputedStyle(e,null);return t?n[t]:n}function o(e){return'HTML'===e.nodeName?e:e.parentNode||e.host}function n(e){if(!e)return document.body;switch(e.nodeName){case'HTML':case'BODY':return e.ownerDocument.body;case'#document':return e.body;}var i=t(e),r=i.overflow,p=i.overflowX,s=i.overflowY;return /(auto|scroll|overlay)/.test(r+s+p)?e:n(o(e))}function i(e){return e&&e.referenceNode?e.referenceNode:e}function r(e){return 11===e?re:10===e?pe:re||pe}function p(e){if(!e)return document.documentElement;for(var o=r(10)?document.body:null,n=e.offsetParent||null;n===o&&e.nextElementSibling;)n=(e=e.nextElementSibling).offsetParent;var i=n&&n.nodeName;return i&&'BODY'!==i&&'HTML'!==i?-1!==['TH','TD','TABLE'].indexOf(n.nodeName)&&'static'===t(n,'position')?p(n):n:e?e.ownerDocument.documentElement:document.documentElement}function s(e){var t=e.nodeName;return'BODY'!==t&&('HTML'===t||p(e.firstElementChild)===e)}function d(e){return null===e.parentNode?e:d(e.parentNode)}function a(e,t){if(!e||!e.nodeType||!t||!t.nodeType)return document.documentElement;var o=e.compareDocumentPosition(t)&Node.DOCUMENT_POSITION_FOLLOWING,n=o?e:t,i=o?t:e,r=document.createRange();r.setStart(n,0),r.setEnd(i,0);var l=r.commonAncestorContainer;if(e!==l&&t!==l||n.contains(i))return s(l)?l:p(l);var f=d(e);return f.host?a(f.host,t):a(e,d(t).host)}function l(e){var t=1=o.clientWidth&&n>=o.clientHeight}),l=0a[e]&&!t.escapeWithReference&&(n=Q(f[o],a[e]-('right'===e?f.width:f.height))),ae({},o,n)}};return l.forEach(function(e){var t=-1===['left','top'].indexOf(e)?'secondary':'primary';f=le({},f,m[t](e))}),e.offsets.popper=f,e},priority:['left','right','top','bottom'],padding:5,boundariesElement:'scrollParent'},keepTogether:{order:400,enabled:!0,fn:function(e){var t=e.offsets,o=t.popper,n=t.reference,i=e.placement.split('-')[0],r=Z,p=-1!==['top','bottom'].indexOf(i),s=p?'right':'bottom',d=p?'left':'top',a=p?'width':'height';return o[s]r(n[s])&&(e.offsets.popper[d]=r(n[s])),e}},arrow:{order:500,enabled:!0,fn:function(e,o){var n;if(!K(e.instance.modifiers,'arrow','keepTogether'))return e;var i=o.element;if('string'==typeof i){if(i=e.instance.popper.querySelector(i),!i)return e;}else if(!e.instance.popper.contains(i))return console.warn('WARNING: `arrow.element` must be child of its popper element!'),e;var r=e.placement.split('-')[0],p=e.offsets,s=p.popper,d=p.reference,a=-1!==['left','right'].indexOf(r),l=a?'height':'width',f=a?'Top':'Left',m=f.toLowerCase(),h=a?'left':'top',c=a?'bottom':'right',u=S(i)[l];d[c]-us[c]&&(e.offsets.popper[m]+=d[m]+u-s[c]),e.offsets.popper=g(e.offsets.popper);var b=d[m]+d[l]/2-u/2,w=t(e.instance.popper),y=parseFloat(w['margin'+f],10),E=parseFloat(w['border'+f+'Width'],10),v=b-e.offsets.popper[m]-y-E;return v=ee(Q(s[l]-u,v),0),e.arrowElement=i,e.offsets.arrow=(n={},ae(n,m,$(v)),ae(n,h,''),n),e},element:'[x-arrow]'},flip:{order:600,enabled:!0,fn:function(e,t){if(W(e.instance.modifiers,'inner'))return e;if(e.flipped&&e.placement===e.originalPlacement)return e;var o=v(e.instance.popper,e.instance.reference,t.padding,t.boundariesElement,e.positionFixed),n=e.placement.split('-')[0],i=T(n),r=e.placement.split('-')[1]||'',p=[];switch(t.behavior){case ce.FLIP:p=[n,i];break;case ce.CLOCKWISE:p=G(n);break;case ce.COUNTERCLOCKWISE:p=G(n,!0);break;default:p=t.behavior;}return p.forEach(function(s,d){if(n!==s||p.length===d+1)return e;n=e.placement.split('-')[0],i=T(n);var a=e.offsets.popper,l=e.offsets.reference,f=Z,m='left'===n&&f(a.right)>f(l.left)||'right'===n&&f(a.left)f(l.top)||'bottom'===n&&f(a.top)f(o.right),g=f(a.top)f(o.bottom),b='left'===n&&h||'right'===n&&c||'top'===n&&g||'bottom'===n&&u,w=-1!==['top','bottom'].indexOf(n),y=!!t.flipVariations&&(w&&'start'===r&&h||w&&'end'===r&&c||!w&&'start'===r&&g||!w&&'end'===r&&u),E=!!t.flipVariationsByContent&&(w&&'start'===r&&c||w&&'end'===r&&h||!w&&'start'===r&&u||!w&&'end'===r&&g),v=y||E;(m||b||v)&&(e.flipped=!0,(m||b)&&(n=p[d+1]),v&&(r=z(r)),e.placement=n+(r?'-'+r:''),e.offsets.popper=le({},e.offsets.popper,C(e.instance.popper,e.offsets.reference,e.placement)),e=P(e.instance.modifiers,e,'flip'))}),e},behavior:'flip',padding:5,boundariesElement:'viewport',flipVariations:!1,flipVariationsByContent:!1},inner:{order:700,enabled:!1,fn:function(e){var t=e.placement,o=t.split('-')[0],n=e.offsets,i=n.popper,r=n.reference,p=-1!==['left','right'].indexOf(o),s=-1===['top','left'].indexOf(o);return i[p?'left':'top']=r[o]-(s?i[p?'width':'height']:0),e.placement=T(t),e.offsets.popper=g(i),e}},hide:{order:800,enabled:!0,fn:function(e){if(!K(e.instance.modifiers,'hide','preventOverflow'))return e;var t=e.offsets.reference,o=D(e.instance.modifiers,function(e){return'preventOverflow'===e.name}).boundaries;if(t.bottomo.right||t.top>o.bottom||t.rightwindow.devicePixelRatio||!fe),c='bottom'===o?'top':'bottom',g='right'===n?'left':'right',b=B('transform');if(d='bottom'==c?'HTML'===l.nodeName?-l.clientHeight+h.bottom:-f.height+h.bottom:h.top,s='right'==g?'HTML'===l.nodeName?-l.clientWidth+h.right:-f.width+h.right:h.left,a&&b)m[b]='translate3d('+s+'px, '+d+'px, 0)',m[c]=0,m[g]=0,m.willChange='transform';else{var w='bottom'==c?-1:1,y='right'==g?-1:1;m[c]=d*w,m[g]=s*y,m.willChange=c+', '+g}var E={"x-placement":e.placement};return e.attributes=le({},E,e.attributes),e.styles=le({},m,e.styles),e.arrowStyles=le({},e.offsets.arrow,e.arrowStyles),e},gpuAcceleration:!0,x:'bottom',y:'right'},applyStyle:{order:900,enabled:!0,fn:function(e){return V(e.instance.popper,e.styles),j(e.instance.popper,e.attributes),e.arrowElement&&Object.keys(e.arrowStyles).length&&V(e.arrowElement,e.arrowStyles),e},onLoad:function(e,t,o,n,i){var r=L(i,t,e,o.positionFixed),p=O(o.placement,r,t,e,o.modifiers.flip.boundariesElement,o.modifiers.flip.padding);return t.setAttribute('x-placement',p),V(t,{position:o.positionFixed?'fixed':'absolute'}),o},gpuAcceleration:void 0}}},ge}); +//# sourceMappingURL=popper.min.js.map diff --git a/imgr/imgrpkg/static-content/styles.css b/imgr/imgrpkg/static-content/styles.css new file mode 100644 index 00000000..fedfd863 --- /dev/null +++ b/imgr/imgrpkg/static-content/styles.css @@ -0,0 +1,71 @@ +/* + Copyright (c) 2015-2021, NVIDIA CORPORATION. + SPDX-License-Identifier: Apache-2.0 +*/ + +.table td.fit, +.table th.fit { + white-space: nowrap; + width: 1%; +} + +body { padding-top: 70px; } + +.no-margin { margin: 0; } + +pre.code { + background-color: #e9ecef; + border-radius: .25rem; + padding: 20px; +} + +.clickable { + cursor: pointer; +} + +span.jstExpand, span.jstFold { + cursor: pointer; +} + +.jstValue { + white-space: pre-wrap; +} +.jstComma { + white-space: pre-wrap; +} +.jstProperty { + color: #666; + word-wrap: break-word; +} +.jstBracket { + white-space: pre-wrap;; +} +.jstBool { + color: #2525CC; +} +.jstNum { + color: #D036D0; +} +.jstNull { + color: gray; +} +.jstStr { + color: #2DB669; +} +.jstFold:after { + content: ' -'; + cursor: pointer; +} +.jstExpand { + white-space: normal; +} +.jstExpand:after { + content: ' +'; + cursor: pointer; +} +.jstFolded { + white-space: normal !important; +} +.jstHiddenBlock { + display: none; +} diff --git a/make-static-content/.gitignore b/make-static-content/.gitignore new file mode 100644 index 00000000..61c4948c --- /dev/null +++ b/make-static-content/.gitignore @@ -0,0 +1 @@ +make-static-content diff --git a/make-static-content/Makefile b/make-static-content/Makefile index e70a6227..e8fd6003 100644 --- a/make-static-content/Makefile +++ b/make-static-content/Makefile @@ -3,4 +3,7 @@ gosubdir := github.com/NVIDIA/proxyfs/make-static-content +generatedfiles := \ + make-static-content + include ../GoMakefile diff --git a/make-static-content/main.go b/make-static-content/main.go index fda341c8..76d90c65 100644 --- a/make-static-content/main.go +++ b/make-static-content/main.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "fmt" "io/ioutil" "os" @@ -33,6 +34,7 @@ func main() { contentName string contentType string dstFile *os.File + dstFileContents bytes.Buffer dstFileName string err error packageName string @@ -90,21 +92,25 @@ func main() { panic(err.Error()) } case "b": - _, err = dstFile.Write([]byte(fmt.Sprintf("var %vContent = []byte{", contentName))) + _, err = dstFileContents.Write([]byte(fmt.Sprintf("var %vContent = []byte{", contentName))) if nil != err { panic(err.Error()) } for srcFileContentIndex, srcFileContentByte = range srcFileContents { - if 0 == (srcFileContentIndex % bytesPerLine) { - _, err = dstFile.Write([]byte(fmt.Sprintf("\n\t0x%02X,", srcFileContentByte))) + if (srcFileContentIndex % bytesPerLine) == 0 { + _, err = dstFileContents.Write([]byte(fmt.Sprintf("\n\t0x%02X,", srcFileContentByte))) } else { - _, err = dstFile.Write([]byte(fmt.Sprintf(" 0x%02X,", srcFileContentByte))) + _, err = dstFileContents.Write([]byte(fmt.Sprintf(" 0x%02X,", srcFileContentByte))) } if nil != err { panic(err.Error()) } } - _, err = dstFile.Write([]byte("\n}\n")) + _, err = dstFileContents.Write([]byte("\n}\n")) + if nil != err { + panic(err.Error()) + } + _, err = dstFile.Write(dstFileContents.Bytes()) if nil != err { panic(err.Error()) } From 809281a00b274d81b2218e8f1d3fd15a04e8b489 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 11 Aug 2021 11:56:33 -0700 Subject: [PATCH 052/258] [WIP]additions to imgr RESTful API --- ilayout/api.go | 3 +- imgr/imgrpkg/globals.go | 20 ++++---- imgr/imgrpkg/http-server.go | 96 +++++++++++++++++++++++++++++++++++-- 3 files changed, 105 insertions(+), 14 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 6abd6832..696297dd 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -380,7 +380,8 @@ type InodeHeadV1Struct struct { PayloadObjectOffset uint64 // For Dir & File Inodes, starting offset in the Object of the root of the Directory or ExtentMap B+Tree PayloadObjectLength uint64 // For Dir & File Inodes, number of bytes in the Object of the root of the Directory or ExtentMap B+Tree SymLinkTarget string // For SymLink Inodes, the target of the link - Layout []InodeHeadLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable + Layout []InodeHeadLayoutEntryV1Struct // For Dir Inodes, describes the data and space occupied by the Payload-described B+Tree + // For File Inodes, describes the data and space occupied by the Payload-described B+Tree as well as the File's contents } // MarshalInodeHeadV1 encodes inodeHeadV1 to inodeHeadV1Buf. diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 634b2042..5529e8a9 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -81,14 +81,18 @@ type chunkedPutContextStruct struct { } type statsStruct struct { - DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ - GetConfigUsecs bucketstats.BucketLog2Round // GET /config - GetStatsUsecs bucketstats.BucketLog2Round // GET /stats - GetVersionUsecs bucketstats.BucketLog2Round // GET /version - GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume - GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ - PostVolumeUsecs bucketstats.BucketLog2Round // POST /volume/ - PutVolumeUsecs bucketstats.BucketLog2Round // PUT /volume/ + DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ + GetConfigUsecs bucketstats.BucketLog2Round // GET /config + GetStatsUsecs bucketstats.BucketLog2Round // GET /stats + GetVersionUsecs bucketstats.BucketLog2Round // GET /version + GetVolumeInodeUsecs bucketstats.BucketLog2Round // GET /volume//inode/ + GetVolumeInodeLayoutUsecs bucketstats.BucketLog2Round // GET /volume//inode//layout + GetVolumeInodePayloadUsecs bucketstats.BucketLog2Round // GET /volume//inode//payload + GetVolumeLayoutUsecs bucketstats.BucketLog2Round // GET /volume//layout + GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume + GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ + PostVolumeUsecs bucketstats.BucketLog2Round // POST /volume/ + PutVolumeUsecs bucketstats.BucketLog2Round // PUT /volume/ AdjustInodeTableEntryOpenCountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).AdjustInodeTableEntryOpenCount() DeleteInodeTableEntryUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).DeleteInodeTableEntry() diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index ca10db03..d8d95c22 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -302,10 +302,16 @@ func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Req func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( - err error - jsonToReturn []byte - pathSplit []string - startTime time.Time + err error + inodeNumberAs16HexDigits string + inodeNumberAsUint64 uint64 + jsonToReturn []byte + mustBeInode string + mustBeLayout string + mustBeLayoutOrPayload string + pathSplit []string + startTime time.Time + volumeName string ) startTime = time.Now() @@ -314,6 +320,8 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ switch len(pathSplit) { case 2: + // Form: /volume + defer func() { globals.stats.GetVolumeListUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -329,11 +337,15 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logWarnf("responseWriter.Write(jsonToReturn) failed: %v", err) } case 3: + // Form: /volume/ + defer func() { globals.stats.GetVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - jsonToReturn, err = getVolumeAsJSON(pathSplit[2]) + volumeName = pathSplit[2] + + jsonToReturn, err = getVolumeAsJSON(volumeName) if nil == err { responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(jsonToReturn))) responseWriter.Header().Set("Content-Type", "application/json") @@ -346,6 +358,80 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ } else { responseWriter.WriteHeader(http.StatusNotFound) } + case 4: + // Form: /volume//layout + + volumeName = pathSplit[2] + mustBeLayout = pathSplit[3] + + switch mustBeLayout { + case "layout": + defer func() { + globals.stats.GetVolumeLayoutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + // TODO + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } + case 5: + // Form: /volume//inode/ + + volumeName = pathSplit[2] + mustBeInode = pathSplit[3] + inodeNumberAs16HexDigits = pathSplit[4] + + switch mustBeInode { + case "inode": + inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAs16HexDigits, 16, 64) + if nil != err { + responseWriter.WriteHeader(http.StatusBadRequest) + } else { + fmt.Println("UNDO: inodeNumberAsUint64:", inodeNumberAsUint64) + defer func() { + globals.stats.GetVolumeInodeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + // TODO + } + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } + case 6: + // Form: /volume//inode//layout + // Form: /volume//inode//payload + + volumeName = pathSplit[2] + mustBeInode = pathSplit[3] + inodeNumberAs16HexDigits = pathSplit[4] + mustBeLayoutOrPayload = pathSplit[5] + + switch mustBeInode { + case "inode": + inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAs16HexDigits, 16, 64) + if nil != err { + responseWriter.WriteHeader(http.StatusBadRequest) + } else { + switch mustBeLayoutOrPayload { + case "layout": + defer func() { + globals.stats.GetVolumeInodeLayoutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + // TODO + case "payload": + defer func() { + globals.stats.GetVolumeInodePayloadUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + // TODO + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } + } + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } default: responseWriter.WriteHeader(http.StatusBadRequest) } From 06438236317df46a6d9d48b71c07931303868e69 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 11 Aug 2021 15:14:44 -0700 Subject: [PATCH 053/258] Flushed out parsing of new imgr WebAPIs --- imgr/imgrpkg/globals.go | 6 +- imgr/imgrpkg/http-server.go | 255 +++++++++++++++++++++++++++++++----- imgr/imgrpkg/volume.go | 106 --------------- 3 files changed, 227 insertions(+), 140 deletions(-) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 5529e8a9..70dbb111 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -85,9 +85,9 @@ type statsStruct struct { GetConfigUsecs bucketstats.BucketLog2Round // GET /config GetStatsUsecs bucketstats.BucketLog2Round // GET /stats GetVersionUsecs bucketstats.BucketLog2Round // GET /version - GetVolumeInodeUsecs bucketstats.BucketLog2Round // GET /volume//inode/ - GetVolumeInodeLayoutUsecs bucketstats.BucketLog2Round // GET /volume//inode//layout - GetVolumeInodePayloadUsecs bucketstats.BucketLog2Round // GET /volume//inode//payload + GetVolumeInodeUsecs bucketstats.BucketLog2Round // GET /volume//inode/ + GetVolumeInodeLayoutUsecs bucketstats.BucketLog2Round // GET /volume//inode//layout + GetVolumeInodePayloadUsecs bucketstats.BucketLog2Round // GET /volume//inode//payload GetVolumeLayoutUsecs bucketstats.BucketLog2Round // GET /volume//layout GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index d8d95c22..ee947cd3 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -17,6 +17,7 @@ import ( "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/version" + "github.com/NVIDIA/sortedmap" ) const ( @@ -300,18 +301,35 @@ func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Req } } +type volumeGETStruct struct { + Name string + StorageURL string + HealthyMounts uint64 + LeasesExpiredMounts uint64 + AuthTokenExpiredMounts uint64 +} + func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( - err error - inodeNumberAs16HexDigits string - inodeNumberAsUint64 uint64 - jsonToReturn []byte - mustBeInode string - mustBeLayout string - mustBeLayoutOrPayload string - pathSplit []string - startTime time.Time - volumeName string + err error + inodeNumberAsHexDigits string + inodeNumberAsUint64 uint64 + mustBeInode string + mustBeLayout string + mustBeLayoutOrPayload string + ok bool + pathSplit []string + startTime time.Time + toReturnTODO string + volumeAsStruct *volumeStruct + volumeAsValue sortedmap.Value + volumeGET *volumeGETStruct + volumeGETAsJSON []byte + volumeGETList []*volumeGETStruct + volumeGETListIndex int + volumeGETListAsJSON []byte + volumeGETListLen int + volumeName string ) startTime = time.Now() @@ -326,15 +344,52 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ globals.stats.GetVolumeListUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - jsonToReturn = getVolumeListAsJSON() + globals.Lock() + + volumeGETListLen, err = globals.volumeMap.Len() + if nil != err { + logFatal(err) + } + + volumeGETList = make([]*volumeGETStruct, volumeGETListLen) + + for volumeGETListIndex = 0; volumeGETListIndex < volumeGETListLen; volumeGETListIndex++ { + _, volumeAsValue, ok, err = globals.volumeMap.GetByIndex(volumeGETListIndex) + if nil != err { + logFatal(err) + } + if !ok { + logFatalf("globals.volumeMap[] len (%d) is wrong", volumeGETListLen) + } + + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("globals.volumeMap[%d] was not a *volumeStruct", volumeGETListIndex) + } + + volumeGETList[volumeGETListIndex] = &volumeGETStruct{ + Name: volumeAsStruct.name, + StorageURL: volumeAsStruct.storageURL, + HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), + LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), + AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), + } + } + + globals.Unlock() - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(jsonToReturn))) + volumeGETListAsJSON, err = json.Marshal(volumeGETList) + if nil != err { + logFatal(err) + } + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeGETListAsJSON))) responseWriter.Header().Set("Content-Type", "application/json") responseWriter.WriteHeader(http.StatusOK) - _, err = responseWriter.Write(jsonToReturn) + _, err = responseWriter.Write(volumeGETListAsJSON) if nil != err { - logWarnf("responseWriter.Write(jsonToReturn) failed: %v", err) + logWarnf("responseWriter.Write(volumeGETListAsJSON) failed: %v", err) } case 3: // Form: /volume/ @@ -345,17 +400,44 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeName = pathSplit[2] - jsonToReturn, err = getVolumeAsJSON(volumeName) - if nil == err { - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(jsonToReturn))) + globals.Lock() + + volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) + if nil != err { + logFatal(err) + } + + if ok { + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + } + + volumeGET = &volumeGETStruct{ + Name: volumeAsStruct.name, + StorageURL: volumeAsStruct.storageURL, + HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), + LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), + AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), + } + + globals.Unlock() + + volumeGETAsJSON, err = json.Marshal(volumeGET) + if nil != err { + logFatal(err) + } + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeGETAsJSON))) responseWriter.Header().Set("Content-Type", "application/json") responseWriter.WriteHeader(http.StatusOK) - _, err = responseWriter.Write(jsonToReturn) + _, err = responseWriter.Write(volumeGETAsJSON) if nil != err { - logWarnf("responseWriter.Write(jsonToReturn) failed: %v", err) + logWarnf("responseWriter.Write(volumeGETAsJSON) failed: %v", err) } } else { + globals.Unlock() responseWriter.WriteHeader(http.StatusNotFound) } case 4: @@ -370,45 +452,100 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ globals.stats.GetVolumeLayoutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO + globals.Lock() + + volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) + if nil != err { + logFatal(err) + } + + if ok { + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + } + + toReturnTODO = fmt.Sprintf("TODO: GET /volume/%s/layout", volumeAsStruct.name) + + globals.Unlock() + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(toReturnTODO))) + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(toReturnTODO)) + if nil != err { + logWarnf("responseWriter.Write([]byte(toReturnTODO)) failed: %v", err) + } + } else { + globals.Unlock() + responseWriter.WriteHeader(http.StatusNotFound) + } default: responseWriter.WriteHeader(http.StatusBadRequest) } case 5: - // Form: /volume//inode/ + // Form: /volume//inode/ volumeName = pathSplit[2] mustBeInode = pathSplit[3] - inodeNumberAs16HexDigits = pathSplit[4] + inodeNumberAsHexDigits = pathSplit[4] switch mustBeInode { case "inode": - inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAs16HexDigits, 16, 64) + inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAsHexDigits, 16, 64) if nil != err { responseWriter.WriteHeader(http.StatusBadRequest) } else { - fmt.Println("UNDO: inodeNumberAsUint64:", inodeNumberAsUint64) defer func() { globals.stats.GetVolumeInodeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO + globals.Lock() + + volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) + if nil != err { + logFatal(err) + } + + if ok { + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + } + + toReturnTODO = fmt.Sprintf("TODO: GET /volume/%s/inode/%016X", volumeAsStruct.name, inodeNumberAsUint64) + + globals.Unlock() + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(toReturnTODO))) + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(toReturnTODO)) + if nil != err { + logWarnf("responseWriter.Write([]byte(toReturnTODO)) failed: %v", err) + } + } else { + globals.Unlock() + responseWriter.WriteHeader(http.StatusNotFound) + } } default: responseWriter.WriteHeader(http.StatusBadRequest) } case 6: - // Form: /volume//inode//layout - // Form: /volume//inode//payload + // Form: /volume//inode//layout + // Form: /volume//inode//payload volumeName = pathSplit[2] mustBeInode = pathSplit[3] - inodeNumberAs16HexDigits = pathSplit[4] + inodeNumberAsHexDigits = pathSplit[4] mustBeLayoutOrPayload = pathSplit[5] switch mustBeInode { case "inode": - inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAs16HexDigits, 16, 64) + inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAsHexDigits, 16, 64) if nil != err { responseWriter.WriteHeader(http.StatusBadRequest) } else { @@ -418,13 +555,69 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ globals.stats.GetVolumeInodeLayoutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO + globals.Lock() + + volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) + if nil != err { + logFatal(err) + } + + if ok { + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + } + + toReturnTODO = fmt.Sprintf("TODO: GET /volume/%s/inode/%016X/layout", volumeAsStruct.name, inodeNumberAsUint64) + + globals.Unlock() + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(toReturnTODO))) + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(toReturnTODO)) + if nil != err { + logWarnf("responseWriter.Write([]byte(toReturnTODO)) failed: %v", err) + } + } else { + globals.Unlock() + responseWriter.WriteHeader(http.StatusNotFound) + } case "payload": defer func() { globals.stats.GetVolumeInodePayloadUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO + globals.Lock() + + volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) + if nil != err { + logFatal(err) + } + + if ok { + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + } + + toReturnTODO = fmt.Sprintf("TODO: GET /volume/%s/inode/%016X/payload", volumeAsStruct.name, inodeNumberAsUint64) + + globals.Unlock() + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(toReturnTODO))) + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(toReturnTODO)) + if nil != err { + logWarnf("responseWriter.Write([]byte(toReturnTODO)) failed: %v", err) + } + } else { + globals.Unlock() + responseWriter.WriteHeader(http.StatusNotFound) + } default: responseWriter.WriteHeader(http.StatusBadRequest) } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index cf261740..70d281d2 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -5,7 +5,6 @@ package imgrpkg import ( "container/list" - "encoding/json" "fmt" "io" "strings" @@ -113,111 +112,6 @@ func deleteVolume(volumeName string) (err error) { return } -type volumeGETStruct struct { - Name string - StorageURL string - HealthyMounts uint64 - LeasesExpiredMounts uint64 - AuthTokenExpiredMounts uint64 -} - -func getVolumeAsJSON(volumeName string) (volume []byte, err error) { - var ( - ok bool - volumeAsStruct *volumeStruct - volumeAsValue sortedmap.Value - volumeToReturn *volumeGETStruct - ) - - globals.Lock() - - volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) - if nil != err { - logFatal(err) - } - if !ok { - globals.Unlock() - err = fmt.Errorf("volumeName \"%s\" does not exist", volumeName) - return - } - - volumeAsStruct, ok = volumeAsValue.(*volumeStruct) - if !ok { - logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) - } - - volumeToReturn = &volumeGETStruct{ - Name: volumeAsStruct.name, - StorageURL: volumeAsStruct.storageURL, - HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), - LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), - AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), - } - - globals.Unlock() - - volume, err = json.Marshal(volumeToReturn) - if nil != err { - logFatal(err) - } - - err = nil - return -} - -func getVolumeListAsJSON() (volumeList []byte) { - var ( - err error - ok bool - volumeAsStruct *volumeStruct - volumeAsValue sortedmap.Value - volumeListIndex int - volumeListLen int - volumeListToReturn []*volumeGETStruct - ) - - globals.Lock() - - volumeListLen, err = globals.volumeMap.Len() - if nil != err { - logFatal(err) - } - - volumeListToReturn = make([]*volumeGETStruct, volumeListLen) - - for volumeListIndex = 0; volumeListIndex < volumeListLen; volumeListIndex++ { - _, volumeAsValue, ok, err = globals.volumeMap.GetByIndex(volumeListIndex) - if nil != err { - logFatal(err) - } - if !ok { - logFatalf("globals.volumeMap[] len (%d) is wrong", volumeListLen) - } - - volumeAsStruct, ok = volumeAsValue.(*volumeStruct) - if !ok { - logFatalf("globals.volumeMap[%d] was not a *volumeStruct", volumeListIndex) - } - - volumeListToReturn[volumeListIndex] = &volumeGETStruct{ - Name: volumeAsStruct.name, - StorageURL: volumeAsStruct.storageURL, - HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), - LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), - AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), - } - } - - globals.Unlock() - - volumeList, err = json.Marshal(volumeListToReturn) - if nil != err { - logFatal(err) - } - - return -} - type postVolumeRootDirDirectoryCallbacksStruct struct { io.ReadSeeker sortedmap.BPlusTreeCallbacks From 752cec6f1e70040a77dfd75f5267e1f0a889edb7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 11 Aug 2021 15:36:09 -0700 Subject: [PATCH 054/258] Fixed 1st mount logic - handle AuthToken expiring between fetching lastCheckPoint & superBlock --- imgr/imgrpkg/retry-rpc.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 12c70386..445a0507 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -126,6 +126,18 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo } lastCheckPointAsString = string(lastCheckPointAsByteSlice[:]) + lastCheckPoint, err = ilayout.UnmarshalCheckPointV1(lastCheckPointAsString) + if nil != err { + logFatalf("ilayout.UnmarshalCheckPointV1(lastCheckPointAsString==\"%s\") failed: %v", lastCheckPointAsString, err) + } + + superBlockAsByteSlice, err = swiftObjectGetTail(volume.storageURL, mountRequest.AuthToken, lastCheckPoint.SuperBlockObjectNumber, lastCheckPoint.SuperBlockLength) + if nil != err { + globals.Unlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mountRequest.AuthToken) + return + } + retryGenerateMountID: mountIDAsByteArray = utils.FetchRandomByteSlice(mountIDByteArrayLen) @@ -153,18 +165,8 @@ retryGenerateMountID: globals.mountMap[mountIDAsString] = mount if nil == volume.checkPointControlChan { - lastCheckPoint, err = ilayout.UnmarshalCheckPointV1(lastCheckPointAsString) - if nil != err { - logFatalf("ilayout.UnmarshalCheckPointV1(lastCheckPointAsString==\"%s\") failed: %v", lastCheckPointAsString, err) - } - volume.checkPoint = lastCheckPoint - superBlockAsByteSlice, err = swiftObjectGetTail(volume.storageURL, mountRequest.AuthToken, volume.checkPoint.SuperBlockObjectNumber, volume.checkPoint.SuperBlockLength) - if nil != err { - logFatalf("swiftObjectGetTail(volume.storageURL, mountRequest.AuthToken, volume.checkPoint.SuperBlockObjectNumber, volume.checkPoint.SuperBlockLength) failed: %v", err) - } - volume.superBlock, err = ilayout.UnmarshalSuperBlockV1(superBlockAsByteSlice) if nil != err { logFatalf("ilayout.UnmarshalSuperBlockV1(superBlockAsByteSlice) failed: %v", err) From 789ac02a3f9314b98b824b5b6ed78f47ecca7341 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 11 Aug 2021 16:28:17 -0700 Subject: [PATCH 055/258] Refactored swiftObject{Delete|Get{|Range|Tail}|Put}() in prep for walking multiple mount AuthToken's --- imgr/imgrpkg/swift-client.go | 340 +++++++++++++++++++---------------- 1 file changed, 187 insertions(+), 153 deletions(-) diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index 2a6e1be8..ba13ffec 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -60,10 +60,54 @@ func stopSwiftClient() (err error) { return } +func swiftObjectDeleteOnce(objectURL string, authToken string) (authOK bool, err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + ) + + httpRequest, err = http.NewRequest("DELETE", objectURL, nil) + if nil != err { + return + } + + if "" != authToken { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", objectURL, err) + return + } + + _, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + authOK = true + err = nil + } else if http.StatusUnauthorized == httpResponse.StatusCode { + authOK = false + err = nil + } else { + err = fmt.Errorf("httpResponse.Status: %s", httpResponse.Status) + } + + return +} + func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) (err error) { var ( - httpRequest *http.Request - httpResponse *http.Response + authOK bool nextSwiftRetryDelay time.Duration numSwiftRetries uint32 objectURL string @@ -81,50 +125,78 @@ func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - httpRequest, err = http.NewRequest("DELETE", objectURL, nil) - if nil != err { + authOK, err = swiftObjectDeleteOnce(objectURL, authToken) + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } return } - if "" != authToken { - httpRequest.Header["X-Auth-Token"] = []string{authToken} - } + time.Sleep(nextSwiftRetryDelay) - httpResponse, err = globals.httpClient.Do(httpRequest) - if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) - return - } + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } - _, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) - return - } - err = httpResponse.Body.Close() - if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) - return - } + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return +} - if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { - err = nil - return - } +func (volume *volumeStruct) swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) (authOK bool, err error) { + return true, nil // TODO +} - time.Sleep(nextSwiftRetryDelay) +func swiftObjectGetOnce(objectURL string, authToken string, rangeHeaderValue string) (buf []byte, authOK bool, err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + ) - nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + httpRequest, err = http.NewRequest("GET", objectURL, nil) + if nil != err { + return + } + + if authToken != "" { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + if rangeHeaderValue != "" { + httpRequest.Header["Range"] = []string{rangeHeaderValue} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", objectURL, err) + return + } + + buf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + authOK = true + err = nil + } else if http.StatusUnauthorized == httpResponse.StatusCode { + authOK = false + err = nil + } else { + err = fmt.Errorf("httpResponse.Status: %s", httpResponse.Status) } - err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") return } func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (buf []byte, err error) { var ( - httpRequest *http.Request - httpResponse *http.Response + authOK bool nextSwiftRetryDelay time.Duration numSwiftRetries uint32 objectURL string @@ -142,34 +214,11 @@ func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (b nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - httpRequest, err = http.NewRequest("GET", objectURL, nil) - if nil != err { - return - } - - if authToken != "" { - httpRequest.Header["X-Auth-Token"] = []string{authToken} - } - - httpResponse, err = globals.httpClient.Do(httpRequest) - if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", storageURL, err) - return - } - - buf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) - return - } - err = httpResponse.Body.Close() - if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) - return - } - - if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { - err = nil + buf, authOK, err = swiftObjectGetOnce(objectURL, authToken, "") + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } return } @@ -182,10 +231,13 @@ func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (b return } +func (volume *volumeStruct) swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (buf []byte, authOK bool, err error) { + return nil, true, nil // TODO +} + func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, err error) { var ( - httpRequest *http.Request - httpResponse *http.Response + authOK bool nextSwiftRetryDelay time.Duration numSwiftRetries uint32 objectURL string @@ -206,36 +258,11 @@ func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint6 nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - httpRequest, err = http.NewRequest("GET", objectURL, nil) - if nil != err { - return - } - - httpRequest.Header["Range"] = []string{rangeHeaderValue} - - if authToken != "" { - httpRequest.Header["X-Auth-Token"] = []string{authToken} - } - - httpResponse, err = globals.httpClient.Do(httpRequest) - if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", storageURL, err) - return - } - - buf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) - return - } - err = httpResponse.Body.Close() - if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) - return - } - - if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { - err = nil + buf, authOK, err = swiftObjectGetOnce(objectURL, authToken, rangeHeaderValue) + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } return } @@ -248,10 +275,13 @@ func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint6 return } +func (volume *volumeStruct) swiftObjectGetRange(storageURL string, authToken string, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, authOK bool, err error) { + return nil, true, nil // TODO +} + func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64, objectLength uint64) (buf []byte, err error) { var ( - httpRequest *http.Request - httpResponse *http.Response + authOK bool nextSwiftRetryDelay time.Duration numSwiftRetries uint32 objectURL string @@ -272,52 +302,77 @@ func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64 nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - httpRequest, err = http.NewRequest("GET", objectURL, nil) - if nil != err { + buf, authOK, err = swiftObjectGetOnce(objectURL, authToken, rangeHeaderValue) + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } return } - httpRequest.Header["Range"] = []string{rangeHeaderValue} + time.Sleep(nextSwiftRetryDelay) - if authToken != "" { - httpRequest.Header["X-Auth-Token"] = []string{authToken} - } + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } - httpResponse, err = globals.httpClient.Do(httpRequest) - if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", storageURL, err) - return - } + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return +} - buf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) - return - } - err = httpResponse.Body.Close() - if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) - return - } +func (volume *volumeStruct) swiftObjectGetTail(objectNumber uint64, objectLength uint64) (buf []byte, authOK bool, err error) { + return nil, true, nil // TODO +} - if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { - err = nil - return - } +func swiftObjectPutOnce(objectURL string, authToken string, body io.ReadSeeker) (authOK bool, err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + ) - time.Sleep(nextSwiftRetryDelay) + body.Seek(0, io.SeekStart) - nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + httpRequest, err = http.NewRequest("PUT", objectURL, body) + if nil != err { + return + } + + if authToken != "" { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(PUT %s) failed: %v", objectURL, err) + return + } + + _, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + authOK = true + err = nil + } else if http.StatusUnauthorized == httpResponse.StatusCode { + authOK = false + err = nil + } else { + err = fmt.Errorf("httpResponse.Status: %s", httpResponse.Status) } - err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") return } func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, body io.ReadSeeker) (err error) { var ( - httpRequest *http.Request - httpResponse *http.Response + authOK bool nextSwiftRetryDelay time.Duration numSwiftRetries uint32 objectURL string @@ -335,36 +390,11 @@ func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, bo nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - body.Seek(0, io.SeekStart) - - httpRequest, err = http.NewRequest("PUT", objectURL, body) - if nil != err { - return - } - - if authToken != "" { - httpRequest.Header["X-Auth-Token"] = []string{authToken} - } - - httpResponse, err = globals.httpClient.Do(httpRequest) - if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", storageURL, err) - return - } - - _, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) - return - } - err = httpResponse.Body.Close() - if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) - return - } - - if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { - err = nil + authOK, err = swiftObjectPutOnce(objectURL, authToken, body) + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } return } @@ -376,3 +406,7 @@ func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, bo err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") return } + +func (volume *volumeStruct) swiftObjectPut(objectNumber uint64, body io.ReadSeeker) (authOK bool, err error) { + return true, nil // TODO +} From f72dad6d8b967dee02217b73c591c695b9f09e39 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 11 Aug 2021 18:25:42 -0700 Subject: [PATCH 056/258] Converted non-mount-specific swift calls to retry all healthy mount AuthToken's --- imgr/imgrpkg/log.go | 18 ++ imgr/imgrpkg/swift-client.go | 394 ++++++++++++++++++++++++++++++++--- imgr/imgrpkg/volume.go | 41 +--- 3 files changed, 392 insertions(+), 61 deletions(-) diff --git a/imgr/imgrpkg/log.go b/imgr/imgrpkg/log.go index cae5bc5b..c5ed5169 100644 --- a/imgr/imgrpkg/log.go +++ b/imgr/imgrpkg/log.go @@ -19,18 +19,36 @@ func logFatalf(format string, args ...interface{}) { os.Exit(1) } +func logError(err error) { + logf("ERROR", "%v", err) +} + func logErrorf(format string, args ...interface{}) { logf("ERROR", format, args...) } +func logWarn(err error) { + logf("WARN", "%v", err) +} + func logWarnf(format string, args ...interface{}) { logf("WARN", format, args...) } +func logInfo(err error) { + logf("INFO", "%v", err) +} + func logInfof(format string, args ...interface{}) { logf("INFO", format, args...) } +func logTrace(err error) { + if globals.config.TraceEnabled { + logf("TRACE", "%v", err) + } +} + func logTracef(format string, args ...interface{}) { if globals.config.TraceEnabled { logf("TRACE", format, args...) diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index ba13ffec..453040c5 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -4,6 +4,7 @@ package imgrpkg import ( + "container/list" "fmt" "io" "io/ioutil" @@ -96,8 +97,8 @@ func swiftObjectDeleteOnce(objectURL string, authToken string) (authOK bool, err authOK = true err = nil } else if http.StatusUnauthorized == httpResponse.StatusCode { - authOK = false - err = nil + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded } else { err = fmt.Errorf("httpResponse.Status: %s", httpResponse.Status) } @@ -105,17 +106,65 @@ func swiftObjectDeleteOnce(objectURL string, authToken string) (authOK bool, err return } +func (volume *volumeStruct) swiftObjectDeleteOnce(objectURL string) (authOK bool, err error) { + var ( + mount *mountStruct + mountListElement *list.Element + ok bool + toRetryMountList *list.List + ) + + toRetryMountList = list.New() + + mountListElement = volume.healthyMountList.Front() + + for nil != mountListElement { + _ = volume.healthyMountList.Remove(mountListElement) + + mount, ok = mountListElement.Value.(*mountStruct) + if !ok { + logFatalf("mountListElement.Value.(*mountStruct) returned !ok") + } + + authOK, err = swiftObjectDeleteOnce(objectURL, mount.authToken) + if nil == err { + if authOK { + volume.healthyMountList.PushBackList(toRetryMountList) + mount.listElement = volume.healthyMountList.PushBack(mount) + return + } else { + mount.authTokenExpired = true + mount.listElement = volume.authTokenExpiredMountList.PushBack(mount) + } + } else { + mount.listElement = toRetryMountList.PushBack(mount) + } + + mountListElement = volume.healthyMountList.Front() + } + + if toRetryMountList.Len() == 0 { + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded + } else { + volume.healthyMountList.PushBackList(toRetryMountList) + + authOK = true + err = fmt.Errorf("volume.healthyMountList not empty - retry possible") + } + + return +} + func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) (err error) { var ( authOK bool nextSwiftRetryDelay time.Duration numSwiftRetries uint32 objectURL string - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.SwiftObjectDeleteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -139,11 +188,49 @@ func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) } err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return } -func (volume *volumeStruct) swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) (authOK bool, err error) { - return true, nil // TODO +func (volume *volumeStruct) swiftObjectDelete(objectNumber uint64) (authOK bool, err error) { + var ( + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.SwiftObjectDeleteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = volume.storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + authOK, err = volume.swiftObjectDeleteOnce(objectURL) + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + if volume.healthyMountList.Len() == 0 { + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded + } else { + authOK = true + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + } + + return } func swiftObjectGetOnce(objectURL string, authToken string, rangeHeaderValue string) (buf []byte, authOK bool, err error) { @@ -185,8 +272,8 @@ func swiftObjectGetOnce(objectURL string, authToken string, rangeHeaderValue str authOK = true err = nil } else if http.StatusUnauthorized == httpResponse.StatusCode { - authOK = false - err = nil + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded } else { err = fmt.Errorf("httpResponse.Status: %s", httpResponse.Status) } @@ -194,17 +281,65 @@ func swiftObjectGetOnce(objectURL string, authToken string, rangeHeaderValue str return } +func (volume *volumeStruct) swiftObjectGetOnce(objectURL string, rangeHeaderValue string) (buf []byte, authOK bool, err error) { + var ( + mount *mountStruct + mountListElement *list.Element + ok bool + toRetryMountList *list.List + ) + + toRetryMountList = list.New() + + mountListElement = volume.healthyMountList.Front() + + for nil != mountListElement { + _ = volume.healthyMountList.Remove(mountListElement) + + mount, ok = mountListElement.Value.(*mountStruct) + if !ok { + logFatalf("mountListElement.Value.(*mountStruct) returned !ok") + } + + buf, authOK, err = swiftObjectGetOnce(objectURL, mount.authToken, rangeHeaderValue) + if nil == err { + if authOK { + volume.healthyMountList.PushBackList(toRetryMountList) + mount.listElement = volume.healthyMountList.PushBack(mount) + return + } else { + mount.authTokenExpired = true + mount.listElement = volume.authTokenExpiredMountList.PushBack(mount) + } + } else { + mount.listElement = toRetryMountList.PushBack(mount) + } + + mountListElement = volume.healthyMountList.Front() + } + + if toRetryMountList.Len() == 0 { + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded + } else { + volume.healthyMountList.PushBackList(toRetryMountList) + + authOK = true + err = fmt.Errorf("volume.healthyMountList not empty - retry possible") + } + + return +} + func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (buf []byte, err error) { var ( authOK bool nextSwiftRetryDelay time.Duration numSwiftRetries uint32 objectURL string - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.SwiftObjectGetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -228,11 +363,49 @@ func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (b } err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return } -func (volume *volumeStruct) swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (buf []byte, authOK bool, err error) { - return nil, true, nil // TODO +func (volume *volumeStruct) swiftObjectGet(objectNumber uint64) (buf []byte, authOK bool, err error) { + var ( + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.SwiftObjectGetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = volume.storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + buf, authOK, err = volume.swiftObjectGetOnce(objectURL, "") + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + if volume.healthyMountList.Len() == 0 { + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded + } else { + authOK = true + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + } + + return } func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, err error) { @@ -242,11 +415,9 @@ func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint6 numSwiftRetries uint32 objectURL string rangeHeaderValue string - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.SwiftObjectGetRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -272,11 +443,52 @@ func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint6 } err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return } -func (volume *volumeStruct) swiftObjectGetRange(storageURL string, authToken string, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, authOK bool, err error) { - return nil, true, nil // TODO +func (volume *volumeStruct) swiftObjectGetRange(objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, authOK bool, err error) { + var ( + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + rangeHeaderValue string + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.SwiftObjectGetRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = volume.storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) + + rangeHeaderValue = fmt.Sprintf("bytes=%d-%d", objectOffset, (objectOffset + objectLength - 1)) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + buf, authOK, err = volume.swiftObjectGetOnce(objectURL, rangeHeaderValue) + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + if volume.healthyMountList.Len() == 0 { + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded + } else { + authOK = true + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + } + + return } func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64, objectLength uint64) (buf []byte, err error) { @@ -286,11 +498,9 @@ func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64 numSwiftRetries uint32 objectURL string rangeHeaderValue string - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.SwiftObjectGetTailUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -316,11 +526,52 @@ func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64 } err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return } func (volume *volumeStruct) swiftObjectGetTail(objectNumber uint64, objectLength uint64) (buf []byte, authOK bool, err error) { - return nil, true, nil // TODO + var ( + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + rangeHeaderValue string + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.SwiftObjectGetTailUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = volume.storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) + + rangeHeaderValue = fmt.Sprintf("bytes=-%d", objectLength) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + buf, authOK, err = volume.swiftObjectGetOnce(objectURL, rangeHeaderValue) + if nil == err { + if !authOK { + err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") + } + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + if volume.healthyMountList.Len() == 0 { + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded + } else { + authOK = true + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + } + + return } func swiftObjectPutOnce(objectURL string, authToken string, body io.ReadSeeker) (authOK bool, err error) { @@ -361,8 +612,8 @@ func swiftObjectPutOnce(objectURL string, authToken string, body io.ReadSeeker) authOK = true err = nil } else if http.StatusUnauthorized == httpResponse.StatusCode { - authOK = false - err = nil + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded } else { err = fmt.Errorf("httpResponse.Status: %s", httpResponse.Status) } @@ -370,17 +621,65 @@ func swiftObjectPutOnce(objectURL string, authToken string, body io.ReadSeeker) return } +func (volume *volumeStruct) swiftObjectPutOnce(objectURL string, body io.ReadSeeker) (authOK bool, err error) { + var ( + mount *mountStruct + mountListElement *list.Element + ok bool + toRetryMountList *list.List + ) + + toRetryMountList = list.New() + + mountListElement = volume.healthyMountList.Front() + + for nil != mountListElement { + _ = volume.healthyMountList.Remove(mountListElement) + + mount, ok = mountListElement.Value.(*mountStruct) + if !ok { + logFatalf("mountListElement.Value.(*mountStruct) returned !ok") + } + + authOK, err = swiftObjectPutOnce(objectURL, mount.authToken, body) + if nil == err { + if authOK { + volume.healthyMountList.PushBackList(toRetryMountList) + mount.listElement = volume.healthyMountList.PushBack(mount) + return + } else { + mount.authTokenExpired = true + mount.listElement = volume.authTokenExpiredMountList.PushBack(mount) + } + } else { + mount.listElement = toRetryMountList.PushBack(mount) + } + + mountListElement = volume.healthyMountList.Front() + } + + if toRetryMountList.Len() == 0 { + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded + } else { + volume.healthyMountList.PushBackList(toRetryMountList) + + authOK = true + err = fmt.Errorf("volume.healthyMountList not empty - retry possible") + } + + return +} + func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, body io.ReadSeeker) (err error) { var ( authOK bool nextSwiftRetryDelay time.Duration numSwiftRetries uint32 objectURL string - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.SwiftObjectPutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -404,9 +703,46 @@ func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, bo } err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return } func (volume *volumeStruct) swiftObjectPut(objectNumber uint64, body io.ReadSeeker) (authOK bool, err error) { - return true, nil // TODO + var ( + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.SwiftObjectPutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = volume.storageURL + "/" + ilayout.GetObjectNameAsString(objectNumber) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + authOK, err = volume.swiftObjectPutOnce(objectURL, body) + if nil == err { + if authOK { + return + } + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + if volume.healthyMountList.Len() == 0 { + authOK = false // Auth failed, + err = nil // but we will still indicate the func succeeded + } else { + authOK = true + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + } + + return } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 70d281d2..1d880a85 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -735,42 +735,19 @@ func (volume *volumeStruct) DumpValue(value sortedmap.Value) (valueAsString stri func (volume *volumeStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { var ( - mount *mountStruct - mountListElement *list.Element - ok bool + authOK bool ) -NextHealthyMount: - - mountListElement = volume.healthyMountList.Front() - if nil == mountListElement { - err = fmt.Errorf("no healthy mounts available [case 1]") - return - } - - mount, ok = mountListElement.Value.(*mountStruct) - if !ok { - logFatalf("mountListElement.Value.(*mountStruct) returned !ok [case 1]") - } - - volume.healthyMountList.MoveToBack(mountListElement) - - nodeByteSlice, err = swiftObjectGetRange(volume.storageURL, mount.authToken, objectNumber, objectOffset, objectLength) - if nil == err { - volume.healthyMountList.MoveToBack(mountListElement) - - return // nil err from swiftObjectGetRange() is used + nodeByteSlice, authOK, err = volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + if nil != err { + err = fmt.Errorf("volume.swiftObjectGetRange(objectNumber==0x%016X, objectOffset==0x%016X, objectLength==0x%016X) failed: %v", objectNumber, objectOffset, objectLength, err) + logError(err) + } else if !authOK { + err = fmt.Errorf("volume.swiftObjectGetRange(objectNumber==0x%016X, objectOffset==0x%016X, objectLength==0x%016X) returned !authOK", objectNumber, objectOffset, objectLength) + logWarn(err) } - // Assume that the failure was due to AuthToken expiration - - _ = volume.healthyMountList.Remove(mount.listElement) - - mount.authTokenExpired = true - - mount.listElement = volume.authTokenExpiredMountList.PushBack(mount) - - goto NextHealthyMount + return } func (volume *volumeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { From 0c6025f7e943c5a531f070fa563d35cec39a56c9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 12 Aug 2021 15:21:48 -0700 Subject: [PATCH 057/258] Firmed up Web API for querying volumes --- ilayout/api.go | 9 +- ilayout/api_test.go | 9 +- ilayout/impl.go | 12 +- imgr/imgrpkg/http-server.go | 239 +++++++++++++------------------ imgr/imgrpkg/http-server_test.go | 4 +- imgr/imgrpkg/volume.go | 2 + 6 files changed, 109 insertions(+), 166 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 696297dd..1550552b 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -403,8 +403,8 @@ func UnmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct // // The struct is serialized as a sequence of uint* fields in LittleEndian format. // -// Note that there is no DirectoryEntryKeyV1Struct as it is simply a string serialized -// by a uint64 length in LittleEndian format followed by the bytes of the string. +// Note that there is no DirectoryEntryKeyV1Struct as it is simply a BaseName string +// serialized by a uint64 length in LittleEndian format followed by the bytes of the string. // type DirectoryEntryValueV1Struct struct { InodeNumber uint64 @@ -430,11 +430,10 @@ func UnmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryE // // The struct is serialized as a sequence of uint64 fields in LittleEndian format. // -// Note that there is no ExtentMapEntryKeyV1Struct as it is simply a uint64 serialized -// in LittleEndian format. +// Note that there is no ExtentMapEntryKeyV1Struct as it is simply a FileOffset uint64 +// serialized in LittleEndian format. // type ExtentMapEntryValueV1Struct struct { - FileOffset uint64 // Offset from the start of the File Length uint64 // Length of this extent (both in the File and in the Object) ObjectNumber uint64 // Identifies the Object containing this extent's data ObjectOffset uint64 // Starting offset in the Object of this extent's data diff --git a/ilayout/api_test.go b/ilayout/api_test.go index b8c30c33..e99f25c4 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -143,16 +143,15 @@ func TestAPI(t *testing.T) { unmarshaledDirectoryEntryValueV1BytesConsumedExpected = int(8 + 1) testExtentMapEntryValueV1 = &ExtentMapEntryValueV1Struct{ - FileOffset: 1, - Length: 2, - ObjectNumber: 3, - ObjectOffset: 4, + Length: 1, + ObjectNumber: 2, + ObjectOffset: 3, } marshaledExtentMapEntryValueV1 []byte unmarshaledExtentMapEntryValueV1 *ExtentMapEntryValueV1Struct unmarshaledExtentMapEntryValueV1BytesConsumed int - unmarshaledExtentMapEntryValueV1BytesConsumedExpected = int(8 + 8 + 8 + 8) + unmarshaledExtentMapEntryValueV1BytesConsumedExpected = int(8 + 8 + 8) testObjectNumber = uint64(0x0123456789ABCDEF) diff --git a/ilayout/impl.go b/ilayout/impl.go index f624a80e..722404b7 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -809,15 +809,10 @@ func (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct) marshalExtentMapEntryV curPos int ) - extentMapEntryValueV1Buf = make([]byte, 8+8+8+8) + extentMapEntryValueV1Buf = make([]byte, 8+8+8) curPos = 0 - curPos, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.FileOffset) - if nil != err { - return - } - curPos, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.Length) if nil != err { return @@ -851,11 +846,6 @@ func unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapE extentMapEntryValueV1 = &ExtentMapEntryValueV1Struct{} - extentMapEntryValueV1.FileOffset, curPos, err = getLEUint64FromBuf(extentMapEntryValueV1Buf, curPos) - if nil != err { - return - } - extentMapEntryValueV1.Length, curPos, err = getLEUint64FromBuf(extentMapEntryValueV1Buf, curPos) if nil != err { return diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index ee947cd3..7c67e723 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -301,7 +301,7 @@ func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Req } } -type volumeGETStruct struct { +type volumeMiniGETStruct struct { Name string StorageURL string HealthyMounts uint64 @@ -309,23 +309,111 @@ type volumeGETStruct struct { AuthTokenExpiredMounts uint64 } +type volumeFullGETInodeTableLayoutEntryStruct struct { + ObjectNumber string + ObjectSize string + BytesReferenced string +} + +type volumeFullGETStruct struct { + Name string + StorageURL string + HealthyMounts uint64 + LeasesExpiredMounts uint64 + AuthTokenExpiredMounts uint64 + InodeTableLayout []volumeFullGETInodeTableLayoutEntryStruct + InodeObjectCount string + InodeObjectSize string + InodeBytesReferenced string + PendingDeleteObjectNumberArray []string +} + +type inodeGETLinkTableEntryStruct struct { + ParentDirInodeNumber string + ParentDirEntryName string +} + +type inodeGETStreamTableEntryStruct struct { + Name string + Value string // "XX XX .. XX" Hex-displayed bytes +} + +type dirInodeGETPayloadEntryStruct struct { + BaseName string + InodeNumber string + InodeType string // One of "InodeTypeDir", "InodeTypeFile", or "InodeTypeSymLink" +} + +type fileInodeGETPayloadEntryStruct struct { + FileOffset string + Length string + ObjectNumber string + ObjectOffset string +} + +type inodeGETLayoutEntryStruct struct { + ObjectNumber string + ObjectSize string + BytesReferenced string +} + +type dirInodeGETStruct struct { + InodeNumber string + InodeType string // == "InodeTypeDir" + LinkTable []inodeGETLinkTableEntryStruct + ModificationTime string + StatusChangeTime string + Mode string + UserID string + GroupID string + StreamTable []inodeGETStreamTableEntryStruct + Payload []dirInodeGETPayloadEntryStruct + Layout []inodeGETLayoutEntryStruct +} + +type fileInodeGETStruct struct { + InodeNumber string + InodeType string // == "InodeTypeFile" + LinkTable []inodeGETLinkTableEntryStruct + Size string + ModificationTime string + StatusChangeTime string + Mode string + UserID string + GroupID string + StreamTable []inodeGETStreamTableEntryStruct + Payload []fileInodeGETPayloadEntryStruct + Layout []inodeGETLayoutEntryStruct +} + +type symLinkInodeGETStruct struct { + InodeNumber string + InodeType string // == "InodeTypeSymLink" + LinkTable []inodeGETLinkTableEntryStruct + ModificationTime string + StatusChangeTime string + Mode string + UserID string + GroupID string + StreamTable []inodeGETStreamTableEntryStruct + SymLinkTarget string +} + func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( err error inodeNumberAsHexDigits string inodeNumberAsUint64 uint64 mustBeInode string - mustBeLayout string - mustBeLayoutOrPayload string ok bool pathSplit []string startTime time.Time toReturnTODO string volumeAsStruct *volumeStruct volumeAsValue sortedmap.Value - volumeGET *volumeGETStruct + volumeGET *volumeFullGETStruct volumeGETAsJSON []byte - volumeGETList []*volumeGETStruct + volumeGETList []*volumeMiniGETStruct volumeGETListIndex int volumeGETListAsJSON []byte volumeGETListLen int @@ -351,7 +439,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatal(err) } - volumeGETList = make([]*volumeGETStruct, volumeGETListLen) + volumeGETList = make([]*volumeMiniGETStruct, volumeGETListLen) for volumeGETListIndex = 0; volumeGETListIndex < volumeGETListLen; volumeGETListIndex++ { _, volumeAsValue, ok, err = globals.volumeMap.GetByIndex(volumeGETListIndex) @@ -367,7 +455,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("globals.volumeMap[%d] was not a *volumeStruct", volumeGETListIndex) } - volumeGETList[volumeGETListIndex] = &volumeGETStruct{ + volumeGETList[volumeGETListIndex] = &volumeMiniGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), @@ -413,7 +501,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } - volumeGET = &volumeGETStruct{ + volumeGET = &volumeFullGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), @@ -440,50 +528,6 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ globals.Unlock() responseWriter.WriteHeader(http.StatusNotFound) } - case 4: - // Form: /volume//layout - - volumeName = pathSplit[2] - mustBeLayout = pathSplit[3] - - switch mustBeLayout { - case "layout": - defer func() { - globals.stats.GetVolumeLayoutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - globals.Lock() - - volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) - if nil != err { - logFatal(err) - } - - if ok { - volumeAsStruct, ok = volumeAsValue.(*volumeStruct) - if !ok { - logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) - } - - toReturnTODO = fmt.Sprintf("TODO: GET /volume/%s/layout", volumeAsStruct.name) - - globals.Unlock() - - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(toReturnTODO))) - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - _, err = responseWriter.Write([]byte(toReturnTODO)) - if nil != err { - logWarnf("responseWriter.Write([]byte(toReturnTODO)) failed: %v", err) - } - } else { - globals.Unlock() - responseWriter.WriteHeader(http.StatusNotFound) - } - default: - responseWriter.WriteHeader(http.StatusBadRequest) - } case 5: // Form: /volume//inode/ @@ -534,97 +578,6 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ default: responseWriter.WriteHeader(http.StatusBadRequest) } - case 6: - // Form: /volume//inode//layout - // Form: /volume//inode//payload - - volumeName = pathSplit[2] - mustBeInode = pathSplit[3] - inodeNumberAsHexDigits = pathSplit[4] - mustBeLayoutOrPayload = pathSplit[5] - - switch mustBeInode { - case "inode": - inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAsHexDigits, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - } else { - switch mustBeLayoutOrPayload { - case "layout": - defer func() { - globals.stats.GetVolumeInodeLayoutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - globals.Lock() - - volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) - if nil != err { - logFatal(err) - } - - if ok { - volumeAsStruct, ok = volumeAsValue.(*volumeStruct) - if !ok { - logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) - } - - toReturnTODO = fmt.Sprintf("TODO: GET /volume/%s/inode/%016X/layout", volumeAsStruct.name, inodeNumberAsUint64) - - globals.Unlock() - - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(toReturnTODO))) - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - _, err = responseWriter.Write([]byte(toReturnTODO)) - if nil != err { - logWarnf("responseWriter.Write([]byte(toReturnTODO)) failed: %v", err) - } - } else { - globals.Unlock() - responseWriter.WriteHeader(http.StatusNotFound) - } - case "payload": - defer func() { - globals.stats.GetVolumeInodePayloadUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - globals.Lock() - - volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) - if nil != err { - logFatal(err) - } - - if ok { - volumeAsStruct, ok = volumeAsValue.(*volumeStruct) - if !ok { - logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) - } - - toReturnTODO = fmt.Sprintf("TODO: GET /volume/%s/inode/%016X/payload", volumeAsStruct.name, inodeNumberAsUint64) - - globals.Unlock() - - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(toReturnTODO))) - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - _, err = responseWriter.Write([]byte(toReturnTODO)) - if nil != err { - logWarnf("responseWriter.Write([]byte(toReturnTODO)) failed: %v", err) - } - } else { - globals.Unlock() - responseWriter.WriteHeader(http.StatusNotFound) - } - default: - responseWriter.WriteHeader(http.StatusBadRequest) - } - } - default: - responseWriter.WriteHeader(http.StatusBadRequest) - } default: responseWriter.WriteHeader(http.StatusBadRequest) } diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index 8dea252e..b858c870 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -76,7 +76,7 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) failed: %v", err) } - responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0}", testVolume, testGlobals.containerURL) + responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"InodeTableLayout\":null,\"InodeObjectCount\":\"\",\"InodeObjectSize\":\"\",\"InodeBytesReferenced\":\"\",\"PendingDeleteObjectNumberArray\":null}", testVolume, testGlobals.containerURL) _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) if nil != err { @@ -86,7 +86,7 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("GET /volume/%s returned unexpected responseBody: \"%s\"", testVolume, responseBody) } - responseBodyExpected = "[" + responseBodyExpected + "]" + responseBodyExpected = fmt.Sprintf("[{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0}]", testVolume, testGlobals.containerURL) _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) if nil != err { diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 1d880a85..ce2d2afb 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -131,6 +131,7 @@ func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksSt } return } + func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) Seek(offset int64, whence int) (int64, error) { var ( newOffset int64 @@ -281,6 +282,7 @@ func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCal } return } + func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) Seek(offset int64, whence int) (int64, error) { var ( newOffset int64 From 81c9fc61289d7fb78325b22fce0fa87ae79286de Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 13 Aug 2021 12:02:16 -0700 Subject: [PATCH 058/258] Updated imgrpkg API to allow optional AuthToken on PUT /volume/ This can be used by imgrpkg to access Swift Container w/out a "healthy" iclient mount --- iclient/iclient.sh | 29 ++++++++++++++- imgr/imgrpkg/api.go | 22 +++++++++++- imgr/imgrpkg/globals.go | 1 + imgr/imgrpkg/http-server.go | 7 +++- imgr/imgrpkg/http-server_test.go | 61 ++++++++++++++++++++++++++++---- imgr/imgrpkg/swift-client.go | 52 +++++++++++++++++++++------ imgr/imgrpkg/volume.go | 3 +- imgr/mkmount.sh | 29 ++++++++++++++- 8 files changed, 182 insertions(+), 22 deletions(-) diff --git a/iclient/iclient.sh b/iclient/iclient.sh index 7753a64b..f4d11047 100755 --- a/iclient/iclient.sh +++ b/iclient/iclient.sh @@ -1,5 +1,27 @@ #!/bin/sh +Usage="$(basename "$0") - Ask swift/imgr to format and serve testvol... and then sleep +where: + -h show this help text + -s supply static AuthToken to imgr" + +StaticAuthToken=false + +while getopts 'hs' option +do + echo "Handling option: $option" + case "$option" in + h) echo "$Usage" + exit 0 + ;; + s) StaticAuthToken=true + ;; + ?) echo "$Usage" + exit 1 + ;; + esac +done + AuthToken="" while [ "$AuthToken" == "" ] @@ -21,6 +43,11 @@ done curl -v -s -f imgr:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" -curl -v -s -f imgr:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\"}" +if $StaticAuthToken +then + curl -v -s -f imgr:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" +else + curl -v -s -f imgr:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\"}" +fi sleep 100000 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 690f6161..55de74c1 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -102,7 +102,10 @@ // "AuthToken" : "AUTH_tk0123456789abcde0123456789abcdef0" // } // -// This will cause the specified StorageURL to be formatted. +// This will cause the specified StorageURL to be formatted. The StorageURL +// specified in the JSON document content identifies the Container for format. +// The AuthToken in the JSON document content provides the authentication to +// use during the formatting process. // // PUT /volume/ // Content-Type: application/json @@ -113,6 +116,23 @@ // // This will cause the specified to be served. The StorageURL // specified in the JSON document content identifies the Container to serve. +// Clients will each supply an AuthToken in their Mount/RenewMount requests +// that will be used to access the Container. +// +// PUT /volume/ +// Content-Type: application/json +// +// { +// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con", +// "AuthToken" : "AUTH_tk0123456789abcde0123456789abcdef0" +// } +// +// This will cause the specified to be served. The StorageURL +// specified in the JSON document content identifies the Container to serve. +// Clients will each supply an AuthToken in their Mount/RenewMount requests +// that will be used to access the Container. As a debugging aid, and in the +// case where no Clients have mounted, the AuthToken in the JSON +// document content will be used to access the Container. // package imgrpkg diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 70dbb111..96234847 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -227,6 +227,7 @@ type inodeTableLayoutElementStruct struct { type volumeStruct struct { name string // storageURL string // + authToken string // if != "" & healthyMountList is empty, this AuthToken will be used; cleared on auth failure mountMap map[string]*mountStruct // key == mountStruct.mountID healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false leasesExpiredMountList *list.List // list of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 7c67e723..8bbdf0d1 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -304,6 +304,7 @@ func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Req type volumeMiniGETStruct struct { Name string StorageURL string + AuthToken string HealthyMounts uint64 LeasesExpiredMounts uint64 AuthTokenExpiredMounts uint64 @@ -318,6 +319,7 @@ type volumeFullGETInodeTableLayoutEntryStruct struct { type volumeFullGETStruct struct { Name string StorageURL string + AuthToken string HealthyMounts uint64 LeasesExpiredMounts uint64 AuthTokenExpiredMounts uint64 @@ -458,6 +460,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeGETList[volumeGETListIndex] = &volumeMiniGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, + AuthToken: volumeAsStruct.authToken, HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), @@ -504,6 +507,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeGET = &volumeFullGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, + AuthToken: volumeAsStruct.authToken, HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), @@ -635,6 +639,7 @@ func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request, req type serveHTTPPutOfVolumeRequestBodyAsJSONStruct struct { StorageURL string + AuthToken string } func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { @@ -661,7 +666,7 @@ func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Requ return } - err = putVolume(pathSplit[2], requestBodyAsJSON.StorageURL) + err = putVolume(pathSplit[2], requestBodyAsJSON.StorageURL, requestBodyAsJSON.AuthToken) if nil == err { responseWriter.WriteHeader(http.StatusCreated) } else { diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index b858c870..6937c71f 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/NVIDIA/proxyfs/ilayout" + "github.com/NVIDIA/proxyfs/version" ) func TestHTTPServer(t *testing.T) { @@ -34,6 +35,14 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("GET /stats failed: %v", err) } + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/version", nil, nil) + if nil != err { + t.Fatalf("GET /version failed: %v", err) + } + if string(responseBody[:]) != version.ProxyFSVersion { + t.Fatalf("GET /version should have returned \"%s\" - it returned \"%s\"", version.ProxyFSVersion, string(responseBody[:])) + } + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) if nil != err { t.Fatalf("GET /volume [case 1] failed: %v", err) @@ -73,20 +82,20 @@ func TestHTTPServer(t *testing.T) { _, _, err = testDoHTTPRequest("PUT", testGlobals.httpServerURL+"/volume/"+testVolume, nil, strings.NewReader(putRequestBody)) if nil != err { - t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) failed: %v", err) + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) [case 1] failed: %v", err) } - responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"InodeTableLayout\":null,\"InodeObjectCount\":\"\",\"InodeObjectSize\":\"\",\"InodeBytesReferenced\":\"\",\"PendingDeleteObjectNumberArray\":null}", testVolume, testGlobals.containerURL) + responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"InodeTableLayout\":null,\"InodeObjectCount\":\"\",\"InodeObjectSize\":\"\",\"InodeBytesReferenced\":\"\",\"PendingDeleteObjectNumberArray\":null}", testVolume, testGlobals.containerURL) _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) if nil != err { - t.Fatalf("GET /volume/%s failed: %v", testVolume, err) + t.Fatalf("GET /volume/%s [case 1] failed: %v", testVolume, err) } if string(responseBody[:]) != responseBodyExpected { - t.Fatalf("GET /volume/%s returned unexpected responseBody: \"%s\"", testVolume, responseBody) + t.Fatalf("GET /volume/%s [case 1] returned unexpected responseBody: \"%s\"", testVolume, responseBody) } - responseBodyExpected = fmt.Sprintf("[{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0}]", testVolume, testGlobals.containerURL) + responseBodyExpected = fmt.Sprintf("[{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0}]", testVolume, testGlobals.containerURL) _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) if nil != err { @@ -98,7 +107,7 @@ func TestHTTPServer(t *testing.T) { _, _, err = testDoHTTPRequest("DELETE", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) if nil != err { - t.Fatalf("testDoHTTPRequest(\"DELETE\", testGlobals.httpServerURL+\"/volume/\"+testVolume, nil, nil) failed: %v", err) + t.Fatalf("testDoHTTPRequest(\"DELETE\", testGlobals.httpServerURL+\"/volume/\"+testVolume, nil, nil) [case 1] failed: %v", err) } _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) @@ -109,5 +118,45 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("GET /volume [case 3] should have returned \"[]\" - it returned \"%s\"", string(responseBody[:])) } + putRequestBody = fmt.Sprintf("{\"StorageURL\":\"%s\",\"AuthToken\":\"%s\"}", testGlobals.containerURL, testGlobals.authToken) + + _, _, err = testDoHTTPRequest("PUT", testGlobals.httpServerURL+"/volume/"+testVolume, nil, strings.NewReader(putRequestBody)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) [case 2] failed: %v", err) + } + + responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"InodeTableLayout\":null,\"InodeObjectCount\":\"\",\"InodeObjectSize\":\"\",\"InodeBytesReferenced\":\"\",\"PendingDeleteObjectNumberArray\":null}", testVolume, testGlobals.containerURL, testGlobals.authToken) + + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) + if nil != err { + t.Fatalf("GET /volume/%s [case 2] failed: %v", testVolume, err) + } + if string(responseBody[:]) != responseBodyExpected { + t.Fatalf("GET /volume/%s [case 2] returned unexpected responseBody: \"%s\"", testVolume, responseBody) + } + + responseBodyExpected = fmt.Sprintf("[{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0}]", testVolume, testGlobals.containerURL, testGlobals.authToken) + + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) + if nil != err { + t.Fatalf("GET /volume [case 4] failed: %v", err) + } + if string(responseBody[:]) != responseBodyExpected { + t.Fatalf("GET /volume [case 4] returned unexpected responseBody: \"%s\"", responseBody) + } + + _, _, err = testDoHTTPRequest("DELETE", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"DELETE\", testGlobals.httpServerURL+\"/volume/\"+testVolume, nil, nil) [case 2] failed: %v", err) + } + + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) + if nil != err { + t.Fatalf("GET /volume [case 5] failed: %v", err) + } + if string(responseBody[:]) != "[]" { + t.Fatalf("GET /volume [case 5] should have returned \"[]\" - it returned \"%s\"", string(responseBody[:])) + } + testTeardown(t) } diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index 453040c5..7d5bf567 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -143,14 +143,24 @@ func (volume *volumeStruct) swiftObjectDeleteOnce(objectURL string) (authOK bool mountListElement = volume.healthyMountList.Front() } - if toRetryMountList.Len() == 0 { + if (toRetryMountList.Len() == 0) && (volume.authToken == "") { authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { volume.healthyMountList.PushBackList(toRetryMountList) + if volume.authToken != "" { + authOK, err = swiftObjectDeleteOnce(objectURL, volume.authToken) + if (nil == err) && !authOK { + logWarnf("swiftObjectDeleteOnce(,volume.authToken) !authOK for volume %s...clearing volume.authToken", volume.name) + volume.authToken = "" + } + + return + } + authOK = true - err = fmt.Errorf("volume.healthyMountList not empty - retry possible") + err = fmt.Errorf("authToken list not empty - retry possible") } return @@ -222,7 +232,7 @@ func (volume *volumeStruct) swiftObjectDelete(objectNumber uint64) (authOK bool, nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) } - if volume.healthyMountList.Len() == 0 { + if (volume.healthyMountList.Len() == 0) && (volume.authToken == "") { authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { @@ -318,14 +328,24 @@ func (volume *volumeStruct) swiftObjectGetOnce(objectURL string, rangeHeaderValu mountListElement = volume.healthyMountList.Front() } - if toRetryMountList.Len() == 0 { + if (toRetryMountList.Len() == 0) && (volume.authToken == "") { authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { volume.healthyMountList.PushBackList(toRetryMountList) + if volume.authToken != "" { + buf, authOK, err = swiftObjectGetOnce(objectURL, volume.authToken, rangeHeaderValue) + if (nil == err) && !authOK { + logWarnf("swiftObjectGetOnce(,volume.authToken,) !authOK for volume %s...clearing volume.authToken", volume.name) + volume.authToken = "" + } + + return + } + authOK = true - err = fmt.Errorf("volume.healthyMountList not empty - retry possible") + err = fmt.Errorf("authToken list not empty - retry possible") } return @@ -397,7 +417,7 @@ func (volume *volumeStruct) swiftObjectGet(objectNumber uint64) (buf []byte, aut nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) } - if volume.healthyMountList.Len() == 0 { + if (volume.healthyMountList.Len() == 0) && (volume.authToken == "") { authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { @@ -480,7 +500,7 @@ func (volume *volumeStruct) swiftObjectGetRange(objectNumber uint64, objectOffse nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) } - if volume.healthyMountList.Len() == 0 { + if (volume.healthyMountList.Len() == 0) && (volume.authToken == "") { authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { @@ -563,7 +583,7 @@ func (volume *volumeStruct) swiftObjectGetTail(objectNumber uint64, objectLength nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) } - if volume.healthyMountList.Len() == 0 { + if (volume.healthyMountList.Len() == 0) && (volume.authToken == "") { authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { @@ -658,14 +678,24 @@ func (volume *volumeStruct) swiftObjectPutOnce(objectURL string, body io.ReadSee mountListElement = volume.healthyMountList.Front() } - if toRetryMountList.Len() == 0 { + if (toRetryMountList.Len() == 0) && (volume.authToken == "") { authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { volume.healthyMountList.PushBackList(toRetryMountList) + if volume.authToken != "" { + authOK, err = swiftObjectPutOnce(objectURL, volume.authToken, body) + if (nil == err) && !authOK { + logWarnf("swiftObjectPutOnce(,volume.authToken,) !authOK for volume %s...clearing volume.authToken", volume.name) + volume.authToken = "" + } + + return + } + authOK = true - err = fmt.Errorf("volume.healthyMountList not empty - retry possible") + err = fmt.Errorf("authToken list not empty - retry possible") } return @@ -736,7 +766,7 @@ func (volume *volumeStruct) swiftObjectPut(objectNumber uint64, body io.ReadSeek nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) } - if volume.healthyMountList.Len() == 0 { + if (volume.healthyMountList.Len() == 0) && (volume.authToken == "") { authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index ce2d2afb..b7a0a88c 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -603,7 +603,7 @@ func postVolume(storageURL string, authToken string) (err error) { return } -func putVolume(name string, storageURL string) (err error) { +func putVolume(name string, storageURL string, authToken string) (err error) { var ( ok bool volume *volumeStruct @@ -612,6 +612,7 @@ func putVolume(name string, storageURL string) (err error) { volume = &volumeStruct{ name: name, storageURL: storageURL, + authToken: authToken, mountMap: make(map[string]*mountStruct), healthyMountList: list.New(), leasesExpiredMountList: list.New(), diff --git a/imgr/mkmount.sh b/imgr/mkmount.sh index 36d1a1e1..10d741d0 100755 --- a/imgr/mkmount.sh +++ b/imgr/mkmount.sh @@ -1,5 +1,27 @@ #!/bin/sh +Usage="$(basename "$0") - Ask swift/imgr to format and serve testvol +where: + -h show this help text + -s supply static AuthToken to imgr" + +StaticAuthToken=false + +while getopts 'hs' option +do + echo "Handling option: $option" + case "$option" in + h) echo "$Usage" + exit 0 + ;; + s) StaticAuthToken=true + ;; + ?) echo "$Usage" + exit 1 + ;; + esac +done + AuthToken="" while [ "$AuthToken" == "" ] @@ -25,4 +47,9 @@ done curl -s -f dev:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" > /dev/null -curl -s -f dev:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\"}" > /dev/null +if $StaticAuthToken +then + curl -s -f dev:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" > /dev/null +else + curl -s -f dev:15346/volume/testvol -X PUT -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\"}" > /dev/null +fi From 59881ebde1ad331a88ce6ea944bf3e0f1714c13e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 13 Aug 2021 12:33:48 -0700 Subject: [PATCH 059/258] Updated auto format/mount to tell imgr PUT /volume the AuthToken used (debug purposes) --- docker-compose.yml | 2 +- iclient/iclient.sh | 1 - imgr/mkmount.sh | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4e6560d5..204f4acc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,5 +116,5 @@ services: depends_on: - swift - imgr - command: ["./iclient.sh"] + command: ["./iclient.sh", "-s"] \ No newline at end of file diff --git a/iclient/iclient.sh b/iclient/iclient.sh index f4d11047..ab5de1de 100755 --- a/iclient/iclient.sh +++ b/iclient/iclient.sh @@ -9,7 +9,6 @@ StaticAuthToken=false while getopts 'hs' option do - echo "Handling option: $option" case "$option" in h) echo "$Usage" exit 0 diff --git a/imgr/mkmount.sh b/imgr/mkmount.sh index 10d741d0..7c5cde45 100755 --- a/imgr/mkmount.sh +++ b/imgr/mkmount.sh @@ -9,7 +9,6 @@ StaticAuthToken=false while getopts 'hs' option do - echo "Handling option: $option" case "$option" in h) echo "$Usage" exit 0 From 0c53751d4f5b15d2d2b5a45f4f84c2ed3d3bdf04 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 13 Aug 2021 19:02:18 -0700 Subject: [PATCH 060/258] Finished server side of GET /volume/ ...and found/fixed a bug in the "mkfs" (POST /volume) as it put the RootDirInode into the InodeTable with the wrong ObjectNumber... --- ilayout/api.go | 5 +- imgr/imgrpkg/http-server.go | 301 +++++++++++++++++++++++++------ imgr/imgrpkg/http-server_test.go | 13 +- imgr/imgrpkg/volume.go | 65 ++++++- 4 files changed, 313 insertions(+), 71 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 1550552b..f9668ba0 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -256,7 +256,8 @@ func UnmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf []byte) (inode // // The struct is serialized as a sequence of LittleEndian formatted fields. // -// Note that there is no InodeTableEntryKeyV1Struct as it is simply a uint64 InodeNumber. +// Note that there is no InodeTableEntryKeyV1Struct as it is simply an InodeNumber uint64 +// serialized in LittleEndian format. // type InodeTableEntryValueV1Struct struct { InodeHeadObjectNumber uint64 // Identifies the Object containing InodeHeadV*Struct @@ -361,7 +362,7 @@ type InodeHeadLayoutEntryV1Struct struct { // specified in the table entry struct. // For time.Time fields, a uint64 in LittleEndian is used to hold the UnixNano() equivalent. // -// Note that the InodeTableEntryValueV1Struct.InodeHeadLength also includes the bytes for +// Note that the SuperBlockV1Struct.InodeTableRootObjectLength also includes the bytes for // holding the ObjectTrailerStruct{ObjType: InodeHeadType, Version: InodeHeadVersionV1} // that is appended. // diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 8bbdf0d1..1dd65af7 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -16,6 +16,7 @@ import ( "time" "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/version" "github.com/NVIDIA/sortedmap" ) @@ -23,6 +24,9 @@ import ( const ( startHTTPServerUpCheckDelay = 100 * time.Millisecond startHTTPServerUpCheckMaxRetries = 10 + + volumeGETInodeTableCacheLowLimit = 1000 + volumeGETInodeTableCacheHighLimit = 1010 ) func startHTTPServer() (err error) { @@ -311,28 +315,39 @@ type volumeMiniGETStruct struct { } type volumeFullGETInodeTableLayoutEntryStruct struct { - ObjectNumber string - ObjectSize string - BytesReferenced string + ObjectName string // == ilayout.GetObjectNameAsString(ObjectNumber) + ObjectSize uint64 + BytesReferenced uint64 +} + +type volumeFullGETInodeTableEntryStruct struct { + InodeNumber uint64 + InodeHeadObjectName string // == ilayout.GetObjectNameAsString(InodeHeadObjectNumber) + InodeHeadLength uint64 } type volumeFullGETStruct struct { - Name string - StorageURL string - AuthToken string - HealthyMounts uint64 - LeasesExpiredMounts uint64 - AuthTokenExpiredMounts uint64 - InodeTableLayout []volumeFullGETInodeTableLayoutEntryStruct - InodeObjectCount string - InodeObjectSize string - InodeBytesReferenced string - PendingDeleteObjectNumberArray []string + volume *volumeStruct // will not be encoded by json.Marshal() + Name string + StorageURL string + AuthToken string + HealthyMounts uint64 + LeasesExpiredMounts uint64 + AuthTokenExpiredMounts uint64 + SuperBlockObjectName string // == ilayout.GetObjectNameAsString(SuperBlockObjectNumber) + SuperBlockLength uint64 + ReservedToNonce uint64 + InodeTableLayout []volumeFullGETInodeTableLayoutEntryStruct + InodeObjectCount uint64 + InodeObjectSize uint64 + InodeBytesReferenced uint64 + PendingDeleteObjectNameArray []string // == []ilayout.GetObjectNameAsString(PendingDeleteObjectNumber) + InodeTable []volumeFullGETInodeTableEntryStruct } type inodeGETLinkTableEntryStruct struct { - ParentDirInodeNumber string - ParentDirEntryName string + ParentDirInodeNumber uint64 + ParentDirEntryName uint64 } type inodeGETStreamTableEntryStruct struct { @@ -342,84 +357,174 @@ type inodeGETStreamTableEntryStruct struct { type dirInodeGETPayloadEntryStruct struct { BaseName string - InodeNumber string + InodeNumber uint64 InodeType string // One of "InodeTypeDir", "InodeTypeFile", or "InodeTypeSymLink" } type fileInodeGETPayloadEntryStruct struct { - FileOffset string - Length string - ObjectNumber string - ObjectOffset string + FileOffset uint64 + Length uint64 + ObjectName string // == ilayout.GetObjectNameAsString(ObjectNumber) + ObjectOffset uint64 } type inodeGETLayoutEntryStruct struct { - ObjectNumber string - ObjectSize string - BytesReferenced string + ObjectName string // == ilayout.GetObjectNameAsString(ObjectNumber) + ObjectSize uint64 + BytesReferenced uint64 } type dirInodeGETStruct struct { - InodeNumber string + InodeNumber uint64 InodeType string // == "InodeTypeDir" LinkTable []inodeGETLinkTableEntryStruct ModificationTime string StatusChangeTime string - Mode string - UserID string - GroupID string + Mode uint16 + UserID uint64 + GroupID uint64 StreamTable []inodeGETStreamTableEntryStruct Payload []dirInodeGETPayloadEntryStruct Layout []inodeGETLayoutEntryStruct } type fileInodeGETStruct struct { - InodeNumber string + InodeNumber uint64 InodeType string // == "InodeTypeFile" LinkTable []inodeGETLinkTableEntryStruct Size string ModificationTime string StatusChangeTime string - Mode string - UserID string - GroupID string + Mode uint16 + UserID uint64 + GroupID uint64 StreamTable []inodeGETStreamTableEntryStruct Payload []fileInodeGETPayloadEntryStruct Layout []inodeGETLayoutEntryStruct } type symLinkInodeGETStruct struct { - InodeNumber string + InodeNumber uint64 InodeType string // == "InodeTypeSymLink" LinkTable []inodeGETLinkTableEntryStruct ModificationTime string StatusChangeTime string - Mode string - UserID string - GroupID string + Mode uint16 + UserID uint64 + GroupID uint64 StreamTable []inodeGETStreamTableEntryStruct SymLinkTarget string } +func (volumeGET *volumeFullGETStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (volumeGET *volumeFullGETStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (volumeGET *volumeFullGETStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { + var ( + authOK bool + ) + + nodeByteSlice, authOK, err = volumeGET.volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + if nil != err { + err = fmt.Errorf("volume.swiftObjectGetRange() failed: %v", err) + return + } + if !authOK { + err = fmt.Errorf("volume.swiftObjectGetRange() returned !authOK") + return + } + + return +} + +func (volumeGET *volumeFullGETStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (volumeGET *volumeFullGETStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + err = fmt.Errorf("not implemented") + return +} + +func (volumeGET *volumeFullGETStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (volumeGET *volumeFullGETStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + var ( + nextPos int + inodeNumber uint64 + ) + + inodeNumber, nextPos, err = ilayout.GetLEUint64FromBuf(payloadData, 0) + if nil == err { + key = inodeNumber + bytesConsumed = uint64(nextPos) + } + + return +} + +func (volumeGET *volumeFullGETStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (volumeGET *volumeFullGETStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + var ( + bytesConsumedAsInt int + inodeTableEntryValueV1 *ilayout.InodeTableEntryValueV1Struct + ) + + inodeTableEntryValueV1, bytesConsumedAsInt, err = ilayout.UnmarshalInodeTableEntryValueV1(payloadData) + if nil == err { + value = inodeTableEntryValueV1 + bytesConsumed = uint64(bytesConsumedAsInt) + } + + return +} + func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( - err error - inodeNumberAsHexDigits string - inodeNumberAsUint64 uint64 - mustBeInode string - ok bool - pathSplit []string - startTime time.Time - toReturnTODO string - volumeAsStruct *volumeStruct - volumeAsValue sortedmap.Value - volumeGET *volumeFullGETStruct - volumeGETAsJSON []byte - volumeGETList []*volumeMiniGETStruct - volumeGETListIndex int - volumeGETListAsJSON []byte - volumeGETListLen int - volumeName string + checkPointV1 *ilayout.CheckPointV1Struct + err error + // inodeHeadV1 *ilayout.InodeHeadV1Struct + inodeNumberAsHexDigits string + inodeNumberAsUint64 uint64 + inodeNumberAsKey sortedmap.Key + inodeTable sortedmap.BPlusTree + inodeTableCache sortedmap.BPlusTreeCache + inodeTableEntryValueV1 *ilayout.InodeTableEntryValueV1Struct + inodeTableEntryValueV1AsValue sortedmap.Value + inodeTableIndex int + inodeTableLayoutIndex int + inodeTableLen int + mustBeInode string + ok bool + pathSplit []string + pendingDeleteObjectNameArrayIndex int + startTime time.Time + superBlockV1 *ilayout.SuperBlockV1Struct + toReturnTODO string + volumeAsStruct *volumeStruct + volumeAsValue sortedmap.Value + volumeGET *volumeFullGETStruct + volumeGETAsJSON []byte + volumeGETList []*volumeMiniGETStruct + volumeGETListIndex int + volumeGETListAsJSON []byte + volumeGETListLen int + volumeName string ) startTime = time.Now() @@ -504,13 +609,91 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } + checkPointV1, err = volumeAsStruct.fetchCheckPoint() + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + superBlockV1, err = volumeAsStruct.fetchSuperBlock(checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength) + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + volumeGET = &volumeFullGETStruct{ - Name: volumeAsStruct.name, - StorageURL: volumeAsStruct.storageURL, - AuthToken: volumeAsStruct.authToken, - HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), - LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), - AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), + volume: volumeAsStruct, + Name: volumeAsStruct.name, + StorageURL: volumeAsStruct.storageURL, + AuthToken: volumeAsStruct.authToken, + HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), + LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), + AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), + SuperBlockObjectName: ilayout.GetObjectNameAsString(checkPointV1.SuperBlockObjectNumber), + SuperBlockLength: checkPointV1.SuperBlockLength, + ReservedToNonce: checkPointV1.ReservedToNonce, + InodeTableLayout: make([]volumeFullGETInodeTableLayoutEntryStruct, len(superBlockV1.InodeTableLayout)), + InodeObjectCount: superBlockV1.InodeObjectCount, + InodeObjectSize: superBlockV1.InodeObjectSize, + InodeBytesReferenced: superBlockV1.InodeBytesReferenced, + PendingDeleteObjectNameArray: make([]string, len(superBlockV1.PendingDeleteObjectNumberArray)), + // InodeTable filled after the BPlusTree has been loaded later + } + + for inodeTableLayoutIndex = range volumeGET.InodeTableLayout { + volumeGET.InodeTableLayout[inodeTableLayoutIndex].ObjectName = ilayout.GetObjectNameAsString(superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectNumber) + volumeGET.InodeTableLayout[inodeTableLayoutIndex].ObjectSize = superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectSize + volumeGET.InodeTableLayout[inodeTableLayoutIndex].BytesReferenced = superBlockV1.InodeTableLayout[inodeTableLayoutIndex].BytesReferenced + } + + for pendingDeleteObjectNameArrayIndex = range volumeGET.PendingDeleteObjectNameArray { + volumeGET.PendingDeleteObjectNameArray[pendingDeleteObjectNameArrayIndex] = ilayout.GetObjectNameAsString(superBlockV1.PendingDeleteObjectNumberArray[pendingDeleteObjectNameArrayIndex]) + } + + inodeTableCache = sortedmap.NewBPlusTreeCache(volumeGETInodeTableCacheLowLimit, volumeGETInodeTableCacheHighLimit) + + inodeTable, err = sortedmap.OldBPlusTree(superBlockV1.InodeTableRootObjectNumber, superBlockV1.InodeTableRootObjectOffset, superBlockV1.InodeTableRootObjectLength, sortedmap.CompareUint64, volumeGET, inodeTableCache) + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + inodeTableLen, err = inodeTable.Len() + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + volumeGET.InodeTable = make([]volumeFullGETInodeTableEntryStruct, inodeTableLen) + + for inodeTableIndex = range volumeGET.InodeTable { + inodeNumberAsKey, inodeTableEntryValueV1AsValue, ok, err = inodeTable.GetByIndex(inodeTableIndex) + if (nil != err) || !ok { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + volumeGET.InodeTable[inodeTableIndex].InodeNumber, ok = inodeNumberAsKey.(uint64) + if !ok { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + inodeTableEntryValueV1, ok = inodeTableEntryValueV1AsValue.(*ilayout.InodeTableEntryValueV1Struct) + if !ok { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + volumeGET.InodeTable[inodeTableIndex].InodeHeadObjectName = ilayout.GetObjectNameAsString(inodeTableEntryValueV1.InodeHeadObjectNumber) + volumeGET.InodeTable[inodeTableIndex].InodeHeadLength = inodeTableEntryValueV1.InodeHeadLength } globals.Unlock() diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index 6937c71f..f4f9bc62 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -85,14 +85,9 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) [case 1] failed: %v", err) } - responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"InodeTableLayout\":null,\"InodeObjectCount\":\"\",\"InodeObjectSize\":\"\",\"InodeBytesReferenced\":\"\",\"PendingDeleteObjectNumberArray\":null}", testVolume, testGlobals.containerURL) - - _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) - if nil != err { - t.Fatalf("GET /volume/%s [case 1] failed: %v", testVolume, err) - } - if string(responseBody[:]) != responseBodyExpected { - t.Fatalf("GET /volume/%s [case 1] returned unexpected responseBody: \"%s\"", testVolume, responseBody) + _, _, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) + if nil == err { + t.Fatalf("GET /volume/%s [case 1] should have failed", testVolume) } responseBodyExpected = fmt.Sprintf("[{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0}]", testVolume, testGlobals.containerURL) @@ -125,7 +120,7 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) [case 2] failed: %v", err) } - responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"InodeTableLayout\":null,\"InodeObjectCount\":\"\",\"InodeObjectSize\":\"\",\"InodeBytesReferenced\":\"\",\"PendingDeleteObjectNumberArray\":null}", testVolume, testGlobals.containerURL, testGlobals.authToken) + responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"SuperBlockObjectName\":\"3000000000000000\",\"SuperBlockLength\":96,\"ReservedToNonce\":3,\"InodeTableLayout\":[{\"ObjectName\":\"3000000000000000\",\"ObjectSize\":58,\"BytesReferenced\":58}],\"InodeObjectCount\":1,\"InodeObjectSize\":237,\"InodeBytesReferenced\":237,\"PendingDeleteObjectNameArray\":[],\"InodeTable\":[{\"InodeNumber\":1,\"InodeHeadObjectName\":\"2000000000000000\",\"InodeHeadLength\":174}]}", testVolume, testGlobals.containerURL, testGlobals.authToken) _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) if nil != err { diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index b7a0a88c..20dbb645 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -536,7 +536,7 @@ func postVolume(storageURL string, authToken string) (err error) { ok, err = inodeTable.Put( ilayout.RootDirInodeNumber, ilayout.InodeTableEntryValueV1Struct{ - InodeHeadObjectNumber: ilayout.RootDirInodeNumber, + InodeHeadObjectNumber: rootDirInodeObjectNumber, InodeHeadLength: uint64(len(rootDirInodeHeadV1Buf)), }) if nil != err { @@ -838,3 +838,66 @@ func (volume *volumeStruct) UnpackValue(payloadData []byte) (value sortedmap.Val return } + +func (volume *volumeStruct) fetchCheckPoint() (checkPointV1 *ilayout.CheckPointV1Struct, err error) { + var ( + authOK bool + checkPointV1Buf []byte + ) + + checkPointV1Buf, authOK, err = volume.swiftObjectGet(ilayout.CheckPointObjectNumber) + if nil != err { + err = fmt.Errorf("volume.swiftObjectGet() failed: %v", err) + return + } + if !authOK { + err = fmt.Errorf("volume.swiftObjectGet() returned !authOK") + return + } + + checkPointV1, err = ilayout.UnmarshalCheckPointV1(string(checkPointV1Buf[:])) + + return +} + +func (volume *volumeStruct) fetchSuperBlock(superBlockObjectNumber uint64, superBlockLength uint64) (superBlockV1 *ilayout.SuperBlockV1Struct, err error) { + var ( + authOK bool + superBlockV1Buf []byte + ) + + superBlockV1Buf, authOK, err = volume.swiftObjectGetTail(superBlockObjectNumber, superBlockLength) + if nil != err { + err = fmt.Errorf("volume.swiftObjectGetTail() failed: %v", err) + return + } + if !authOK { + err = fmt.Errorf("volume.swiftObjectGetTail() returned !authOK") + return + } + + superBlockV1, err = ilayout.UnmarshalSuperBlockV1(superBlockV1Buf) + + return +} + +func (volume *volumeStruct) fetchInodeHead(inodeHeadObjectNumber uint64, inodeHeadLength uint64) (inodeHeadV1 *ilayout.InodeHeadV1Struct, err error) { + var ( + authOK bool + inodeHeadV1Buf []byte + ) + + inodeHeadV1Buf, authOK, err = volume.swiftObjectGetTail(inodeHeadObjectNumber, inodeHeadLength) + if nil != err { + err = fmt.Errorf("volume.swiftObjectGetTail() failed: %v", err) + return + } + if !authOK { + err = fmt.Errorf("volume.swiftObjectGetTail() returned !authOK") + return + } + + inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) + + return +} From 0242b0ddcefc3180a23a42033aad533f647a755e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 15 Aug 2021 11:36:48 -0700 Subject: [PATCH 061/258] Implemented server side of GET /volume//inode/XXX --- go.mod | 2 +- go.sum | 2 + imgr/imgrpkg/http-server.go | 672 ++++++++++++++++++++++++++++++------ 3 files changed, 567 insertions(+), 109 deletions(-) diff --git a/go.mod b/go.mod index 836b373c..00537fff 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/cast v1.4.0 // indirect + github.com/spf13/cast v1.4.1 // indirect github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect diff --git a/go.sum b/go.sum index e8a71cea..6dfac08d 100644 --- a/go.sum +++ b/go.sum @@ -378,6 +378,8 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 1dd65af7..2335d8c8 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -25,8 +25,12 @@ const ( startHTTPServerUpCheckDelay = 100 * time.Millisecond startHTTPServerUpCheckMaxRetries = 10 - volumeGETInodeTableCacheLowLimit = 1000 - volumeGETInodeTableCacheHighLimit = 1010 + httpServerInodeTableCacheLowLimit = 1000 + httpServerInodeTableCacheHighLimit = 1010 + httpServerDirectoryCacheLowLimit = 1000 + httpServerDirectoryCacheHighLimit = 1010 + httpServerExtentMapCacheLowLimit = 1000 + httpServerExtentMapCacheHighLimit = 1010 ) func startHTTPServer() (err error) { @@ -305,7 +309,7 @@ func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Req } } -type volumeMiniGETStruct struct { +type volumeListGETEntryStruct struct { Name string StorageURL string AuthToken string @@ -314,20 +318,25 @@ type volumeMiniGETStruct struct { AuthTokenExpiredMounts uint64 } -type volumeFullGETInodeTableLayoutEntryStruct struct { +type volumeGETInodeTableLayoutEntryStruct struct { ObjectName string // == ilayout.GetObjectNameAsString(ObjectNumber) ObjectSize uint64 BytesReferenced uint64 } -type volumeFullGETInodeTableEntryStruct struct { +type volumeGETInodeTableEntryStruct struct { InodeNumber uint64 InodeHeadObjectName string // == ilayout.GetObjectNameAsString(InodeHeadObjectNumber) InodeHeadLength uint64 } -type volumeFullGETStruct struct { - volume *volumeStruct // will not be encoded by json.Marshal() +type httpServerInodeTableWrapperStruct struct { + volume *volumeStruct + cache sortedmap.BPlusTreeCache + table sortedmap.BPlusTree +} + +type volumeGETStruct struct { Name string StorageURL string AuthToken string @@ -337,17 +346,17 @@ type volumeFullGETStruct struct { SuperBlockObjectName string // == ilayout.GetObjectNameAsString(SuperBlockObjectNumber) SuperBlockLength uint64 ReservedToNonce uint64 - InodeTableLayout []volumeFullGETInodeTableLayoutEntryStruct + InodeTableLayout []volumeGETInodeTableLayoutEntryStruct InodeObjectCount uint64 InodeObjectSize uint64 InodeBytesReferenced uint64 PendingDeleteObjectNameArray []string // == []ilayout.GetObjectNameAsString(PendingDeleteObjectNumber) - InodeTable []volumeFullGETInodeTableEntryStruct + InodeTable []volumeGETInodeTableEntryStruct } type inodeGETLinkTableEntryStruct struct { ParentDirInodeNumber uint64 - ParentDirEntryName uint64 + ParentDirEntryName string } type inodeGETStreamTableEntryStruct struct { @@ -358,7 +367,7 @@ type inodeGETStreamTableEntryStruct struct { type dirInodeGETPayloadEntryStruct struct { BaseName string InodeNumber uint64 - InodeType string // One of "InodeTypeDir", "InodeTypeFile", or "InodeTypeSymLink" + InodeType string // One of "Dir", "File", or "SymLink" } type fileInodeGETPayloadEntryStruct struct { @@ -374,9 +383,9 @@ type inodeGETLayoutEntryStruct struct { BytesReferenced uint64 } -type dirInodeGETStruct struct { +type inodeGETStruct struct { InodeNumber uint64 - InodeType string // == "InodeTypeDir" + InodeType string // == "Dir", "File", or "Symlink" LinkTable []inodeGETLinkTableEntryStruct ModificationTime string StatusChangeTime string @@ -384,54 +393,54 @@ type dirInodeGETStruct struct { UserID uint64 GroupID uint64 StreamTable []inodeGETStreamTableEntryStruct - Payload []dirInodeGETPayloadEntryStruct - Layout []inodeGETLayoutEntryStruct +} + +type httpServerDirectoryWrapperStruct struct { + volume *volumeStruct + cache sortedmap.BPlusTreeCache + table sortedmap.BPlusTree +} + +type dirInodeGETStruct struct { + inodeGETStruct + Payload []dirInodeGETPayloadEntryStruct + Layout []inodeGETLayoutEntryStruct +} + +type httpServerExtentMapWrapperStruct struct { + volume *volumeStruct + cache sortedmap.BPlusTreeCache + table sortedmap.BPlusTree } type fileInodeGETStruct struct { - InodeNumber uint64 - InodeType string // == "InodeTypeFile" - LinkTable []inodeGETLinkTableEntryStruct - Size string - ModificationTime string - StatusChangeTime string - Mode uint16 - UserID uint64 - GroupID uint64 - StreamTable []inodeGETStreamTableEntryStruct - Payload []fileInodeGETPayloadEntryStruct - Layout []inodeGETLayoutEntryStruct + inodeGETStruct + Size uint64 + Payload []fileInodeGETPayloadEntryStruct + Layout []inodeGETLayoutEntryStruct } type symLinkInodeGETStruct struct { - InodeNumber uint64 - InodeType string // == "InodeTypeSymLink" - LinkTable []inodeGETLinkTableEntryStruct - ModificationTime string - StatusChangeTime string - Mode uint16 - UserID uint64 - GroupID uint64 - StreamTable []inodeGETStreamTableEntryStruct - SymLinkTarget string + inodeGETStruct + SymLinkTarget string } -func (volumeGET *volumeFullGETStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { err = fmt.Errorf("not implemented") return } -func (volumeGET *volumeFullGETStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { err = fmt.Errorf("not implemented") return } -func (volumeGET *volumeFullGETStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { var ( authOK bool ) - nodeByteSlice, authOK, err = volumeGET.volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + nodeByteSlice, authOK, err = inodeTableWrapper.volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) if nil != err { err = fmt.Errorf("volume.swiftObjectGetRange() failed: %v", err) return @@ -444,22 +453,22 @@ func (volumeGET *volumeFullGETStruct) GetNode(objectNumber uint64, objectOffset return } -func (volumeGET *volumeFullGETStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { err = fmt.Errorf("not implemented") return } -func (volumeGET *volumeFullGETStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { err = fmt.Errorf("not implemented") return } -func (volumeGET *volumeFullGETStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { err = fmt.Errorf("not implemented") return } -func (volumeGET *volumeFullGETStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { var ( nextPos int inodeNumber uint64 @@ -474,12 +483,12 @@ func (volumeGET *volumeFullGETStruct) UnpackKey(payloadData []byte) (key sortedm return } -func (volumeGET *volumeFullGETStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { err = fmt.Errorf("not implemented") return } -func (volumeGET *volumeFullGETStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { +func (inodeTableWrapper *httpServerInodeTableWrapperStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { var ( bytesConsumedAsInt int inodeTableEntryValueV1 *ilayout.InodeTableEntryValueV1Struct @@ -494,37 +503,210 @@ func (volumeGET *volumeFullGETStruct) UnpackValue(payloadData []byte) (value sor return } +func (directoryWrapper *httpServerDirectoryWrapperStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (directoryWrapper *httpServerDirectoryWrapperStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (directoryWrapper *httpServerDirectoryWrapperStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { + var ( + authOK bool + ) + + nodeByteSlice, authOK, err = directoryWrapper.volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + if nil != err { + err = fmt.Errorf("volume.swiftObjectGetRange() failed: %v", err) + return + } + if !authOK { + err = fmt.Errorf("volume.swiftObjectGetRange() returned !authOK") + return + } + + return +} + +func (directoryWrapper *httpServerDirectoryWrapperStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (directoryWrapper *httpServerDirectoryWrapperStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + err = fmt.Errorf("not implemented") + return +} + +func (directoryWrapper *httpServerDirectoryWrapperStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (directoryWrapper *httpServerDirectoryWrapperStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + var ( + nextPos int + baseName string + ) + + baseName, nextPos, err = ilayout.GetLEStringFromBuf(payloadData, 0) + if nil == err { + key = baseName + bytesConsumed = uint64(nextPos) + } + + return +} + +func (directoryWrapper *httpServerDirectoryWrapperStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (directoryWrapper *httpServerDirectoryWrapperStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + var ( + bytesConsumedAsInt int + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + ) + + directoryEntryValueV1, bytesConsumedAsInt, err = ilayout.UnmarshalDirectoryEntryValueV1(payloadData) + if nil == err { + value = directoryEntryValueV1 + bytesConsumed = uint64(bytesConsumedAsInt) + } + + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { + var ( + authOK bool + ) + + nodeByteSlice, authOK, err = extentMapWrapper.volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + if nil != err { + err = fmt.Errorf("volume.swiftObjectGetRange() failed: %v", err) + return + } + if !authOK { + err = fmt.Errorf("volume.swiftObjectGetRange() returned !authOK") + return + } + + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + err = fmt.Errorf("not implemented") + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + var ( + nextPos int + fileOffset uint64 + ) + + fileOffset, nextPos, err = ilayout.GetLEUint64FromBuf(payloadData, 0) + if nil == err { + key = fileOffset + bytesConsumed = uint64(nextPos) + } + + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + err = fmt.Errorf("not implemented") + return +} + +func (extentMapWrapper *httpServerExtentMapWrapperStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + var ( + bytesConsumedAsInt int + extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct + ) + + extentMapEntryValueV1, bytesConsumedAsInt, err = ilayout.UnmarshalExtentMapEntryValueV1(payloadData) + if nil == err { + value = extentMapEntryValueV1 + bytesConsumed = uint64(bytesConsumedAsInt) + } + + return +} + func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( - checkPointV1 *ilayout.CheckPointV1Struct - err error - // inodeHeadV1 *ilayout.InodeHeadV1Struct - inodeNumberAsHexDigits string - inodeNumberAsUint64 uint64 - inodeNumberAsKey sortedmap.Key - inodeTable sortedmap.BPlusTree - inodeTableCache sortedmap.BPlusTreeCache - inodeTableEntryValueV1 *ilayout.InodeTableEntryValueV1Struct - inodeTableEntryValueV1AsValue sortedmap.Value - inodeTableIndex int - inodeTableLayoutIndex int - inodeTableLen int - mustBeInode string - ok bool - pathSplit []string - pendingDeleteObjectNameArrayIndex int - startTime time.Time - superBlockV1 *ilayout.SuperBlockV1Struct - toReturnTODO string - volumeAsStruct *volumeStruct - volumeAsValue sortedmap.Value - volumeGET *volumeFullGETStruct - volumeGETAsJSON []byte - volumeGETList []*volumeMiniGETStruct - volumeGETListIndex int - volumeGETListAsJSON []byte - volumeGETListLen int - volumeName string + checkPointV1 *ilayout.CheckPointV1Struct + directoryEntryIndex int + directoryEntryKeyV1AsKey sortedmap.Key + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + directoryLen int + directoryWrapper *httpServerDirectoryWrapperStruct + err error + extentMapEntryIndex int + extentMapEntryKeyV1AsKey sortedmap.Key + extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct + extentMapEntryValueV1AsValue sortedmap.Value + extentMapLen int + extentMapWrapper *httpServerExtentMapWrapperStruct + inodeGET interface{} + inodeGETAsJSON []byte + inodeGETLayoutEntryIndex int + inodeGETLinkTableEntryIndex int + inodeGETStreamTableEntryIndex int + inodeGETStreamTableEntryValueByte byte + inodeGETStreamTableEntryValueIndex int + inodeHeadV1 *ilayout.InodeHeadV1Struct + inodeNumberAsHexDigits string + inodeNumberAsUint64 uint64 + inodeNumberAsKey sortedmap.Key + inodeTableEntryValueV1 *ilayout.InodeTableEntryValueV1Struct + inodeTableEntryValueV1AsValue sortedmap.Value + inodeTableIndex int + inodeTableLayoutIndex int + inodeTableLen int + inodeTableWrapper *httpServerInodeTableWrapperStruct + mustBeInode string + ok bool + pathSplit []string + pendingDeleteObjectNameArrayIndex int + startTime time.Time + superBlockV1 *ilayout.SuperBlockV1Struct + volumeAsStruct *volumeStruct + volumeAsValue sortedmap.Value + volumeGET *volumeGETStruct + volumeGETAsJSON []byte + volumeListGET []*volumeListGETEntryStruct + volumeListGETIndex int + volumeListGETAsJSON []byte + volumeListGETLen int + volumeName string ) startTime = time.Now() @@ -541,28 +723,28 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ globals.Lock() - volumeGETListLen, err = globals.volumeMap.Len() + volumeListGETLen, err = globals.volumeMap.Len() if nil != err { logFatal(err) } - volumeGETList = make([]*volumeMiniGETStruct, volumeGETListLen) + volumeListGET = make([]*volumeListGETEntryStruct, volumeListGETLen) - for volumeGETListIndex = 0; volumeGETListIndex < volumeGETListLen; volumeGETListIndex++ { - _, volumeAsValue, ok, err = globals.volumeMap.GetByIndex(volumeGETListIndex) + for volumeListGETIndex = 0; volumeListGETIndex < volumeListGETLen; volumeListGETIndex++ { + _, volumeAsValue, ok, err = globals.volumeMap.GetByIndex(volumeListGETIndex) if nil != err { logFatal(err) } if !ok { - logFatalf("globals.volumeMap[] len (%d) is wrong", volumeGETListLen) + logFatalf("globals.volumeMap[] len (%d) is wrong", volumeListGETLen) } volumeAsStruct, ok = volumeAsValue.(*volumeStruct) if !ok { - logFatalf("globals.volumeMap[%d] was not a *volumeStruct", volumeGETListIndex) + logFatalf("globals.volumeMap[%d] was not a *volumeStruct", volumeListGETIndex) } - volumeGETList[volumeGETListIndex] = &volumeMiniGETStruct{ + volumeListGET[volumeListGETIndex] = &volumeListGETEntryStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, AuthToken: volumeAsStruct.authToken, @@ -574,18 +756,18 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ globals.Unlock() - volumeGETListAsJSON, err = json.Marshal(volumeGETList) + volumeListGETAsJSON, err = json.Marshal(volumeListGET) if nil != err { logFatal(err) } - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeGETListAsJSON))) + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeListGETAsJSON))) responseWriter.Header().Set("Content-Type", "application/json") responseWriter.WriteHeader(http.StatusOK) - _, err = responseWriter.Write(volumeGETListAsJSON) + _, err = responseWriter.Write(volumeListGETAsJSON) if nil != err { - logWarnf("responseWriter.Write(volumeGETListAsJSON) failed: %v", err) + logWarnf("responseWriter.Write(volumeListGETAsJSON) failed: %v", err) } case 3: // Form: /volume/ @@ -623,8 +805,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ return } - volumeGET = &volumeFullGETStruct{ - volume: volumeAsStruct, + volumeGET = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, AuthToken: volumeAsStruct.authToken, @@ -634,12 +815,11 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ SuperBlockObjectName: ilayout.GetObjectNameAsString(checkPointV1.SuperBlockObjectNumber), SuperBlockLength: checkPointV1.SuperBlockLength, ReservedToNonce: checkPointV1.ReservedToNonce, - InodeTableLayout: make([]volumeFullGETInodeTableLayoutEntryStruct, len(superBlockV1.InodeTableLayout)), + InodeTableLayout: make([]volumeGETInodeTableLayoutEntryStruct, len(superBlockV1.InodeTableLayout)), InodeObjectCount: superBlockV1.InodeObjectCount, InodeObjectSize: superBlockV1.InodeObjectSize, InodeBytesReferenced: superBlockV1.InodeBytesReferenced, PendingDeleteObjectNameArray: make([]string, len(superBlockV1.PendingDeleteObjectNumberArray)), - // InodeTable filled after the BPlusTree has been loaded later } for inodeTableLayoutIndex = range volumeGET.InodeTableLayout { @@ -652,26 +832,29 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeGET.PendingDeleteObjectNameArray[pendingDeleteObjectNameArrayIndex] = ilayout.GetObjectNameAsString(superBlockV1.PendingDeleteObjectNumberArray[pendingDeleteObjectNameArrayIndex]) } - inodeTableCache = sortedmap.NewBPlusTreeCache(volumeGETInodeTableCacheLowLimit, volumeGETInodeTableCacheHighLimit) + inodeTableWrapper = &httpServerInodeTableWrapperStruct{ + volume: volumeAsStruct, + cache: sortedmap.NewBPlusTreeCache(httpServerInodeTableCacheLowLimit, httpServerInodeTableCacheHighLimit), + } - inodeTable, err = sortedmap.OldBPlusTree(superBlockV1.InodeTableRootObjectNumber, superBlockV1.InodeTableRootObjectOffset, superBlockV1.InodeTableRootObjectLength, sortedmap.CompareUint64, volumeGET, inodeTableCache) + inodeTableWrapper.table, err = sortedmap.OldBPlusTree(superBlockV1.InodeTableRootObjectNumber, superBlockV1.InodeTableRootObjectOffset, superBlockV1.InodeTableRootObjectLength, sortedmap.CompareUint64, inodeTableWrapper, inodeTableWrapper.cache) if nil != err { globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return } - inodeTableLen, err = inodeTable.Len() + inodeTableLen, err = inodeTableWrapper.table.Len() if nil != err { globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return } - volumeGET.InodeTable = make([]volumeFullGETInodeTableEntryStruct, inodeTableLen) + volumeGET.InodeTable = make([]volumeGETInodeTableEntryStruct, inodeTableLen) for inodeTableIndex = range volumeGET.InodeTable { - inodeNumberAsKey, inodeTableEntryValueV1AsValue, ok, err = inodeTable.GetByIndex(inodeTableIndex) + inodeNumberAsKey, inodeTableEntryValueV1AsValue, ok, err = inodeTableWrapper.table.GetByIndex(inodeTableIndex) if (nil != err) || !ok { globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) @@ -680,16 +863,12 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeGET.InodeTable[inodeTableIndex].InodeNumber, ok = inodeNumberAsKey.(uint64) if !ok { - globals.Unlock() - responseWriter.WriteHeader(http.StatusUnauthorized) - return + logFatalf("inodeNumberAsKey.(uint64) returned !ok") } inodeTableEntryValueV1, ok = inodeTableEntryValueV1AsValue.(*ilayout.InodeTableEntryValueV1Struct) if !ok { - globals.Unlock() - responseWriter.WriteHeader(http.StatusUnauthorized) - return + logFatalf("inodeTableEntryValueV1AsValue.(*ilayout.InodeTableEntryValueV1Struct) returned !ok") } volumeGET.InodeTable[inodeTableIndex].InodeHeadObjectName = ilayout.GetObjectNameAsString(inodeTableEntryValueV1.InodeHeadObjectNumber) @@ -745,17 +924,294 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } - toReturnTODO = fmt.Sprintf("TODO: GET /volume/%s/inode/%016X", volumeAsStruct.name, inodeNumberAsUint64) + checkPointV1, err = volumeAsStruct.fetchCheckPoint() + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + superBlockV1, err = volumeAsStruct.fetchSuperBlock(checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength) + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + inodeTableWrapper = &httpServerInodeTableWrapperStruct{ + volume: volumeAsStruct, + cache: sortedmap.NewBPlusTreeCache(httpServerInodeTableCacheLowLimit, httpServerInodeTableCacheHighLimit), + } + + inodeTableWrapper.table, err = sortedmap.OldBPlusTree(superBlockV1.InodeTableRootObjectNumber, superBlockV1.InodeTableRootObjectOffset, superBlockV1.InodeTableRootObjectLength, sortedmap.CompareUint64, inodeTableWrapper, inodeTableWrapper.cache) + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + inodeTableEntryValueV1AsValue, ok, err = inodeTableWrapper.table.GetByKey(inodeNumberAsUint64) + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + if !ok { + globals.Unlock() + responseWriter.WriteHeader(http.StatusNotFound) + return + } + + inodeTableEntryValueV1, ok = inodeTableEntryValueV1AsValue.(*ilayout.InodeTableEntryValueV1Struct) + if !ok { + logFatalf("inodeTableEntryValueV1AsValue.(*ilayout.InodeTableEntryValueV1Struct) returned !ok") + } + + inodeHeadV1, err = volumeAsStruct.fetchInodeHead(inodeTableEntryValueV1.InodeHeadObjectNumber, inodeTableEntryValueV1.InodeHeadLength) + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + if inodeNumberAsUint64 != inodeHeadV1.InodeNumber { + logFatalf("Lookup of Inode 0x%016X returned an Inode with InodeNumber 0x%016X", inodeNumberAsUint64, inodeHeadV1.InodeNumber) + } + + // Apologies for the hack below due to Golang not supporting full inheritance + + switch inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + directoryWrapper = &httpServerDirectoryWrapperStruct{ + volume: volumeAsStruct, + cache: sortedmap.NewBPlusTreeCache(httpServerDirectoryCacheLowLimit, httpServerDirectoryCacheHighLimit), + } + + directoryWrapper.table, err = sortedmap.OldBPlusTree(inodeHeadV1.PayloadObjectNumber, inodeHeadV1.PayloadObjectOffset, inodeHeadV1.PayloadObjectLength, sortedmap.CompareString, directoryWrapper, directoryWrapper.cache) + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + directoryLen, err = directoryWrapper.table.Len() + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + inodeGET = &dirInodeGETStruct{ + inodeGETStruct{ + InodeNumber: inodeHeadV1.InodeNumber, + InodeType: "Dir", + LinkTable: make([]inodeGETLinkTableEntryStruct, len(inodeHeadV1.LinkTable)), + ModificationTime: inodeHeadV1.ModificationTime.Format(time.RFC3339), + StatusChangeTime: inodeHeadV1.StatusChangeTime.Format(time.RFC3339), + Mode: inodeHeadV1.Mode, + UserID: inodeHeadV1.UserID, + GroupID: inodeHeadV1.GroupID, + StreamTable: make([]inodeGETStreamTableEntryStruct, len(inodeHeadV1.StreamTable)), + }, + make([]dirInodeGETPayloadEntryStruct, directoryLen), + make([]inodeGETLayoutEntryStruct, len(inodeHeadV1.Layout)), + } + + for inodeGETLinkTableEntryIndex = range inodeHeadV1.LinkTable { + inodeGET.(*dirInodeGETStruct).LinkTable[inodeGETLinkTableEntryIndex].ParentDirInodeNumber = inodeHeadV1.LinkTable[inodeGETLinkTableEntryIndex].ParentDirInodeNumber + inodeGET.(*dirInodeGETStruct).LinkTable[inodeGETLinkTableEntryIndex].ParentDirEntryName = inodeHeadV1.LinkTable[inodeGETLinkTableEntryIndex].ParentDirEntryName + } + + for inodeGETStreamTableEntryIndex = range inodeHeadV1.StreamTable { + inodeGET.(*dirInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Name = inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Name + if len(inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Value) == 0 { + inodeGET.(*dirInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value = "" + } else { + for inodeGETStreamTableEntryValueIndex, inodeGETStreamTableEntryValueByte = range inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Value { + if inodeGETStreamTableEntryValueIndex == 0 { + inodeGET.(*dirInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value = fmt.Sprintf("%02X", inodeGETStreamTableEntryValueByte) + } else { + inodeGET.(*dirInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value += fmt.Sprintf(" %02X", inodeGETStreamTableEntryValueByte) + } + } + } + } + + for directoryEntryIndex = 0; directoryEntryIndex < directoryLen; directoryEntryIndex++ { + directoryEntryKeyV1AsKey, directoryEntryValueV1AsValue, ok, err = directoryWrapper.table.GetByIndex(directoryEntryIndex) + if (nil != err) || !ok { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + inodeGET.(*dirInodeGETStruct).Payload[directoryEntryIndex].BaseName, ok = directoryEntryKeyV1AsKey.(string) + if !ok { + logFatalf("directoryEntryKeyV1AsKey.(string) returned !ok") + } + + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + inodeGET.(*dirInodeGETStruct).Payload[directoryEntryIndex].InodeNumber = directoryEntryValueV1.InodeNumber + + switch directoryEntryValueV1.InodeType { + case ilayout.InodeTypeDir: + inodeGET.(*dirInodeGETStruct).Payload[directoryEntryIndex].InodeType = "Dir" + case ilayout.InodeTypeFile: + inodeGET.(*dirInodeGETStruct).Payload[directoryEntryIndex].InodeType = "File" + case ilayout.InodeTypeSymLink: + inodeGET.(*dirInodeGETStruct).Payload[directoryEntryIndex].InodeType = "SymLink" + default: + logFatalf("Directory entry in DirInode 0x%016X contains unknown InodeType: 0x%02X", inodeGET.(*dirInodeGETStruct).Payload[directoryEntryIndex].InodeNumber, directoryEntryValueV1.InodeType) + } + } + + for inodeGETLayoutEntryIndex = range inodeHeadV1.Layout { + inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].ObjectName = ilayout.GetObjectNameAsString(inodeHeadV1.Layout[inodeGETLayoutEntryIndex].ObjectNumber) + inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].ObjectSize = inodeHeadV1.Layout[inodeGETLayoutEntryIndex].ObjectSize + inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].BytesReferenced = inodeHeadV1.Layout[inodeGETLayoutEntryIndex].BytesReferenced + } + case ilayout.InodeTypeFile: + extentMapWrapper = &httpServerExtentMapWrapperStruct{ + volume: volumeAsStruct, + cache: sortedmap.NewBPlusTreeCache(httpServerExtentMapCacheLowLimit, httpServerExtentMapCacheHighLimit), + } + + extentMapWrapper.table, err = sortedmap.OldBPlusTree(inodeHeadV1.PayloadObjectNumber, inodeHeadV1.PayloadObjectOffset, inodeHeadV1.PayloadObjectLength, sortedmap.CompareUint64, extentMapWrapper, extentMapWrapper.cache) + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + extentMapLen, err = extentMapWrapper.table.Len() + if nil != err { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + inodeGET = &fileInodeGETStruct{ + inodeGETStruct{ + InodeNumber: inodeHeadV1.InodeNumber, + InodeType: "File", + LinkTable: make([]inodeGETLinkTableEntryStruct, len(inodeHeadV1.LinkTable)), + ModificationTime: inodeHeadV1.ModificationTime.Format(time.RFC3339), + StatusChangeTime: inodeHeadV1.StatusChangeTime.Format(time.RFC3339), + Mode: inodeHeadV1.Mode, + UserID: inodeHeadV1.UserID, + GroupID: inodeHeadV1.GroupID, + StreamTable: make([]inodeGETStreamTableEntryStruct, len(inodeHeadV1.StreamTable)), + }, + inodeHeadV1.Size, + make([]fileInodeGETPayloadEntryStruct, extentMapLen), + make([]inodeGETLayoutEntryStruct, len(inodeHeadV1.Layout)), + } + + for inodeGETLinkTableEntryIndex = range inodeHeadV1.LinkTable { + inodeGET.(*fileInodeGETStruct).LinkTable[inodeGETLinkTableEntryIndex].ParentDirInodeNumber = inodeHeadV1.LinkTable[inodeGETLinkTableEntryIndex].ParentDirInodeNumber + inodeGET.(*fileInodeGETStruct).LinkTable[inodeGETLinkTableEntryIndex].ParentDirEntryName = inodeHeadV1.LinkTable[inodeGETLinkTableEntryIndex].ParentDirEntryName + } + + for inodeGETStreamTableEntryIndex = range inodeHeadV1.StreamTable { + inodeGET.(*fileInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Name = inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Name + if len(inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Value) == 0 { + inodeGET.(*fileInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value = "" + } else { + for inodeGETStreamTableEntryValueIndex, inodeGETStreamTableEntryValueByte = range inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Value { + if inodeGETStreamTableEntryValueIndex == 0 { + inodeGET.(*fileInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value = fmt.Sprintf("%02X", inodeGETStreamTableEntryValueByte) + } else { + inodeGET.(*fileInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value += fmt.Sprintf(" %02X", inodeGETStreamTableEntryValueByte) + } + } + } + } + + for extentMapEntryIndex = 0; extentMapEntryIndex < extentMapLen; extentMapEntryIndex++ { + extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = extentMapWrapper.table.GetByIndex(extentMapEntryIndex) + if (nil != err) || !ok { + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + inodeGET.(*fileInodeGETStruct).Payload[extentMapEntryIndex].FileOffset, ok = extentMapEntryKeyV1AsKey.(uint64) + if !ok { + logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") + } + + extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) + if !ok { + logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") + } + + inodeGET.(*fileInodeGETStruct).Payload[extentMapEntryIndex].Length = extentMapEntryValueV1.Length + inodeGET.(*fileInodeGETStruct).Payload[extentMapEntryIndex].ObjectName = ilayout.GetObjectNameAsString(extentMapEntryValueV1.ObjectNumber) + inodeGET.(*fileInodeGETStruct).Payload[extentMapEntryIndex].ObjectOffset = extentMapEntryValueV1.ObjectOffset + } + + for inodeGETLayoutEntryIndex = range inodeHeadV1.Layout { + inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].ObjectName = ilayout.GetObjectNameAsString(inodeHeadV1.Layout[inodeGETLayoutEntryIndex].ObjectNumber) + inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].ObjectSize = inodeHeadV1.Layout[inodeGETLayoutEntryIndex].ObjectSize + inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].BytesReferenced = inodeHeadV1.Layout[inodeGETLayoutEntryIndex].BytesReferenced + } + case ilayout.InodeTypeSymLink: + inodeGET = &symLinkInodeGETStruct{ + inodeGETStruct{ + InodeNumber: inodeHeadV1.InodeNumber, + InodeType: "SymLink", + LinkTable: make([]inodeGETLinkTableEntryStruct, len(inodeHeadV1.LinkTable)), + ModificationTime: inodeHeadV1.ModificationTime.Format(time.RFC3339), + StatusChangeTime: inodeHeadV1.StatusChangeTime.Format(time.RFC3339), + Mode: inodeHeadV1.Mode, + UserID: inodeHeadV1.UserID, + GroupID: inodeHeadV1.GroupID, + StreamTable: make([]inodeGETStreamTableEntryStruct, len(inodeHeadV1.StreamTable)), + }, + inodeHeadV1.SymLinkTarget, + } + + for inodeGETLinkTableEntryIndex = range inodeHeadV1.LinkTable { + inodeGET.(*symLinkInodeGETStruct).LinkTable[inodeGETLinkTableEntryIndex].ParentDirInodeNumber = inodeHeadV1.LinkTable[inodeGETLinkTableEntryIndex].ParentDirInodeNumber + inodeGET.(*symLinkInodeGETStruct).LinkTable[inodeGETLinkTableEntryIndex].ParentDirEntryName = inodeHeadV1.LinkTable[inodeGETLinkTableEntryIndex].ParentDirEntryName + } + + for inodeGETStreamTableEntryIndex = range inodeHeadV1.StreamTable { + inodeGET.(*symLinkInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Name = inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Name + if len(inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Value) == 0 { + inodeGET.(*symLinkInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value = "" + } else { + for inodeGETStreamTableEntryValueIndex, inodeGETStreamTableEntryValueByte = range inodeHeadV1.StreamTable[inodeGETStreamTableEntryIndex].Value { + if inodeGETStreamTableEntryValueIndex == 0 { + inodeGET.(*symLinkInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value = fmt.Sprintf("%02X", inodeGETStreamTableEntryValueByte) + } else { + inodeGET.(*symLinkInodeGETStruct).StreamTable[inodeGETStreamTableEntryIndex].Value += fmt.Sprintf(" %02X", inodeGETStreamTableEntryValueByte) + } + } + } + } + default: + logFatalf("Inode 0x%016X contains unknown InodeType: 0x%02X", inodeNumberAsUint64, inodeHeadV1.InodeType) + } globals.Unlock() - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(toReturnTODO))) - responseWriter.Header().Set("Content-Type", "text/plain") + inodeGETAsJSON, err = json.Marshal(inodeGET) + if nil != err { + logFatal(err) + } + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(inodeGETAsJSON))) + responseWriter.Header().Set("Content-Type", "application/json") responseWriter.WriteHeader(http.StatusOK) - _, err = responseWriter.Write([]byte(toReturnTODO)) + _, err = responseWriter.Write(inodeGETAsJSON) if nil != err { - logWarnf("responseWriter.Write([]byte(toReturnTODO)) failed: %v", err) + logWarnf("responseWriter.Write(inodeGETAsJSON) failed: %v", err) } } else { globals.Unlock() @@ -772,7 +1228,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ func serveHTTPPost(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { switch { - case "/volume" == requestPath: + case requestPath == "/volume": serveHTTPPostOfVolume(responseWriter, request, requestBody) default: responseWriter.WriteHeader(http.StatusNotFound) From 8c5b0d5613deec6a9b94b05684e716715e2fdefe Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 16 Aug 2021 17:47:03 -0700 Subject: [PATCH 062/258] Added support for X-Auth-Token Header in GET /volume/] --- go.mod | 2 +- go.sum | 6 ++---- imgr/imgrpkg/http-server.go | 37 ++++++++++++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 00537fff..0523f849 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( go.etcd.io/etcd v3.3.25+incompatible go.uber.org/zap v1.18.1 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e - golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e + golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect diff --git a/go.sum b/go.sum index 6dfac08d..af8bb0eb 100644 --- a/go.sum +++ b/go.sum @@ -376,8 +376,6 @@ github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= -github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= @@ -608,8 +606,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 2335d8c8..07db913a 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -696,10 +696,12 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ ok bool pathSplit []string pendingDeleteObjectNameArrayIndex int + requestAuthToken string startTime time.Time superBlockV1 *ilayout.SuperBlockV1Struct volumeAsStruct *volumeStruct volumeAsValue sortedmap.Value + volumeAuthToken string volumeGET *volumeGETStruct volumeGETAsJSON []byte volumeListGET []*volumeListGETEntryStruct @@ -791,8 +793,15 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } + requestAuthToken = request.Header.Get("X-Auth-Token") + volumeAuthToken = volumeAsStruct.authToken + if requestAuthToken != "" { + volumeAsStruct.authToken = requestAuthToken + } + checkPointV1, err = volumeAsStruct.fetchCheckPoint() if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -800,6 +809,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ superBlockV1, err = volumeAsStruct.fetchSuperBlock(checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength) if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -808,7 +818,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeGET = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, - AuthToken: volumeAsStruct.authToken, + AuthToken: volumeAuthToken, HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), @@ -839,6 +849,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ inodeTableWrapper.table, err = sortedmap.OldBPlusTree(superBlockV1.InodeTableRootObjectNumber, superBlockV1.InodeTableRootObjectOffset, superBlockV1.InodeTableRootObjectLength, sortedmap.CompareUint64, inodeTableWrapper, inodeTableWrapper.cache) if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -846,6 +857,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ inodeTableLen, err = inodeTableWrapper.table.Len() if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -856,6 +868,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ for inodeTableIndex = range volumeGET.InodeTable { inodeNumberAsKey, inodeTableEntryValueV1AsValue, ok, err = inodeTableWrapper.table.GetByIndex(inodeTableIndex) if (nil != err) || !ok { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -875,6 +888,8 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeGET.InodeTable[inodeTableIndex].InodeHeadLength = inodeTableEntryValueV1.InodeHeadLength } + volumeAsStruct.authToken = volumeAuthToken + globals.Unlock() volumeGETAsJSON, err = json.Marshal(volumeGET) @@ -924,8 +939,15 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } + volumeAuthToken = volumeAsStruct.authToken + requestAuthToken = request.Header.Get("X-Auth-Token") + if requestAuthToken != "" { + volumeAsStruct.authToken = requestAuthToken + } + checkPointV1, err = volumeAsStruct.fetchCheckPoint() if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -933,6 +955,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ superBlockV1, err = volumeAsStruct.fetchSuperBlock(checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength) if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -945,6 +968,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ inodeTableWrapper.table, err = sortedmap.OldBPlusTree(superBlockV1.InodeTableRootObjectNumber, superBlockV1.InodeTableRootObjectOffset, superBlockV1.InodeTableRootObjectLength, sortedmap.CompareUint64, inodeTableWrapper, inodeTableWrapper.cache) if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -952,11 +976,13 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ inodeTableEntryValueV1AsValue, ok, err = inodeTableWrapper.table.GetByKey(inodeNumberAsUint64) if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return } if !ok { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusNotFound) return @@ -969,6 +995,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ inodeHeadV1, err = volumeAsStruct.fetchInodeHead(inodeTableEntryValueV1.InodeHeadObjectNumber, inodeTableEntryValueV1.InodeHeadLength) if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -989,6 +1016,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ directoryWrapper.table, err = sortedmap.OldBPlusTree(inodeHeadV1.PayloadObjectNumber, inodeHeadV1.PayloadObjectOffset, inodeHeadV1.PayloadObjectLength, sortedmap.CompareString, directoryWrapper, directoryWrapper.cache) if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -996,6 +1024,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ directoryLen, err = directoryWrapper.table.Len() if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -1040,6 +1069,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ for directoryEntryIndex = 0; directoryEntryIndex < directoryLen; directoryEntryIndex++ { directoryEntryKeyV1AsKey, directoryEntryValueV1AsValue, ok, err = directoryWrapper.table.GetByIndex(directoryEntryIndex) if (nil != err) || !ok { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -1082,6 +1112,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ extentMapWrapper.table, err = sortedmap.OldBPlusTree(inodeHeadV1.PayloadObjectNumber, inodeHeadV1.PayloadObjectOffset, inodeHeadV1.PayloadObjectLength, sortedmap.CompareUint64, extentMapWrapper, extentMapWrapper.cache) if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -1089,6 +1120,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ extentMapLen, err = extentMapWrapper.table.Len() if nil != err { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -1134,6 +1166,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ for extentMapEntryIndex = 0; extentMapEntryIndex < extentMapLen; extentMapEntryIndex++ { extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = extentMapWrapper.table.GetByIndex(extentMapEntryIndex) if (nil != err) || !ok { + volumeAsStruct.authToken = volumeAuthToken globals.Unlock() responseWriter.WriteHeader(http.StatusUnauthorized) return @@ -1198,6 +1231,8 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("Inode 0x%016X contains unknown InodeType: 0x%02X", inodeNumberAsUint64, inodeHeadV1.InodeType) } + volumeAsStruct.authToken = volumeAuthToken + globals.Unlock() inodeGETAsJSON, err = json.Marshal(inodeGET) From f1f95045c70cd942989b3a4c38e814a6ff42eee7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 17 Aug 2021 15:14:22 -0700 Subject: [PATCH 063/258] Cleaned up bucketstats in imgr --- go.mod | 2 +- go.sum | 2 ++ imgr/imgrpkg/globals.go | 21 +++++++++------------ imgr/imgrpkg/http-server.go | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 0523f849..e926e9f2 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( go.etcd.io/etcd v3.3.25+incompatible go.uber.org/zap v1.18.1 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e - golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 + golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect diff --git a/go.sum b/go.sum index af8bb0eb..240bbc53 100644 --- a/go.sum +++ b/go.sum @@ -608,6 +608,8 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= +golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 96234847..b6bcd09b 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -81,18 +81,15 @@ type chunkedPutContextStruct struct { } type statsStruct struct { - DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ - GetConfigUsecs bucketstats.BucketLog2Round // GET /config - GetStatsUsecs bucketstats.BucketLog2Round // GET /stats - GetVersionUsecs bucketstats.BucketLog2Round // GET /version - GetVolumeInodeUsecs bucketstats.BucketLog2Round // GET /volume//inode/ - GetVolumeInodeLayoutUsecs bucketstats.BucketLog2Round // GET /volume//inode//layout - GetVolumeInodePayloadUsecs bucketstats.BucketLog2Round // GET /volume//inode//payload - GetVolumeLayoutUsecs bucketstats.BucketLog2Round // GET /volume//layout - GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume - GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ - PostVolumeUsecs bucketstats.BucketLog2Round // POST /volume/ - PutVolumeUsecs bucketstats.BucketLog2Round // PUT /volume/ + DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ + GetConfigUsecs bucketstats.BucketLog2Round // GET /config + GetStatsUsecs bucketstats.BucketLog2Round // GET /stats + GetVersionUsecs bucketstats.BucketLog2Round // GET /version + GetVolumeInodeUsecs bucketstats.BucketLog2Round // GET /volume//inode/ + GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume + GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ + PostVolumeUsecs bucketstats.BucketLog2Round // POST /volume/ + PutVolumeUsecs bucketstats.BucketLog2Round // PUT /volume/ AdjustInodeTableEntryOpenCountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).AdjustInodeTableEntryOpenCount() DeleteInodeTableEntryUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).DeleteInodeTableEntry() diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 07db913a..97519500 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -910,7 +910,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ responseWriter.WriteHeader(http.StatusNotFound) } case 5: - // Form: /volume//inode/ + // Form: /volume//inode/ volumeName = pathSplit[2] mustBeInode = pathSplit[3] From dfaa6c3efbd369d3892e46572ec6c089ab19a62c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 17 Aug 2021 18:30:17 -0700 Subject: [PATCH 064/258] Updated Dockerfile & docker-compose.yml to enable /dev/fuse access ...only applicable to development & iclient services/targets --- Dockerfile | 14 ++++++++++---- docker-compose.yml | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index bbabcbe9..dfc25bc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,13 +47,17 @@ # [-d|--detach] \ # [-it] \ # [--rm] \ +# [--privileged] \ # [--mount src="$(pwd)",target="/src",type=bind] \ # |[:] # # Notes: -# -d|--detach: tells Docker to detach from running container -# -it: tells Docker to run container interactively -# --rm: tells Docker to destroy container upon exit +# -d|--detach: tells Docker to detach from running container +# -it: tells Docker to run container interactively +# --rm: tells Docker to destroy container upon exit +# --privileged: +# 1) tells Docker to, among other things, grant access to /dev/fuse +# 2) only useful for --target development and --target iclient # --mount: # 1) bind mounts the context into /src in the container # 2) /src will be a read-write'able equivalent to the context dir @@ -62,13 +66,14 @@ # 5) /src doesn't exist for --target build FROM alpine:3.14.0 as base -ARG GolangVersion=1.16.7 +ARG GolangVersion=1.17 ARG MakeTarget RUN apk add --no-cache libc6-compat FROM base as dev RUN apk add --no-cache bind-tools \ curl \ + fuse \ gcc \ git \ jq \ @@ -103,6 +108,7 @@ COPY --from=build /src/imgr/imgr.conf ./ FROM imgr as iclient RUN rm caKey.pem cert.pem key.pem imgr imgr.conf +RUN apk add --no-cache fuse COPY --from=build /src/iclient/iclient ./ COPY --from=build /src/iclient/iclient.conf ./ COPY --from=build /src/iclient/iclient.sh ./ diff --git a/docker-compose.yml b/docker-compose.yml index 204f4acc..53e957b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,6 +68,7 @@ services: depends_on: - etcd - swift + privileged: true expose: - 32356 # IMGR.RetryRPCPort - 15346 # IMGR.HTTPServerPort @@ -116,5 +117,6 @@ services: depends_on: - swift - imgr + privileged: true command: ["./iclient.sh", "-s"] \ No newline at end of file From f788262f3dfc4f59e6b9932e5f7f101d2bd4ce21 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 17 Aug 2021 18:46:13 -0700 Subject: [PATCH 065/258] Updated to golang 1.17, picked up cstruct/fission/sortedmap, & fixed Dockerfile/docker-compose.yml to enable /dev/fuse access in dev & iclient --- Dockerfile | 4 ++-- go.mod | 48 +++++++++++++++++++++++++++++++++++++++--------- go.sum | 16 ++++++---------- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index dfc25bc7..91c1b8d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,11 +57,11 @@ # --rm: tells Docker to destroy container upon exit # --privileged: # 1) tells Docker to, among other things, grant access to /dev/fuse -# 2) only useful for --target development and --target iclient +# 2) only useful for --target dev and --target iclient # --mount: # 1) bind mounts the context into /src in the container # 2) /src will be a read-write'able equivalent to the context dir -# 3) only useful for --target development +# 3) only useful for --target dev # 4) /src will be a local copy instead for --target build # 5) /src doesn't exist for --target build diff --git a/go.mod b/go.mod index e926e9f2..05a41b5e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/NVIDIA/proxyfs -go 1.16 +go 1.17 replace github.com/coreos/bbolt v1.3.4 => go.etcd.io/bbolt v1.3.4 @@ -10,9 +10,9 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 - github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d - github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc - github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d + github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 + github.com/NVIDIA/fission v0.0.0-20210818012801-322f6346f124 + github.com/NVIDIA/sortedmap v0.0.0-20210818004215-48fa0266cfdd github.com/ansel1/merry v1.6.1 github.com/coreos/bbolt v1.3.4 // indirect github.com/coreos/etcd v3.3.25+incompatible // indirect @@ -20,7 +20,6 @@ require ( github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 - github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -33,13 +32,9 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kr/pretty v0.2.0 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/pborman/uuid v1.2.1 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/cobra v1.2.1 // indirect github.com/stretchr/testify v1.7.0 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect @@ -53,3 +48,38 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) + +require ( + github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/ory/viper v1.7.5 // indirect + github.com/pborman/uuid v1.2.1 // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cobra v1.2.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/grpc v1.38.0 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) diff --git a/go.sum b/go.sum index 240bbc53..4af92faf 100644 --- a/go.sum +++ b/go.sum @@ -41,14 +41,12 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= -github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d h1:DSX7RxYXf0jevVSL+NbPdaMKNr8YADNcAFYvP1L10JE= -github.com/NVIDIA/cstruct v0.0.0-20210804235153-e02d4f5f012d/go.mod h1:d68dR75IgEqHcodexLfWr6dEFVUs90ApRj5KSYkL59k= -github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc h1:UqHloRP/FbKINAD21ORQRXEkbJcDu9AUbY1AMqbbZmo= -github.com/NVIDIA/fission v0.0.0-20210804235922-f7f250e19ccc/go.mod h1:WCQif2Y+7Y0CtS2kbupILmlNEmFnQRRaJRoDAKhreFg= -github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= -github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d h1:x/cEPpdNatZGZlnQ0lalpfxy7bd7dy9zW/SZKMGn5Ys= -github.com/NVIDIA/sortedmap v0.0.0-20210804235713-3e115b37465d/go.mod h1:94gmSfIoS2wX1UFVNLFTfLaueExrrVVpVVp8QP1iGmw= +github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= +github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= +github.com/NVIDIA/fission v0.0.0-20210818012801-322f6346f124 h1:BpGpcCGFIrP/oEM9USVzt+Sihjv/la/OBM311LF9dyE= +github.com/NVIDIA/fission v0.0.0-20210818012801-322f6346f124/go.mod h1:WWipHJR7/4F9y5SQ4RPTo4PtmNHY/nhAMnJ08Mjlx3A= +github.com/NVIDIA/sortedmap v0.0.0-20210818004215-48fa0266cfdd h1:ny3/NI9XLbOegzBjq4xLGQ8FOCR168rjWpvs7KNxCww= +github.com/NVIDIA/sortedmap v0.0.0-20210818004215-48fa0266cfdd/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -606,8 +604,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From b72e01996aa5d04666afded72af679ec30b7b1f5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 18 Aug 2021 14:22:19 -0700 Subject: [PATCH 066/258] Added B+Tree "dimensions report" to imgr's GET /volume/[/inode/] --- go.mod | 4 +- go.sum | 4 ++ imgr/imgrpkg/http-server.go | 118 +++++++++++++++++++++--------------- 3 files changed, 76 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index 05a41b5e..15098063 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 - github.com/NVIDIA/fission v0.0.0-20210818012801-322f6346f124 - github.com/NVIDIA/sortedmap v0.0.0-20210818004215-48fa0266cfdd + github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab + github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18 github.com/ansel1/merry v1.6.1 github.com/coreos/bbolt v1.3.4 // indirect github.com/coreos/etcd v3.3.25+incompatible // indirect diff --git a/go.sum b/go.sum index 4af92faf..5e001b4a 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,12 @@ github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1Dnxq github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= github.com/NVIDIA/fission v0.0.0-20210818012801-322f6346f124 h1:BpGpcCGFIrP/oEM9USVzt+Sihjv/la/OBM311LF9dyE= github.com/NVIDIA/fission v0.0.0-20210818012801-322f6346f124/go.mod h1:WWipHJR7/4F9y5SQ4RPTo4PtmNHY/nhAMnJ08Mjlx3A= +github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab h1:/t1tCaj3eWQXCRIme+JzSy+p78OPDjzNRirWAisWuVc= +github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab/go.mod h1:cDh6DCOixUxCkPkyqNroWeSSSZpryqnmFKIylvc6aSE= github.com/NVIDIA/sortedmap v0.0.0-20210818004215-48fa0266cfdd h1:ny3/NI9XLbOegzBjq4xLGQ8FOCR168rjWpvs7KNxCww= github.com/NVIDIA/sortedmap v0.0.0-20210818004215-48fa0266cfdd/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= +github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18 h1:yV4uip6oWf2Z9wkz7azNPeeVAVQDqR6a52HPotwjMtg= +github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 97519500..bcf692c7 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -318,7 +318,14 @@ type volumeListGETEntryStruct struct { AuthTokenExpiredMounts uint64 } -type volumeGETInodeTableLayoutEntryStruct struct { +type volumeOrInodeGETDimensionStruct struct { + MinKeysPerNode uint64 // only applies to non-Root nodes + MaxKeysPerNode uint64 + Items uint64 + Height uint64 +} + +type volumeOrInodeGETLayoutEntryStruct struct { ObjectName string // == ilayout.GetObjectNameAsString(ObjectNumber) ObjectSize uint64 BytesReferenced uint64 @@ -346,7 +353,11 @@ type volumeGETStruct struct { SuperBlockObjectName string // == ilayout.GetObjectNameAsString(SuperBlockObjectNumber) SuperBlockLength uint64 ReservedToNonce uint64 - InodeTableLayout []volumeGETInodeTableLayoutEntryStruct + InodeTableMinInodesPerNode uint64 + InodeTableMaxInodesPerNode uint64 + InodeTableInodeCount uint64 + InodeTableHeight uint64 + InodeTableLayout []volumeOrInodeGETLayoutEntryStruct InodeObjectCount uint64 InodeObjectSize uint64 InodeBytesReferenced uint64 @@ -377,12 +388,6 @@ type fileInodeGETPayloadEntryStruct struct { ObjectOffset uint64 } -type inodeGETLayoutEntryStruct struct { - ObjectName string // == ilayout.GetObjectNameAsString(ObjectNumber) - ObjectSize uint64 - BytesReferenced uint64 -} - type inodeGETStruct struct { InodeNumber uint64 InodeType string // == "Dir", "File", or "Symlink" @@ -403,8 +408,12 @@ type httpServerDirectoryWrapperStruct struct { type dirInodeGETStruct struct { inodeGETStruct - Payload []dirInodeGETPayloadEntryStruct - Layout []inodeGETLayoutEntryStruct + MinDirEntriesPerNode uint64 + MaxDirEntriesPerNode uint64 + DirEntryCount uint64 + DirectoryHeight uint64 + Payload []dirInodeGETPayloadEntryStruct + Layout []volumeOrInodeGETLayoutEntryStruct } type httpServerExtentMapWrapperStruct struct { @@ -415,9 +424,13 @@ type httpServerExtentMapWrapperStruct struct { type fileInodeGETStruct struct { inodeGETStruct - Size uint64 - Payload []fileInodeGETPayloadEntryStruct - Layout []inodeGETLayoutEntryStruct + Size uint64 + MinExtentsPerNode uint64 + MaxExtentsPerNode uint64 + ExtentCount uint64 + ExtentMapHeight uint64 + Payload []fileInodeGETPayloadEntryStruct + Layout []volumeOrInodeGETLayoutEntryStruct } type symLinkInodeGETStruct struct { @@ -662,18 +675,17 @@ func (extentMapWrapper *httpServerExtentMapWrapperStruct) UnpackValue(payloadDat func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( checkPointV1 *ilayout.CheckPointV1Struct + dimensionsReport sortedmap.DimensionsReport directoryEntryIndex int directoryEntryKeyV1AsKey sortedmap.Key directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct directoryEntryValueV1AsValue sortedmap.Value - directoryLen int directoryWrapper *httpServerDirectoryWrapperStruct err error extentMapEntryIndex int extentMapEntryKeyV1AsKey sortedmap.Key extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct extentMapEntryValueV1AsValue sortedmap.Value - extentMapLen int extentMapWrapper *httpServerExtentMapWrapperStruct inodeGET interface{} inodeGETAsJSON []byte @@ -690,7 +702,6 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ inodeTableEntryValueV1AsValue sortedmap.Value inodeTableIndex int inodeTableLayoutIndex int - inodeTableLen int inodeTableWrapper *httpServerInodeTableWrapperStruct mustBeInode string ok bool @@ -815,6 +826,27 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ return } + inodeTableWrapper = &httpServerInodeTableWrapperStruct{ + volume: volumeAsStruct, + cache: sortedmap.NewBPlusTreeCache(httpServerInodeTableCacheLowLimit, httpServerInodeTableCacheHighLimit), + } + + inodeTableWrapper.table, err = sortedmap.OldBPlusTree(superBlockV1.InodeTableRootObjectNumber, superBlockV1.InodeTableRootObjectOffset, superBlockV1.InodeTableRootObjectLength, sortedmap.CompareUint64, inodeTableWrapper, inodeTableWrapper.cache) + if nil != err { + volumeAsStruct.authToken = volumeAuthToken + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + dimensionsReport, err = inodeTableWrapper.table.FetchDimensionsReport() + if nil != err { + volumeAsStruct.authToken = volumeAuthToken + globals.Unlock() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + volumeGET = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, @@ -825,11 +857,16 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ SuperBlockObjectName: ilayout.GetObjectNameAsString(checkPointV1.SuperBlockObjectNumber), SuperBlockLength: checkPointV1.SuperBlockLength, ReservedToNonce: checkPointV1.ReservedToNonce, - InodeTableLayout: make([]volumeGETInodeTableLayoutEntryStruct, len(superBlockV1.InodeTableLayout)), + InodeTableMinInodesPerNode: dimensionsReport.MinKeysPerNode, + InodeTableMaxInodesPerNode: dimensionsReport.MaxKeysPerNode, + InodeTableInodeCount: dimensionsReport.Items, + InodeTableHeight: dimensionsReport.Height, + InodeTableLayout: make([]volumeOrInodeGETLayoutEntryStruct, len(superBlockV1.InodeTableLayout)), InodeObjectCount: superBlockV1.InodeObjectCount, InodeObjectSize: superBlockV1.InodeObjectSize, InodeBytesReferenced: superBlockV1.InodeBytesReferenced, PendingDeleteObjectNameArray: make([]string, len(superBlockV1.PendingDeleteObjectNumberArray)), + InodeTable: make([]volumeGETInodeTableEntryStruct, dimensionsReport.Items), } for inodeTableLayoutIndex = range volumeGET.InodeTableLayout { @@ -842,29 +879,6 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeGET.PendingDeleteObjectNameArray[pendingDeleteObjectNameArrayIndex] = ilayout.GetObjectNameAsString(superBlockV1.PendingDeleteObjectNumberArray[pendingDeleteObjectNameArrayIndex]) } - inodeTableWrapper = &httpServerInodeTableWrapperStruct{ - volume: volumeAsStruct, - cache: sortedmap.NewBPlusTreeCache(httpServerInodeTableCacheLowLimit, httpServerInodeTableCacheHighLimit), - } - - inodeTableWrapper.table, err = sortedmap.OldBPlusTree(superBlockV1.InodeTableRootObjectNumber, superBlockV1.InodeTableRootObjectOffset, superBlockV1.InodeTableRootObjectLength, sortedmap.CompareUint64, inodeTableWrapper, inodeTableWrapper.cache) - if nil != err { - volumeAsStruct.authToken = volumeAuthToken - globals.Unlock() - responseWriter.WriteHeader(http.StatusUnauthorized) - return - } - - inodeTableLen, err = inodeTableWrapper.table.Len() - if nil != err { - volumeAsStruct.authToken = volumeAuthToken - globals.Unlock() - responseWriter.WriteHeader(http.StatusUnauthorized) - return - } - - volumeGET.InodeTable = make([]volumeGETInodeTableEntryStruct, inodeTableLen) - for inodeTableIndex = range volumeGET.InodeTable { inodeNumberAsKey, inodeTableEntryValueV1AsValue, ok, err = inodeTableWrapper.table.GetByIndex(inodeTableIndex) if (nil != err) || !ok { @@ -1022,7 +1036,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ return } - directoryLen, err = directoryWrapper.table.Len() + dimensionsReport, err = directoryWrapper.table.FetchDimensionsReport() if nil != err { volumeAsStruct.authToken = volumeAuthToken globals.Unlock() @@ -1042,8 +1056,12 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ GroupID: inodeHeadV1.GroupID, StreamTable: make([]inodeGETStreamTableEntryStruct, len(inodeHeadV1.StreamTable)), }, - make([]dirInodeGETPayloadEntryStruct, directoryLen), - make([]inodeGETLayoutEntryStruct, len(inodeHeadV1.Layout)), + dimensionsReport.MinKeysPerNode, + dimensionsReport.MaxKeysPerNode, + dimensionsReport.Items, + dimensionsReport.Height, + make([]dirInodeGETPayloadEntryStruct, dimensionsReport.Items), + make([]volumeOrInodeGETLayoutEntryStruct, len(inodeHeadV1.Layout)), } for inodeGETLinkTableEntryIndex = range inodeHeadV1.LinkTable { @@ -1066,7 +1084,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ } } - for directoryEntryIndex = 0; directoryEntryIndex < directoryLen; directoryEntryIndex++ { + for directoryEntryIndex = range inodeGET.(*dirInodeGETStruct).Payload { directoryEntryKeyV1AsKey, directoryEntryValueV1AsValue, ok, err = directoryWrapper.table.GetByIndex(directoryEntryIndex) if (nil != err) || !ok { volumeAsStruct.authToken = volumeAuthToken @@ -1118,7 +1136,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ return } - extentMapLen, err = extentMapWrapper.table.Len() + dimensionsReport, err = extentMapWrapper.table.FetchDimensionsReport() if nil != err { volumeAsStruct.authToken = volumeAuthToken globals.Unlock() @@ -1139,8 +1157,12 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ StreamTable: make([]inodeGETStreamTableEntryStruct, len(inodeHeadV1.StreamTable)), }, inodeHeadV1.Size, - make([]fileInodeGETPayloadEntryStruct, extentMapLen), - make([]inodeGETLayoutEntryStruct, len(inodeHeadV1.Layout)), + dimensionsReport.MinKeysPerNode, + dimensionsReport.MaxKeysPerNode, + dimensionsReport.Items, + dimensionsReport.Height, + make([]fileInodeGETPayloadEntryStruct, dimensionsReport.Items), + make([]volumeOrInodeGETLayoutEntryStruct, len(inodeHeadV1.Layout)), } for inodeGETLinkTableEntryIndex = range inodeHeadV1.LinkTable { @@ -1163,7 +1185,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ } } - for extentMapEntryIndex = 0; extentMapEntryIndex < extentMapLen; extentMapEntryIndex++ { + for extentMapEntryIndex = range inodeGET.(*fileInodeGETStruct).Payload { extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = extentMapWrapper.table.GetByIndex(extentMapEntryIndex) if (nil != err) || !ok { volumeAsStruct.authToken = volumeAuthToken From fd1a6abce095e0bd5e6d3d890d23afb497891456 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 18 Aug 2021 18:27:30 -0700 Subject: [PATCH 067/258] Fixed imgrpkg's TestHTTPServer() to properly validate GET /volume//inode/ response --- imgr/imgrpkg/http-server_test.go | 52 +++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index f4f9bc62..79c45aec 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -8,6 +8,7 @@ import ( "net/http" "strings" "testing" + "time" "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/version" @@ -15,12 +16,17 @@ import ( func TestHTTPServer(t *testing.T) { var ( - err error - getRequestHeaders http.Header - postRequestBody string - putRequestBody string - responseBody []byte - responseBodyExpected string + err error + getRequestHeaders http.Header + postRequestBody string + putRequestBody string + responseBody []byte + responseBodyAsString string + responseBodyAsStringSplit []string + responseBodyExpected string + responseBodyExpectedStaticPrefix string + responseBodyExpectedStaticMiddle string + responseBodyExpectedStaticSuffix string ) testSetup(t, nil) @@ -120,7 +126,7 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) [case 2] failed: %v", err) } - responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"SuperBlockObjectName\":\"3000000000000000\",\"SuperBlockLength\":96,\"ReservedToNonce\":3,\"InodeTableLayout\":[{\"ObjectName\":\"3000000000000000\",\"ObjectSize\":58,\"BytesReferenced\":58}],\"InodeObjectCount\":1,\"InodeObjectSize\":237,\"InodeBytesReferenced\":237,\"PendingDeleteObjectNameArray\":[],\"InodeTable\":[{\"InodeNumber\":1,\"InodeHeadObjectName\":\"2000000000000000\",\"InodeHeadLength\":174}]}", testVolume, testGlobals.containerURL, testGlobals.authToken) + responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0,\"SuperBlockObjectName\":\"3000000000000000\",\"SuperBlockLength\":96,\"ReservedToNonce\":3,\"InodeTableMinInodesPerNode\":1024,\"InodeTableMaxInodesPerNode\":2048,\"InodeTableInodeCount\":1,\"InodeTableHeight\":1,\"InodeTableLayout\":[{\"ObjectName\":\"3000000000000000\",\"ObjectSize\":58,\"BytesReferenced\":58}],\"InodeObjectCount\":1,\"InodeObjectSize\":237,\"InodeBytesReferenced\":237,\"PendingDeleteObjectNameArray\":[],\"InodeTable\":[{\"InodeNumber\":1,\"InodeHeadObjectName\":\"2000000000000000\",\"InodeHeadLength\":174}]}", testVolume, testGlobals.containerURL, testGlobals.authToken) _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) if nil != err { @@ -130,6 +136,38 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("GET /volume/%s [case 2] returned unexpected responseBody: \"%s\"", testVolume, responseBody) } + responseBodyExpectedStaticPrefix = "{\"InodeNumber\":1,\"InodeType\":\"Dir\",\"LinkTable\":[{\"ParentDirInodeNumber\":1,\"ParentDirEntryName\":\".\"},{\"ParentDirInodeNumber\":1,\"ParentDirEntryName\":\"..\"}],\"ModificationTime\":\"" + // Next segment to be a quoted ModificationTime.Format(time.RFC3339) + responseBodyExpectedStaticMiddle = "\",\"StatusChangeTime\":\"" + // Next segment to be a quoted StatusChangeTime.Format(time.RFC3339) + responseBodyExpectedStaticSuffix = "\",\"Mode\":511,\"UserID\":0,\"GroupID\":0,\"StreamTable\":[],\"MinDirEntriesPerNode\":512,\"MaxDirEntriesPerNode\":1024,\"DirEntryCount\":2,\"DirectoryHeight\":1,\"Payload\":[{\"BaseName\":\".\",\"InodeNumber\":1,\"InodeType\":\"Dir\"},{\"BaseName\":\"..\",\"InodeNumber\":1,\"InodeType\":\"Dir\"}],\"Layout\":[{\"ObjectName\":\"2000000000000000\",\"ObjectSize\":63,\"BytesReferenced\":63}]}" + + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume+"/inode/1", nil, nil) + if nil != err { + t.Fatalf("GET /volume/%s/inode/1 failed: %v", testVolume, err) + } + responseBodyAsString = string(responseBody[:]) + if !strings.HasPrefix(responseBodyAsString, responseBodyExpectedStaticPrefix) { + t.Fatalf("GET /volume/%s/inode/1 returned unexpected responseBody [case 1]: \"%s\"", testVolume, responseBody) + } + if !strings.HasSuffix(responseBodyAsString, responseBodyExpectedStaticSuffix) { + t.Fatalf("GET /volume/%s/inode/1 returned unexpected responseBody [case 2]: \"%s\"", testVolume, responseBody) + } + responseBodyAsString = strings.TrimPrefix(responseBodyAsString, responseBodyExpectedStaticPrefix) + responseBodyAsString = strings.TrimSuffix(responseBodyAsString, responseBodyExpectedStaticSuffix) + responseBodyAsStringSplit = strings.Split(responseBodyAsString, responseBodyExpectedStaticMiddle) + if len(responseBodyAsStringSplit) != 2 { + t.Fatalf("GET /volume/%s/inode/1 returned unexpected responseBody [case 3]: \"%s\"", testVolume, responseBody) + } + _, err = time.Parse(time.RFC3339, responseBodyAsStringSplit[0]) + if nil != err { + t.Fatalf("GET /volume/%s/inode/1 returned unexpected responseBody [case 4]: \"%s\"", testVolume, responseBody) + } + _, err = time.Parse(time.RFC3339, responseBodyAsStringSplit[1]) + if nil != err { + t.Fatalf("GET /volume/%s/inode/1 returned unexpected responseBody [case 5]: \"%s\"", testVolume, responseBody) + } + responseBodyExpected = fmt.Sprintf("[{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"AuthToken\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0}]", testVolume, testGlobals.containerURL, testGlobals.authToken) _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) From 28a52b88d5b3fc6ee882e89cf76d360d46b36906 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 18 Aug 2021 19:12:29 -0700 Subject: [PATCH 068/258] go.{mod|sum} updated Attempted resolution of CVE against jwt-go v3.2.0 resolved in v3.3.0 apparently Intention was primarily to upgrade to etcd v3.5.0... which was unsuccessful --- go.mod | 57 ++++++++++++++++++++++++++++----------------------------- go.sum | 38 +++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 15098063..60417a46 100644 --- a/go.mod +++ b/go.mod @@ -14,52 +14,41 @@ require ( github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18 github.com/ansel1/merry v1.6.1 - github.com/coreos/bbolt v1.3.4 // indirect - github.com/coreos/etcd v3.3.25+incompatible // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.0.1 - github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/jonboulle/clockwork v0.2.2 // indirect - github.com/kr/pretty v0.2.0 // indirect - github.com/prometheus/client_golang v1.11.0 // indirect github.com/sirupsen/logrus v1.8.1 - github.com/soheilhy/cmux v0.1.5 // indirect github.com/stretchr/testify v1.7.0 - github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/etcd v3.3.25+incompatible - go.uber.org/zap v1.18.1 // indirect - golang.org/x/net v0.0.0-20210614182718-04defd469f4e - golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect - google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect + golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d + golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7 ) require ( github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/coreos/bbolt v1.3.4 // indirect + github.com/coreos/etcd v3.3.25+incompatible // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/fsnotify/fsnotify v1.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/ory/go-acc v0.2.6 // indirect @@ -68,18 +57,28 @@ require ( github.com/pelletier/go-toml v1.9.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.11.0 // indirect + github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/cobra v1.2.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.0 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect google.golang.org/grpc v1.38.0 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index 5e001b4a..c7b7ade5 100644 --- a/go.sum +++ b/go.sum @@ -43,12 +43,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= -github.com/NVIDIA/fission v0.0.0-20210818012801-322f6346f124 h1:BpGpcCGFIrP/oEM9USVzt+Sihjv/la/OBM311LF9dyE= -github.com/NVIDIA/fission v0.0.0-20210818012801-322f6346f124/go.mod h1:WWipHJR7/4F9y5SQ4RPTo4PtmNHY/nhAMnJ08Mjlx3A= github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab h1:/t1tCaj3eWQXCRIme+JzSy+p78OPDjzNRirWAisWuVc= github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab/go.mod h1:cDh6DCOixUxCkPkyqNroWeSSSZpryqnmFKIylvc6aSE= -github.com/NVIDIA/sortedmap v0.0.0-20210818004215-48fa0266cfdd h1:ny3/NI9XLbOegzBjq4xLGQ8FOCR168rjWpvs7KNxCww= -github.com/NVIDIA/sortedmap v0.0.0-20210818004215-48fa0266cfdd/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18 h1:yV4uip6oWf2Z9wkz7azNPeeVAVQDqR6a52HPotwjMtg= github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -120,8 +116,9 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.0 h1:NO5hkcB+srp1x6QmwvNZLeaOgbM8cmBTN32THzjvu2k= +github.com/fsnotify/fsnotify v1.5.0/go.mod h1:BX0DCEr5pT4jm2CnQdVP1lFV521fcCNcyEeNp4DQQDk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= @@ -140,6 +137,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -170,10 +168,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -272,8 +270,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= @@ -442,8 +440,8 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.18.1 h1:CSUJ2mjFszzEWt4CdKISEuChVIXGBn3lAPwkRGyVrc4= -go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -471,6 +469,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -529,8 +528,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -608,8 +607,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7 h1:/bmDWM82ZX7TawqxuI8kVjKI0TXHdSY6pHJArewwHtU= +golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -618,14 +617,13 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -676,6 +674,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= @@ -751,12 +750,12 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 h1:bXUwz2WkXXrXgiLxww3vWmoSHLOGv4ipdPdTvKymcKw= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -770,8 +769,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -791,6 +790,7 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= From 97896e990366e6b4d35f3dcd5f413f34d1ffd4c9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 19 Aug 2021 07:32:04 -0700 Subject: [PATCH 069/258] Attempts to update go.{mod|sum} to latest --- go.mod | 44 +++------ go.sum | 275 ++++++--------------------------------------------------- 2 files changed, 40 insertions(+), 279 deletions(-) diff --git a/go.mod b/go.mod index 60417a46..29e0458d 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,9 @@ module github.com/NVIDIA/proxyfs go 1.17 -replace github.com/coreos/bbolt v1.3.4 => go.etcd.io/bbolt v1.3.4 +replace github.com/coreos/bbolt => go.etcd.io/bbolt v1.3.6 -replace go.etcd.io/bbolt v1.3.4 => github.com/coreos/bbolt v1.3.4 +replace go.etcd.io/bbolt v1.3.6 => github.com/coreos/bbolt v1.3.6 replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 @@ -20,25 +20,20 @@ require ( github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7 + golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c ) require ( github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect - github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/coreos/bbolt v1.3.4 // indirect + github.com/coreos/bbolt v1.3.6 // indirect github.com/coreos/etcd v3.3.25+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fsnotify/fsnotify v1.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect @@ -46,38 +41,25 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/ory/viper v1.7.5 // indirect - github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml v1.9.3 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect + github.com/prometheus/common v0.30.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/cobra v1.2.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.19.0 // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect - google.golang.org/grpc v1.38.0 // indirect - google.golang.org/protobuf v1.26.0 // indirect + google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f // indirect + google.golang.org/grpc v1.40.0 // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/go.sum b/go.sum index c7b7ade5..e04e1855 100644 --- a/go.sum +++ b/go.sum @@ -14,11 +14,6 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -27,7 +22,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -47,7 +41,6 @@ github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab h1:/t1tCaj3eWQXCRIm github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab/go.mod h1:cDh6DCOixUxCkPkyqNroWeSSSZpryqnmFKIylvc6aSE= github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18 h1:yV4uip6oWf2Z9wkz7azNPeeVAVQDqR6a52HPotwjMtg= github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -61,64 +54,39 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.0 h1:NO5hkcB+srp1x6QmwvNZLeaOgbM8cmBTN32THzjvu2k= -github.com/fsnotify/fsnotify v1.5.0/go.mod h1:BX0DCEr5pT4jm2CnQdVP1lFV521fcCNcyEeNp4DQQDk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= @@ -132,16 +100,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= -github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -154,7 +117,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -168,10 +130,9 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -184,15 +145,12 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -200,55 +158,22 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -258,45 +183,24 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -305,30 +209,14 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= -github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= -github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= -github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -338,120 +226,76 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -469,12 +313,10 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -485,21 +327,15 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -520,14 +356,10 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -535,13 +367,7 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -552,10 +378,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -567,7 +390,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -575,7 +397,6 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -586,51 +407,41 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7 h1:/bmDWM82ZX7TawqxuI8kVjKI0TXHdSY6pHJArewwHtU= -golang.org/x/sys v0.0.0-20210818153620-00dd8d7831e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c h1:Lyn7+CqXIiC+LOR9aHD6jDK+hPcmAuCfuXztd1v4w1Q= +golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -641,7 +452,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -669,16 +479,9 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -701,19 +504,11 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -744,19 +539,9 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 h1:bXUwz2WkXXrXgiLxww3vWmoSHLOGv4ipdPdTvKymcKw= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f h1:enWPderunHptc5pzJkSYGx0olpF8goXzG0rY3kL0eSg= +google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -769,28 +554,22 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= From 5e9b23702a9545b555628188be7d2711fc63eec3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 19 Aug 2021 10:04:27 -0700 Subject: [PATCH 070/258] Integrated GET /volume/ HTML --- imgr/imgrpkg/html_templates.go | 348 ++++++++++--------------- imgr/imgrpkg/http-server.go | 29 ++- imgr/imgrpkg/static-content/styles.css | 48 ++-- 3 files changed, 195 insertions(+), 230 deletions(-) diff --git a/imgr/imgrpkg/html_templates.go b/imgr/imgrpkg/html_templates.go index 29f31dd6..881c42e7 100644 --- a/imgr/imgrpkg/html_templates.go +++ b/imgr/imgrpkg/html_templates.go @@ -4,6 +4,7 @@ package imgrpkg // To use: fmt.Sprintf(indexDotHTMLTemplate, proxyfsVersion) +// %[1]v const indexDotHTMLTemplate string = ` @@ -92,6 +93,7 @@ const indexDotHTMLTemplate string = ` ` // To use: fmt.Sprintf(configTemplate, proxyfsVersion, confMapJSONString) +// %[1]v %[2]v const configTemplate string = ` @@ -150,6 +152,7 @@ const configTemplate string = ` ` // To use: fmt.Sprintf(volumeListTopTemplate, proxyfsVersion) +// %[1]v const volumeListTopTemplate string = ` @@ -206,6 +209,7 @@ const volumeListTopTemplate string = ` ` // To use: fmt.Sprintf(volumeListPerVolumeTemplate, volumeName) +// %[1]v const volumeListPerVolumeTemplate string = ` %[1]v SnapShots @@ -226,15 +230,17 @@ const volumeListBottom string = ` ` -// To use: fmt.Sprintf(layoutReportTopTemplate, proxyfsVersion, volumeName) -const layoutReportTopTemplate string = ` +// To use: fmt.Sprintf(volumeTemplate, proxyfsVersion, volumeName, volumeJSONString) +// %[1]v %[2]v %[3]v +const volumeTemplate string = ` - Layout Report %[2]v + + Volume %[2]v

- -
- -

- Layout Report - %[2]v -

- Show validated version

-` - -// To use: fmt.Sprintf(layoutReportTableTopTemplate, TreeName, NumDiscrepencies, {"success"|"danger"}) -const layoutReportTableTopTemplate string = `
-
-

%[1]v

%[2]v discrepancies

-
- - - - - - - - -` - -// To use: fmt.Sprintf(layoutReportTableRowTemplate, ObjectName, ObjectBytes) -const layoutReportTableRowTemplate string = ` - - - -` - -const layoutReportTableBottom string = ` -
ObjectNameObjectBytes
%016[1]X
%[2]v
-` - -const layoutReportBottom string = `
- - - - - - -` - -// To use: fmt.Sprintf(extentMapTemplate, proxyfsVersion, volumeName, extentMapJSONString, pathDoubleQuotedString, serverErrorBoolString) -const extentMapTemplate string = ` - - - - - - - Extent Map %[2]v - - -

- Extent Map + Volume %[2]v

- +
+
-
-
- -
- -
-
-
- -
- - - - - - - - - - -
File OffsetContainer/ObjectObject OffsetLength
+ + +
+
+ +
- + diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index bcf692c7..6726a7c5 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -674,6 +674,7 @@ func (extentMapWrapper *httpServerExtentMapWrapperStruct) UnpackValue(payloadDat func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( + acceptHeader string checkPointV1 *ilayout.CheckPointV1Struct dimensionsReport sortedmap.DimensionsReport directoryEntryIndex int @@ -714,6 +715,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeAsValue sortedmap.Value volumeAuthToken string volumeGET *volumeGETStruct + volumeGETAsHTML []byte volumeGETAsJSON []byte volumeListGET []*volumeListGETEntryStruct volumeListGETIndex int @@ -911,13 +913,28 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatal(err) } - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeGETAsJSON))) - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) + acceptHeader = request.Header.Get("Accept") - _, err = responseWriter.Write(volumeGETAsJSON) - if nil != err { - logWarnf("responseWriter.Write(volumeGETAsJSON) failed: %v", err) + if strings.Contains(acceptHeader, "text/html") { + volumeGETAsHTML = []byte(fmt.Sprintf(volumeTemplate, version.ProxyFSVersion, volumeGET.Name, string(volumeGETAsJSON))) + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeGETAsHTML))) + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write(volumeGETAsHTML) + if nil != err { + logWarnf("responseWriter.Write(volumeGETAsHTML) failed: %v", err) + } + } else { + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(volumeGETAsJSON))) + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write(volumeGETAsJSON) + if nil != err { + logWarnf("responseWriter.Write(volumeGETAsJSON) failed: %v", err) + } } } else { globals.Unlock() diff --git a/imgr/imgrpkg/static-content/styles.css b/imgr/imgrpkg/static-content/styles.css index fedfd863..b39be2f0 100644 --- a/imgr/imgrpkg/static-content/styles.css +++ b/imgr/imgrpkg/static-content/styles.css @@ -3,7 +3,7 @@ SPDX-License-Identifier: Apache-2.0 */ -.table td.fit, +.table td.fit, .table th.fit { white-space: nowrap; width: 1%; @@ -27,45 +27,59 @@ span.jstExpand, span.jstFold { cursor: pointer; } +#btn-back-to-top { + position: fixed; + bottom: 20px; + right: 20px; + display: none; +} + +/* jsontree.css */ + .jstValue { - white-space: pre-wrap; + white-space: pre-wrap; + /*font-size: 10px; + font-weight: 400; + font-family: "Lucida Console", Monaco, monospace;*/ } .jstComma { - white-space: pre-wrap; + white-space: pre-wrap; } .jstProperty { - color: #666; - word-wrap: break-word; + color: #666; + word-wrap: break-word; } .jstBracket { - white-space: pre-wrap;; + white-space: pre-wrap;; } .jstBool { - color: #2525CC; + color: #2525CC; } .jstNum { - color: #D036D0; + color: #D036D0; } .jstNull { - color: gray; + color: gray; } .jstStr { - color: #2DB669; + color: #2DB669; } .jstFold:after { - content: ' -'; - cursor: pointer; + content: ' -'; + cursor: pointer; } .jstExpand { - white-space: normal; + white-space: normal; } .jstExpand:after { - content: ' +'; - cursor: pointer; + content: ' +'; + cursor: pointer; } .jstFolded { - white-space: normal !important; + white-space: normal !important; } .jstHiddenBlock { - display: none; + display: none; } + +/* End of jsontree.css */ From 92d630bd5a1baba43c7e0c3664948908959ad40f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 19 Aug 2021 10:54:31 -0700 Subject: [PATCH 071/258] Updated imgrpkg's html_templates.go file to prepare for remaining 2 HTML pages ...volumeList & inode --- go.mod | 21 +++ go.sum | 231 +++++++++++++++++++++++++++++++++ imgr/imgrpkg/html_templates.go | 99 ++++++++++---- 3 files changed, 328 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 29e0458d..28555fbd 100644 --- a/go.mod +++ b/go.mod @@ -25,15 +25,20 @@ require ( require ( github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/coreos/bbolt v1.3.6 // indirect github.com/coreos/etcd v3.3.25+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/fsnotify/fsnotify v1.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect @@ -41,12 +46,27 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/ory/viper v1.7.5 // indirect + github.com/pborman/uuid v1.2.1 // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/common v0.30.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cobra v1.2.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.uber.org/atomic v1.9.0 // indirect @@ -60,6 +80,7 @@ require ( google.golang.org/grpc v1.40.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/go.sum b/go.sum index e04e1855..2c2afebe 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,11 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -22,6 +27,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -41,6 +47,7 @@ github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab h1:/t1tCaj3eWQXCRIm github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab/go.mod h1:cDh6DCOixUxCkPkyqNroWeSSSZpryqnmFKIylvc6aSE= github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18 h1:yV4uip6oWf2Z9wkz7azNPeeVAVQDqR6a52HPotwjMtg= github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -54,39 +61,63 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.0 h1:NO5hkcB+srp1x6QmwvNZLeaOgbM8cmBTN32THzjvu2k= +github.com/fsnotify/fsnotify v1.5.0/go.mod h1:BX0DCEr5pT4jm2CnQdVP1lFV521fcCNcyEeNp4DQQDk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= @@ -100,11 +131,16 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= +github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -117,6 +153,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -132,6 +169,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -145,12 +183,15 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -158,22 +199,55 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -183,24 +257,46 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -209,14 +305,30 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -226,55 +338,100 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -286,13 +443,17 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -317,6 +478,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -327,15 +489,21 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -356,8 +524,13 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= @@ -367,6 +540,13 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -378,7 +558,10 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -390,6 +573,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -397,6 +581,7 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -407,14 +592,24 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -428,6 +623,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= @@ -437,11 +633,13 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -452,6 +650,7 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -479,7 +678,14 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -504,11 +710,19 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -539,7 +753,18 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f h1:enWPderunHptc5pzJkSYGx0olpF8goXzG0rY3kL0eSg= google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -564,6 +789,12 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/imgr/imgrpkg/html_templates.go b/imgr/imgrpkg/html_templates.go index 881c42e7..e635dfb2 100644 --- a/imgr/imgrpkg/html_templates.go +++ b/imgr/imgrpkg/html_templates.go @@ -151,9 +151,9 @@ const configTemplate string = ` ` -// To use: fmt.Sprintf(volumeListTopTemplate, proxyfsVersion) -// %[1]v -const volumeListTopTemplate string = ` +// To use: fmt.Sprintf(volumeListTemplate, proxyfsVersion, volumeListJSONString) +// %[1]v %[2]v +const volumeListTemplate string = ` @@ -172,14 +172,14 @@ const volumeListTopTemplate string = ` - - Version %[1]v @@ -206,21 +206,8 @@ const volumeListTopTemplate string = ` -` - -// To use: fmt.Sprintf(volumeListPerVolumeTemplate, volumeName) -// %[1]v -const volumeListPerVolumeTemplate string = ` - %[1]v - SnapShots - FSCK jobs - SCRUB jobs - Layout Report - Extent Map - -` - -const volumeListBottom string = ` + +
@@ -323,7 +310,7 @@ const volumeTemplate string = ` if (complex_keys.includes(key)) { table_markup += " \n"; table_markup += " " + key + "\n"; - table_markup += " Go to " + key + " table\n"; + table_markup += " Go to " + key + "\n"; table_markup += " \n"; } else if (Array.isArray(data[key]) && data[key].length === 0) { table_markup += " \n"; @@ -434,3 +421,69 @@ const volumeTemplate string = ` ` + +// To use: fmt.Sprintf(inodeTemplate, proxyfsVersion, volumeName, inodeNumber, inodeJSONString) +// %[1]v %[2]v %[3]v %[4]v +const inodeTemplate string = ` + + + + + + + Volumes + + + +
+ + +

Volumes

+ + + + + + + + + + + + + + +
Volume Name     
+
+ + + + + +` From 9c47e5a930229a3a58ef1562d44d69456fe82098 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 19 Aug 2021 16:13:00 -0700 Subject: [PATCH 072/258] Integrated GET /volume//inode/ HTML --- imgr/imgrpkg/html_templates.go | 193 +++++++++++++++++++++++++++++---- imgr/imgrpkg/http-server.go | 30 +++-- 2 files changed, 192 insertions(+), 31 deletions(-) diff --git a/imgr/imgrpkg/html_templates.go b/imgr/imgrpkg/html_templates.go index e635dfb2..2e0f7d55 100644 --- a/imgr/imgrpkg/html_templates.go +++ b/imgr/imgrpkg/html_templates.go @@ -173,13 +173,13 @@ const volumeListTemplate string = ` Home Version %[1]v @@ -240,13 +240,13 @@ const volumeTemplate string = ` Home Version %[1]v @@ -431,7 +431,8 @@ const inodeTemplate string = ` - Volumes + + Inode %[3]v | Volume %[2]v +
+ +

+ ProxyFS iclient +

+
+
+
+
Configuration parameters
+

Diplays a JSON representation of the active configuration.

+
+ +
+
+
+
+
Stats
+

Displays current statistics.

+
+ +
+
+
+
+
+
+
+
+ + + + + +` + +// To use: fmt.Sprintf(configTemplate, proxyfsVersion, confMapJSONString) +// %[1]v %[2]v +const configTemplate string = ` + + + + + + + Config + + + +
+ +

+ Config +

+

+    
+ + + + + + + +` diff --git a/iclient/iclientpkg/http-server.go b/iclient/iclientpkg/http-server.go new file mode 100644 index 00000000..8f2880b6 --- /dev/null +++ b/iclient/iclientpkg/http-server.go @@ -0,0 +1,252 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/version" +) + +const ( + startHTTPServerUpCheckDelay = 100 * time.Millisecond + startHTTPServerUpCheckMaxRetries = 10 +) + +func startHTTPServer() (err error) { + var ( + ipAddrTCPPort string + startHTTPServerUpCheckRetries uint32 + ) + + ipAddrTCPPort = net.JoinHostPort(globals.config.HTTPServerIPAddr, strconv.Itoa(int(globals.config.HTTPServerPort))) + + globals.httpServer = &http.Server{ + Addr: ipAddrTCPPort, + Handler: &globals, + } + + globals.httpServerWG.Add(1) + + go func() { + var ( + err error + ) + + err = globals.httpServer.ListenAndServe() + if http.ErrServerClosed != err { + log.Fatalf("httpServer.ListenAndServe() exited unexpectedly: %v", err) + } + + globals.httpServerWG.Done() + }() + + for startHTTPServerUpCheckRetries = 0; startHTTPServerUpCheckRetries < startHTTPServerUpCheckMaxRetries; startHTTPServerUpCheckRetries++ { + _, err = http.Get("http://" + ipAddrTCPPort + "/version") + if nil == err { + return + } + + time.Sleep(startHTTPServerUpCheckDelay) + } + + err = fmt.Errorf("startHTTPServerUpCheckMaxRetries (%v) exceeded", startHTTPServerUpCheckMaxRetries) + return +} + +func stopHTTPServer() (err error) { + err = globals.httpServer.Shutdown(context.TODO()) + if nil == err { + globals.httpServerWG.Wait() + } + + return +} + +func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + requestPath string + ) + + requestPath = strings.TrimRight(request.URL.Path, "/") + + _, err = ioutil.ReadAll(request.Body) + if nil == err { + err = request.Body.Close() + if nil != err { + responseWriter.WriteHeader(http.StatusBadRequest) + return + } + } else { + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusBadRequest) + return + } + + switch request.Method { + case http.MethodGet: + serveHTTPGet(responseWriter, request, requestPath) + default: + responseWriter.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { + switch { + case "" == requestPath: + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) + case "/bootstrap.min.css" == requestPath: + responseWriter.Header().Set("Content-Type", bootstrapDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(bootstrapDotCSSContent)) + case "/bootstrap.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", bootstrapDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(bootstrapDotJSContent)) + case "/config" == requestPath: + serveHTTPGetOfConfig(responseWriter, request) + case "/index.html" == requestPath: + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) + case "/jquery.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", jqueryDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(jqueryDotJSContent)) + case "/jsontree.js" == requestPath: + responseWriter.Header().Set("Content-Type", jsontreeDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(jsontreeDotJSContent)) + case "/open-iconic/font/css/open-iconic-bootstrap.min.css" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicBootstrapDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(openIconicBootstrapDotCSSContent)) + case "/open-iconic/font/fonts/open-iconic.eot" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotEOTContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotEOTContent) + case "/open-iconic/font/fonts/open-iconic.otf" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotOTFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotOTFContent) + case "/open-iconic/font/fonts/open-iconic.svg" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotSVGContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(openIconicDotSVGContent)) + case "/open-iconic/font/fonts/open-iconic.ttf" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotTTFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotTTFContent) + case "/open-iconic/font/fonts/open-iconic.woff" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotWOFFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotWOFFContent) + case "/popper.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", popperDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(popperDotJSContent) + case "/stats" == requestPath: + serveHTTPGetOfStats(responseWriter, request) + case "/styles.css" == requestPath: + responseWriter.Header().Set("Content-Type", stylesDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(stylesDotCSSContent)) + case "/version" == requestPath: + serveHTTPGetOfVersion(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusNotFound) + } +} + +func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Request) { + var ( + confMapJSON []byte + err error + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.GetConfigUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + confMapJSON, err = json.Marshal(globals.config) + if nil != err { + logFatalf("json.Marshal(globals.config) failed: %v", err) + } + + if strings.Contains(request.Header.Get("Accept"), "text/html") { + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(fmt.Sprintf(configTemplate, version.ProxyFSVersion, string(confMapJSON[:])))) + if nil != err { + logWarnf("responseWriter.Write([]byte(fmt.Sprintf(configTemplate, version.ProxyFSVersion, string(confMapJSON[:])))) failed: %v", err) + } + } else { + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(confMapJSON))) + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write(confMapJSON) + if nil != err { + logWarnf("responseWriter.Write(confMapJSON) failed: %v", err) + } + } +} + +func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + startTime time.Time = time.Now() + statsAsString string + ) + + defer func() { + globals.stats.GetStatsUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + statsAsString = bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*") + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(statsAsString))) + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(statsAsString)) + if nil != err { + logWarnf("responseWriter.Write([]byte(statsAsString)) failed: %v", err) + } +} + +func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.GetVersionUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(version.ProxyFSVersion))) + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(version.ProxyFSVersion)) + if nil != err { + logWarnf("responseWriter.Write([]byte(statsAsString)) failed: %v", err) + } +} diff --git a/iclient/iclientpkg/impl.go b/iclient/iclientpkg/impl.go index 488c112a..70e898a7 100644 --- a/iclient/iclientpkg/impl.go +++ b/iclient/iclientpkg/impl.go @@ -22,10 +22,20 @@ func start(confMap conf.ConfMap, fissionErrChan chan error) (err error) { // TODO + err = startHTTPServer() + if nil != err { + return + } + return } func stop() (err error) { + err = stopHTTPServer() + if nil != err { + return + } + // TODO err = performUnmountFUSE() diff --git a/iclient/iclientpkg/static-content.go b/iclient/iclientpkg/static-content.go new file mode 100644 index 00000000..133d22e0 --- /dev/null +++ b/iclient/iclientpkg/static-content.go @@ -0,0 +1,22 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +//go:generate ../../make-static-content/make-static-content iclientpkg stylesDotCSS text/css s static-content/styles.css styles_dot_css_.go + +//go:generate ../../make-static-content/make-static-content iclientpkg jsontreeDotJS application/javascript s static-content/jsontree.js jsontree_dot_js_.go + +//go:generate ../../make-static-content/make-static-content iclientpkg bootstrapDotCSS text/css s static-content/bootstrap.min.css bootstrap_dot_min_dot_css_.go + +//go:generate ../../make-static-content/make-static-content iclientpkg bootstrapDotJS application/javascript s static-content/bootstrap.min.js bootstrap_dot_min_dot_js_.go +//go:generate ../../make-static-content/make-static-content iclientpkg jqueryDotJS application/javascript s static-content/jquery.min.js jquery_dot_min_dot_js_.go +//go:generate ../../make-static-content/make-static-content iclientpkg popperDotJS application/javascript b static-content/popper.min.js popper_dot_min_dot_js_.go + +//go:generate ../../make-static-content/make-static-content iclientpkg openIconicBootstrapDotCSS text/css s static-content/open-iconic/font/css/open-iconic-bootstrap.min.css open_iconic_bootstrap_dot_css_.go + +//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotEOT application/vnd.ms-fontobject b static-content/open-iconic/font/fonts/open-iconic.eot open_iconic_dot_eot_.go +//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotOTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.otf open_iconic_dot_otf_.go +//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotSVG image/svg+xml s static-content/open-iconic/font/fonts/open-iconic.svg open_iconic_dot_svg_.go +//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotTTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.ttf open_iconic_dot_ttf_.go +//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotWOFF application/font-woff b static-content/open-iconic/font/fonts/open-iconic.woff open_iconic_dot_woff_.go diff --git a/iclient/iclientpkg/static-content/bootstrap.min.css b/iclient/iclientpkg/static-content/bootstrap.min.css new file mode 100644 index 00000000..7d2a868f --- /dev/null +++ b/iclient/iclientpkg/static-content/bootstrap.min.css @@ -0,0 +1,7 @@ +/*! + * Bootstrap v4.5.0 (https://getbootstrap.com/) + * Copyright 2011-2020 The Bootstrap Authors + * Copyright 2011-2020 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */:root{--blue:#007bff;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#dc3545;--orange:#fd7e14;--yellow:#ffc107;--green:#28a745;--teal:#20c997;--cyan:#17a2b8;--white:#fff;--gray:#6c757d;--gray-dark:#343a40;--primary:#007bff;--secondary:#6c757d;--success:#28a745;--info:#17a2b8;--warning:#ffc107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus:not(:focus-visible){outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]){color:inherit;text-decoration:none}a:not([href]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-weight:500;line-height:1.2}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#6c757d}.blockquote-footer::before{content:"\2014\00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#6c757d}code{font-size:87.5%;color:#e83e8c;word-wrap:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:87.5%;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;font-size:87.5%;color:#212529}pre code{font-size:inherit;color:inherit;word-break:normal}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container{max-width:540px}}@media (min-width:768px){.container{max-width:720px}}@media (min-width:992px){.container{max-width:960px}}@media (min-width:1200px){.container{max-width:1140px}}.container-fluid,.container-lg,.container-md,.container-sm,.container-xl{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container,.container-sm{max-width:540px}}@media (min-width:768px){.container,.container-md,.container-sm{max-width:720px}}@media (min-width:992px){.container,.container-lg,.container-md,.container-sm{max-width:960px}}@media (min-width:1200px){.container,.container-lg,.container-md,.container-sm,.container-xl{max-width:1140px}}.row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-auto,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-auto,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-auto,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-auto,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9,.col-xl-auto{position:relative;width:100%;padding-right:15px;padding-left:15px}.col{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-first{-ms-flex-order:-1;order:-1}.order-last{-ms-flex-order:13;order:13}.order-0{-ms-flex-order:0;order:0}.order-1{-ms-flex-order:1;order:1}.order-2{-ms-flex-order:2;order:2}.order-3{-ms-flex-order:3;order:3}.order-4{-ms-flex-order:4;order:4}.order-5{-ms-flex-order:5;order:5}.order-6{-ms-flex-order:6;order:6}.order-7{-ms-flex-order:7;order:7}.order-8{-ms-flex-order:8;order:8}.order-9{-ms-flex-order:9;order:9}.order-10{-ms-flex-order:10;order:10}.order-11{-ms-flex-order:11;order:11}.order-12{-ms-flex-order:12;order:12}.offset-1{margin-left:8.333333%}.offset-2{margin-left:16.666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.333333%}.offset-5{margin-left:41.666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.333333%}.offset-8{margin-left:66.666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.333333%}.offset-11{margin-left:91.666667%}@media (min-width:576px){.col-sm{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-sm-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-sm-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-sm-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-sm-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-sm-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-sm-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-sm-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-sm-first{-ms-flex-order:-1;order:-1}.order-sm-last{-ms-flex-order:13;order:13}.order-sm-0{-ms-flex-order:0;order:0}.order-sm-1{-ms-flex-order:1;order:1}.order-sm-2{-ms-flex-order:2;order:2}.order-sm-3{-ms-flex-order:3;order:3}.order-sm-4{-ms-flex-order:4;order:4}.order-sm-5{-ms-flex-order:5;order:5}.order-sm-6{-ms-flex-order:6;order:6}.order-sm-7{-ms-flex-order:7;order:7}.order-sm-8{-ms-flex-order:8;order:8}.order-sm-9{-ms-flex-order:9;order:9}.order-sm-10{-ms-flex-order:10;order:10}.order-sm-11{-ms-flex-order:11;order:11}.order-sm-12{-ms-flex-order:12;order:12}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.333333%}.offset-sm-2{margin-left:16.666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.333333%}.offset-sm-5{margin-left:41.666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.333333%}.offset-sm-8{margin-left:66.666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.333333%}.offset-sm-11{margin-left:91.666667%}}@media (min-width:768px){.col-md{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-md-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-md-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-md-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-md-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-md-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-md-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-md-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-md-first{-ms-flex-order:-1;order:-1}.order-md-last{-ms-flex-order:13;order:13}.order-md-0{-ms-flex-order:0;order:0}.order-md-1{-ms-flex-order:1;order:1}.order-md-2{-ms-flex-order:2;order:2}.order-md-3{-ms-flex-order:3;order:3}.order-md-4{-ms-flex-order:4;order:4}.order-md-5{-ms-flex-order:5;order:5}.order-md-6{-ms-flex-order:6;order:6}.order-md-7{-ms-flex-order:7;order:7}.order-md-8{-ms-flex-order:8;order:8}.order-md-9{-ms-flex-order:9;order:9}.order-md-10{-ms-flex-order:10;order:10}.order-md-11{-ms-flex-order:11;order:11}.order-md-12{-ms-flex-order:12;order:12}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.333333%}.offset-md-2{margin-left:16.666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.333333%}.offset-md-5{margin-left:41.666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.333333%}.offset-md-8{margin-left:66.666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.333333%}.offset-md-11{margin-left:91.666667%}}@media (min-width:992px){.col-lg{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-lg-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-lg-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-lg-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-lg-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-lg-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-lg-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-lg-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-lg-first{-ms-flex-order:-1;order:-1}.order-lg-last{-ms-flex-order:13;order:13}.order-lg-0{-ms-flex-order:0;order:0}.order-lg-1{-ms-flex-order:1;order:1}.order-lg-2{-ms-flex-order:2;order:2}.order-lg-3{-ms-flex-order:3;order:3}.order-lg-4{-ms-flex-order:4;order:4}.order-lg-5{-ms-flex-order:5;order:5}.order-lg-6{-ms-flex-order:6;order:6}.order-lg-7{-ms-flex-order:7;order:7}.order-lg-8{-ms-flex-order:8;order:8}.order-lg-9{-ms-flex-order:9;order:9}.order-lg-10{-ms-flex-order:10;order:10}.order-lg-11{-ms-flex-order:11;order:11}.order-lg-12{-ms-flex-order:12;order:12}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.333333%}.offset-lg-2{margin-left:16.666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.333333%}.offset-lg-5{margin-left:41.666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.333333%}.offset-lg-8{margin-left:66.666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.333333%}.offset-lg-11{margin-left:91.666667%}}@media (min-width:1200px){.col-xl{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-xl-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-xl-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-xl-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-xl-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-xl-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-xl-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-xl-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-xl-first{-ms-flex-order:-1;order:-1}.order-xl-last{-ms-flex-order:13;order:13}.order-xl-0{-ms-flex-order:0;order:0}.order-xl-1{-ms-flex-order:1;order:1}.order-xl-2{-ms-flex-order:2;order:2}.order-xl-3{-ms-flex-order:3;order:3}.order-xl-4{-ms-flex-order:4;order:4}.order-xl-5{-ms-flex-order:5;order:5}.order-xl-6{-ms-flex-order:6;order:6}.order-xl-7{-ms-flex-order:7;order:7}.order-xl-8{-ms-flex-order:8;order:8}.order-xl-9{-ms-flex-order:9;order:9}.order-xl-10{-ms-flex-order:10;order:10}.order-xl-11{-ms-flex-order:11;order:11}.order-xl-12{-ms-flex-order:12;order:12}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.333333%}.offset-xl-2{margin-left:16.666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.333333%}.offset-xl-5{margin-left:41.666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.333333%}.offset-xl-8{margin-left:66.666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.333333%}.offset-xl-11{margin-left:91.666667%}}.table{width:100%;margin-bottom:1rem;color:#212529}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #dee2e6}.table thead th{vertical-align:bottom;border-bottom:2px solid #dee2e6}.table tbody+tbody{border-top:2px solid #dee2e6}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #dee2e6}.table-bordered td,.table-bordered th{border:1px solid #dee2e6}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-borderless tbody+tbody,.table-borderless td,.table-borderless th,.table-borderless thead th{border:0}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{color:#212529;background-color:rgba(0,0,0,.075)}.table-primary,.table-primary>td,.table-primary>th{background-color:#b8daff}.table-primary tbody+tbody,.table-primary td,.table-primary th,.table-primary thead th{border-color:#7abaff}.table-hover .table-primary:hover{background-color:#9fcdff}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#9fcdff}.table-secondary,.table-secondary>td,.table-secondary>th{background-color:#d6d8db}.table-secondary tbody+tbody,.table-secondary td,.table-secondary th,.table-secondary thead th{border-color:#b3b7bb}.table-hover .table-secondary:hover{background-color:#c8cbcf}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#c8cbcf}.table-success,.table-success>td,.table-success>th{background-color:#c3e6cb}.table-success tbody+tbody,.table-success td,.table-success th,.table-success thead th{border-color:#8fd19e}.table-hover .table-success:hover{background-color:#b1dfbb}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#b1dfbb}.table-info,.table-info>td,.table-info>th{background-color:#bee5eb}.table-info tbody+tbody,.table-info td,.table-info th,.table-info thead th{border-color:#86cfda}.table-hover .table-info:hover{background-color:#abdde5}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#abdde5}.table-warning,.table-warning>td,.table-warning>th{background-color:#ffeeba}.table-warning tbody+tbody,.table-warning td,.table-warning th,.table-warning thead th{border-color:#ffdf7e}.table-hover .table-warning:hover{background-color:#ffe8a1}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#ffe8a1}.table-danger,.table-danger>td,.table-danger>th{background-color:#f5c6cb}.table-danger tbody+tbody,.table-danger td,.table-danger th,.table-danger thead th{border-color:#ed969e}.table-hover .table-danger:hover{background-color:#f1b0b7}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#f1b0b7}.table-light,.table-light>td,.table-light>th{background-color:#fdfdfe}.table-light tbody+tbody,.table-light td,.table-light th,.table-light thead th{border-color:#fbfcfc}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>td,.table-dark>th{background-color:#c6c8ca}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#95999c}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.table .thead-dark th{color:#fff;background-color:#343a40;border-color:#454d55}.table .thead-light th{color:#495057;background-color:#e9ecef;border-color:#dee2e6}.table-dark{color:#fff;background-color:#343a40}.table-dark td,.table-dark th,.table-dark thead th{border-color:#454d55}.table-dark.table-bordered{border:0}.table-dark.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,.05)}.table-dark.table-hover tbody tr:hover{color:#fff;background-color:rgba(255,255,255,.075)}@media (max-width:575.98px){.table-responsive-sm{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-sm>.table-bordered{border:0}}@media (max-width:767.98px){.table-responsive-md{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-md>.table-bordered{border:0}}@media (max-width:991.98px){.table-responsive-lg{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-lg>.table-bordered{border:0}}@media (max-width:1199.98px){.table-responsive-xl{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-xl>.table-bordered{border:0}}.table-responsive{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive>.table-bordered{border:0}.form-control{display:block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.form-control:focus{color:#495057;background-color:#fff;border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.form-control::-webkit-input-placeholder{color:#6c757d;opacity:1}.form-control::-moz-placeholder{color:#6c757d;opacity:1}.form-control:-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}input[type=date].form-control,input[type=datetime-local].form-control,input[type=month].form-control,input[type=time].form-control{-webkit-appearance:none;-moz-appearance:none;appearance:none}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block;width:100%}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem;line-height:1.5}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem;line-height:1.5}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;font-size:1rem;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.form-control-lg{height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}select.form-control[multiple],select.form-control[size]{height:auto}textarea.form-control{height:auto}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*=col-]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;padding-left:1.25rem}.form-check-input{position:absolute;margin-top:.3rem;margin-left:-1.25rem}.form-check-input:disabled~.form-check-label,.form-check-input[disabled]~.form-check-label{color:#6c757d}.form-check-label{margin-bottom:0}.form-check-inline{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-align:center;align-items:center;padding-left:0;margin-right:.75rem}.form-check-inline .form-check-input{position:static;margin-top:0;margin-right:.3125rem;margin-left:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#28a745}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(40,167,69,.9);border-radius:.25rem}.is-valid~.valid-feedback,.is-valid~.valid-tooltip,.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip{display:block}.form-control.is-valid,.was-validated .form-control:valid{border-color:#28a745;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-valid,.was-validated .custom-select:valid{border-color:#28a745;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-valid:focus,.was-validated .custom-select:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:#28a745}.form-check-input.is-valid~.valid-feedback,.form-check-input.is-valid~.valid-tooltip,.was-validated .form-check-input:valid~.valid-feedback,.was-validated .form-check-input:valid~.valid-tooltip{display:block}.custom-control-input.is-valid~.custom-control-label,.was-validated .custom-control-input:valid~.custom-control-label{color:#28a745}.custom-control-input.is-valid~.custom-control-label::before,.was-validated .custom-control-input:valid~.custom-control-label::before{border-color:#28a745}.custom-control-input.is-valid:checked~.custom-control-label::before,.was-validated .custom-control-input:valid:checked~.custom-control-label::before{border-color:#34ce57;background-color:#34ce57}.custom-control-input.is-valid:focus~.custom-control-label::before,.was-validated .custom-control-input:valid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-control-input.is-valid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:valid:focus:not(:checked)~.custom-control-label::before{border-color:#28a745}.custom-file-input.is-valid~.custom-file-label,.was-validated .custom-file-input:valid~.custom-file-label{border-color:#28a745}.custom-file-input.is-valid:focus~.custom-file-label,.was-validated .custom-file-input:valid:focus~.custom-file-label{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(220,53,69,.9);border-radius:.25rem}.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip,.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip{display:block}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:#dc3545;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-invalid,.was-validated .custom-select:invalid{border-color:#dc3545;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-invalid:focus,.was-validated .custom-select:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:#dc3545}.form-check-input.is-invalid~.invalid-feedback,.form-check-input.is-invalid~.invalid-tooltip,.was-validated .form-check-input:invalid~.invalid-feedback,.was-validated .form-check-input:invalid~.invalid-tooltip{display:block}.custom-control-input.is-invalid~.custom-control-label,.was-validated .custom-control-input:invalid~.custom-control-label{color:#dc3545}.custom-control-input.is-invalid~.custom-control-label::before,.was-validated .custom-control-input:invalid~.custom-control-label::before{border-color:#dc3545}.custom-control-input.is-invalid:checked~.custom-control-label::before,.was-validated .custom-control-input:invalid:checked~.custom-control-label::before{border-color:#e4606d;background-color:#e4606d}.custom-control-input.is-invalid:focus~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.custom-control-input.is-invalid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus:not(:checked)~.custom-control-label::before{border-color:#dc3545}.custom-file-input.is-invalid~.custom-file-label,.was-validated .custom-file-input:invalid~.custom-file-label{border-color:#dc3545}.custom-file-input.is-invalid:focus~.custom-file-label,.was-validated .custom-file-input:invalid:focus~.custom-file-label{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-inline{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-ms-flexbox;display:flex;-ms-flex:0 0 auto;flex:0 0 auto;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .custom-select,.form-inline .input-group{width:auto}.form-inline .form-check{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:auto;padding-left:0}.form-inline .form-check-input{position:relative;-ms-flex-negative:0;flex-shrink:0;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center}.form-inline .custom-control-label{margin-bottom:0}}.btn{display:inline-block;font-weight:400;color:#212529;text-align:center;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;line-height:1.5;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:#212529;text-decoration:none}.btn.focus,.btn:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.btn.disabled,.btn:disabled{opacity:.65}.btn:not(:disabled):not(.disabled){cursor:pointer}a.btn.disabled,fieldset:disabled a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:hover{color:#fff;background-color:#0069d9;border-color:#0062cc}.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#0069d9;border-color:#0062cc;box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:not(:disabled):not(.disabled).active,.btn-primary:not(:disabled):not(.disabled):active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0062cc;border-color:#005cbf}.btn-primary:not(:disabled):not(.disabled).active:focus,.btn-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-secondary{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:hover{color:#fff;background-color:#5a6268;border-color:#545b62}.btn-secondary.focus,.btn-secondary:focus{color:#fff;background-color:#5a6268;border-color:#545b62;box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:not(:disabled):not(.disabled).active,.btn-secondary:not(:disabled):not(.disabled):active,.show>.btn-secondary.dropdown-toggle{color:#fff;background-color:#545b62;border-color:#4e555b}.btn-secondary:not(:disabled):not(.disabled).active:focus,.btn-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-success{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:hover{color:#fff;background-color:#218838;border-color:#1e7e34}.btn-success.focus,.btn-success:focus{color:#fff;background-color:#218838;border-color:#1e7e34;box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:not(:disabled):not(.disabled).active,.btn-success:not(:disabled):not(.disabled):active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#1e7e34;border-color:#1c7430}.btn-success:not(:disabled):not(.disabled).active:focus,.btn-success:not(:disabled):not(.disabled):active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-info{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:hover{color:#fff;background-color:#138496;border-color:#117a8b}.btn-info.focus,.btn-info:focus{color:#fff;background-color:#138496;border-color:#117a8b;box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-info.disabled,.btn-info:disabled{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:not(:disabled):not(.disabled).active,.btn-info:not(:disabled):not(.disabled):active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#117a8b;border-color:#10707f}.btn-info:not(:disabled):not(.disabled).active:focus,.btn-info:not(:disabled):not(.disabled):active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-warning{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#212529;background-color:#e0a800;border-color:#d39e00}.btn-warning.focus,.btn-warning:focus{color:#212529;background-color:#e0a800;border-color:#d39e00;box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-warning.disabled,.btn-warning:disabled{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:not(:disabled):not(.disabled).active,.btn-warning:not(:disabled):not(.disabled):active,.show>.btn-warning.dropdown-toggle{color:#212529;background-color:#d39e00;border-color:#c69500}.btn-warning:not(:disabled):not(.disabled).active:focus,.btn-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#c82333;border-color:#bd2130}.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c82333;border-color:#bd2130;box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:not(:disabled):not(.disabled).active,.btn-danger:not(:disabled):not(.disabled):active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#bd2130;border-color:#b21f2d}.btn-danger:not(:disabled):not(.disabled).active:focus,.btn-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-light{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#212529;background-color:#e2e6ea;border-color:#dae0e5}.btn-light.focus,.btn-light:focus{color:#212529;background-color:#e2e6ea;border-color:#dae0e5;box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-light.disabled,.btn-light:disabled{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:not(:disabled):not(.disabled).active,.btn-light:not(:disabled):not(.disabled):active,.show>.btn-light.dropdown-toggle{color:#212529;background-color:#dae0e5;border-color:#d3d9df}.btn-light:not(:disabled):not(.disabled).active:focus,.btn-light:not(:disabled):not(.disabled):active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark.focus,.btn-dark:focus{color:#fff;background-color:#23272b;border-color:#1d2124;box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:not(:disabled):not(.disabled).active,.btn-dark:not(:disabled):not(.disabled):active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1d2124;border-color:#171a1d}.btn-dark:not(:disabled):not(.disabled).active:focus,.btn-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-outline-primary{color:#007bff;border-color:#007bff}.btn-outline-primary:hover{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary.focus,.btn-outline-primary:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#007bff;background-color:transparent}.btn-outline-primary:not(:disabled):not(.disabled).active,.btn-outline-primary:not(:disabled):not(.disabled):active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary:not(:disabled):not(.disabled).active:focus,.btn-outline-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-secondary{color:#6c757d;border-color:#6c757d}.btn-outline-secondary:hover{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary.focus,.btn-outline-secondary:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#6c757d;background-color:transparent}.btn-outline-secondary:not(:disabled):not(.disabled).active,.btn-outline-secondary:not(:disabled):not(.disabled):active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary:not(:disabled):not(.disabled).active:focus,.btn-outline-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-success{color:#28a745;border-color:#28a745}.btn-outline-success:hover{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success.focus,.btn-outline-success:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#28a745;background-color:transparent}.btn-outline-success:not(:disabled):not(.disabled).active,.btn-outline-success:not(:disabled):not(.disabled):active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success:not(:disabled):not(.disabled).active:focus,.btn-outline-success:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-info{color:#17a2b8;border-color:#17a2b8}.btn-outline-info:hover{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info.focus,.btn-outline-info:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#17a2b8;background-color:transparent}.btn-outline-info:not(:disabled):not(.disabled).active,.btn-outline-info:not(:disabled):not(.disabled):active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info:not(:disabled):not(.disabled).active:focus,.btn-outline-info:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-warning{color:#ffc107;border-color:#ffc107}.btn-outline-warning:hover{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning.focus,.btn-outline-warning:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-warning:not(:disabled):not(.disabled).active,.btn-outline-warning:not(:disabled):not(.disabled):active,.show>.btn-outline-warning.dropdown-toggle{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning:not(:disabled):not(.disabled).active:focus,.btn-outline-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-danger{color:#dc3545;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger.focus,.btn-outline-danger:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-danger:not(:disabled):not(.disabled).active,.btn-outline-danger:not(:disabled):not(.disabled):active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger:not(:disabled):not(.disabled).active:focus,.btn-outline-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light.focus,.btn-outline-light:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light:not(:disabled):not(.disabled).active,.btn-outline-light:not(:disabled):not(.disabled):active,.show>.btn-outline-light.dropdown-toggle{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:not(:disabled):not(.disabled).active:focus,.btn-outline-light:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-dark{color:#343a40;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark.focus,.btn-outline-dark:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark:not(:disabled):not(.disabled).active,.btn-outline-dark:not(:disabled):not(.disabled):active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:not(:disabled):not(.disabled).active:focus,.btn-outline-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-link{font-weight:400;color:#007bff;text-decoration:none}.btn-link:hover{color:#0056b3;text-decoration:underline}.btn-link.focus,.btn-link:focus{text-decoration:underline}.btn-link.disabled,.btn-link:disabled{color:#6c757d;pointer-events:none}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{position:relative;height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.dropdown,.dropleft,.dropright,.dropup{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-menu-left{right:auto;left:0}.dropdown-menu-right{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-left{right:auto;left:0}.dropdown-menu-sm-right{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-left{right:auto;left:0}.dropdown-menu-md-right{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-left{right:auto;left:0}.dropdown-menu-lg-right{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-left{right:auto;left:0}.dropdown-menu-xl-right{right:0;left:auto}}.dropup .dropdown-menu{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-menu{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropright .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropright .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-toggle::after{vertical-align:0}.dropleft .dropdown-menu{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropleft .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropleft .dropdown-toggle::after{display:none}.dropleft .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropleft .dropdown-toggle:empty::after{margin-left:0}.dropleft .dropdown-toggle::before{vertical-align:0}.dropdown-menu[x-placement^=bottom],.dropdown-menu[x-placement^=left],.dropdown-menu[x-placement^=right],.dropdown-menu[x-placement^=top]{right:auto;bottom:auto}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:.25rem 1.5rem;clear:both;font-weight:400;color:#212529;text-align:inherit;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#16181b;text-decoration:none;background-color:#f8f9fa}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#007bff}.dropdown-item.disabled,.dropdown-item:disabled{color:#6c757d;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1.5rem;color:#212529}.btn-group,.btn-group-vertical{position:relative;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-ms-flex:1 1 auto;flex:1 1 auto}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:1}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:1}.btn-toolbar{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn-group:not(:first-child),.btn-group>.btn:not(:first-child){margin-left:-1px}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropright .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropleft .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{-ms-flex-direction:column;flex-direction:column;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:center;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn:not(:first-child){border-top-left-radius:0;border-top-right-radius:0}.btn-group-toggle>.btn,.btn-group-toggle>.btn-group>.btn{margin-bottom:0}.btn-group-toggle>.btn input[type=checkbox],.btn-group-toggle>.btn input[type=radio],.btn-group-toggle>.btn-group>.btn input[type=checkbox],.btn-group-toggle>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:stretch;align-items:stretch;width:100%}.input-group>.custom-file,.input-group>.custom-select,.input-group>.form-control,.input-group>.form-control-plaintext{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;min-width:0;margin-bottom:0}.input-group>.custom-file+.custom-file,.input-group>.custom-file+.custom-select,.input-group>.custom-file+.form-control,.input-group>.custom-select+.custom-file,.input-group>.custom-select+.custom-select,.input-group>.custom-select+.form-control,.input-group>.form-control+.custom-file,.input-group>.form-control+.custom-select,.input-group>.form-control+.form-control,.input-group>.form-control-plaintext+.custom-file,.input-group>.form-control-plaintext+.custom-select,.input-group>.form-control-plaintext+.form-control{margin-left:-1px}.input-group>.custom-file .custom-file-input:focus~.custom-file-label,.input-group>.custom-select:focus,.input-group>.form-control:focus{z-index:3}.input-group>.custom-file .custom-file-input:focus{z-index:4}.input-group>.custom-select:not(:last-child),.input-group>.form-control:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-select:not(:first-child),.input-group>.form-control:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.custom-file{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center}.input-group>.custom-file:not(:last-child) .custom-file-label,.input-group>.custom-file:not(:last-child) .custom-file-label::after{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-file:not(:first-child) .custom-file-label{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-append,.input-group-prepend{display:-ms-flexbox;display:flex}.input-group-append .btn,.input-group-prepend .btn{position:relative;z-index:2}.input-group-append .btn:focus,.input-group-prepend .btn:focus{z-index:3}.input-group-append .btn+.btn,.input-group-append .btn+.input-group-text,.input-group-append .input-group-text+.btn,.input-group-append .input-group-text+.input-group-text,.input-group-prepend .btn+.btn,.input-group-prepend .btn+.input-group-text,.input-group-prepend .input-group-text+.btn,.input-group-prepend .input-group-text+.input-group-text{margin-left:-1px}.input-group-prepend{margin-right:-1px}.input-group-append{margin-left:-1px}.input-group-text{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.375rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-text input[type=checkbox],.input-group-text input[type=radio]{margin-top:0}.input-group-lg>.custom-select,.input-group-lg>.form-control:not(textarea){height:calc(1.5em + 1rem + 2px)}.input-group-lg>.custom-select,.input-group-lg>.form-control,.input-group-lg>.input-group-append>.btn,.input-group-lg>.input-group-append>.input-group-text,.input-group-lg>.input-group-prepend>.btn,.input-group-lg>.input-group-prepend>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.input-group-sm>.custom-select,.input-group-sm>.form-control:not(textarea){height:calc(1.5em + .5rem + 2px)}.input-group-sm>.custom-select,.input-group-sm>.form-control,.input-group-sm>.input-group-append>.btn,.input-group-sm>.input-group-append>.input-group-text,.input-group-sm>.input-group-prepend>.btn,.input-group-sm>.input-group-prepend>.input-group-text{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.input-group-lg>.custom-select,.input-group-sm>.custom-select{padding-right:1.75rem}.input-group>.input-group-append:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group>.input-group-append:last-child>.input-group-text:not(:last-child),.input-group>.input-group-append:not(:last-child)>.btn,.input-group>.input-group-append:not(:last-child)>.input-group-text,.input-group>.input-group-prepend>.btn,.input-group>.input-group-prepend>.input-group-text{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.input-group-append>.btn,.input-group>.input-group-append>.input-group-text,.input-group>.input-group-prepend:first-child>.btn:not(:first-child),.input-group>.input-group-prepend:first-child>.input-group-text:not(:first-child),.input-group>.input-group-prepend:not(:first-child)>.btn,.input-group>.input-group-prepend:not(:first-child)>.input-group-text{border-top-left-radius:0;border-bottom-left-radius:0}.custom-control{position:relative;display:block;min-height:1.5rem;padding-left:1.5rem}.custom-control-inline{display:-ms-inline-flexbox;display:inline-flex;margin-right:1rem}.custom-control-input{position:absolute;left:0;z-index:-1;width:1rem;height:1.25rem;opacity:0}.custom-control-input:checked~.custom-control-label::before{color:#fff;border-color:#007bff;background-color:#007bff}.custom-control-input:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-control-input:focus:not(:checked)~.custom-control-label::before{border-color:#80bdff}.custom-control-input:not(:disabled):active~.custom-control-label::before{color:#fff;background-color:#b3d7ff;border-color:#b3d7ff}.custom-control-input:disabled~.custom-control-label,.custom-control-input[disabled]~.custom-control-label{color:#6c757d}.custom-control-input:disabled~.custom-control-label::before,.custom-control-input[disabled]~.custom-control-label::before{background-color:#e9ecef}.custom-control-label{position:relative;margin-bottom:0;vertical-align:top}.custom-control-label::before{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;pointer-events:none;content:"";background-color:#fff;border:#adb5bd solid 1px}.custom-control-label::after{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;content:"";background:no-repeat 50%/50% 50%}.custom-checkbox .custom-control-label::before{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26l2.974 2.99L8 2.193z'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::before{border-color:#007bff;background-color:#007bff}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='4' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-checkbox .custom-control-input:disabled:indeterminate~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-radio .custom-control-label::before{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.custom-radio .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-switch{padding-left:2.25rem}.custom-switch .custom-control-label::before{left:-2.25rem;width:1.75rem;pointer-events:all;border-radius:.5rem}.custom-switch .custom-control-label::after{top:calc(.25rem + 2px);left:calc(-2.25rem + 2px);width:calc(1rem - 4px);height:calc(1rem - 4px);background-color:#adb5bd;border-radius:.5rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-switch .custom-control-label::after{transition:none}}.custom-switch .custom-control-input:checked~.custom-control-label::after{background-color:#fff;-webkit-transform:translateX(.75rem);transform:translateX(.75rem)}.custom-switch .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-select{display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem 1.75rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;vertical-align:middle;background:#fff url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px;border:1px solid #ced4da;border-radius:.25rem;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-select:focus{border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select[multiple],.custom-select[size]:not([size="1"]){height:auto;padding-right:.75rem;background-image:none}.custom-select:disabled{color:#6c757d;background-color:#e9ecef}.custom-select::-ms-expand{display:none}.custom-select:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.custom-select-sm{height:calc(1.5em + .5rem + 2px);padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem}.custom-select-lg{height:calc(1.5em + 1rem + 2px);padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}.custom-file{position:relative;display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);margin-bottom:0}.custom-file-input{position:relative;z-index:2;width:100%;height:calc(1.5em + .75rem + 2px);margin:0;opacity:0}.custom-file-input:focus~.custom-file-label{border-color:#80bdff;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-file-input:disabled~.custom-file-label,.custom-file-input[disabled]~.custom-file-label{background-color:#e9ecef}.custom-file-input:lang(en)~.custom-file-label::after{content:"Browse"}.custom-file-input~.custom-file-label[data-browse]::after{content:attr(data-browse)}.custom-file-label{position:absolute;top:0;right:0;left:0;z-index:1;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem}.custom-file-label::after{position:absolute;top:0;right:0;bottom:0;z-index:3;display:block;height:calc(1.5em + .75rem);padding:.375rem .75rem;line-height:1.5;color:#495057;content:"Browse";background-color:#e9ecef;border-left:inherit;border-radius:0 .25rem .25rem 0}.custom-range{width:100%;height:1.4rem;padding:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-range:focus{outline:0}.custom-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-ms-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range::-moz-focus-outer{border:0}.custom-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#007bff;border:0;border-radius:1rem;-webkit-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.custom-range::-webkit-slider-thumb:active{background-color:#b3d7ff}.custom-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#007bff;border:0;border-radius:1rem;-moz-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-moz-range-thumb{-moz-transition:none;transition:none}}.custom-range::-moz-range-thumb:active{background-color:#b3d7ff}.custom-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-ms-thumb{width:1rem;height:1rem;margin-top:0;margin-right:.2rem;margin-left:.2rem;background-color:#007bff;border:0;border-radius:1rem;-ms-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-ms-thumb{-ms-transition:none;transition:none}}.custom-range::-ms-thumb:active{background-color:#b3d7ff}.custom-range::-ms-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:transparent;border-color:transparent;border-width:.5rem}.custom-range::-ms-fill-lower{background-color:#dee2e6;border-radius:1rem}.custom-range::-ms-fill-upper{margin-right:15px;background-color:#dee2e6;border-radius:1rem}.custom-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.custom-range:disabled::-webkit-slider-runnable-track{cursor:default}.custom-range:disabled::-moz-range-thumb{background-color:#adb5bd}.custom-range:disabled::-moz-range-track{cursor:default}.custom-range:disabled::-ms-thumb{background-color:#adb5bd}.custom-control-label::before,.custom-file-label,.custom-select{transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-control-label::before,.custom-file-label,.custom-select{transition:none}}.nav{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #dee2e6}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#007bff}.nav-fill .nav-item{-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between;padding:.5rem 1rem}.navbar .container,.navbar .container-fluid,.navbar .container-lg,.navbar .container-md,.navbar .container-sm,.navbar .container-xl{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{-ms-flex-preferred-size:100%;flex-basis:100%;-ms-flex-positive:1;flex-grow:1;-ms-flex-align:center;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width:575.98px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-expand-sm{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-sm .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width:767.98px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-expand-md{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-md .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width:991.98px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-expand-lg{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-lg .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width:1199.98px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-expand-xl{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-xl .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.5);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%280, 0, 0, 0.5%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-light .navbar-text a{color:rgba(0,0,0,.9)}.navbar-light .navbar-text a:focus,.navbar-light .navbar-text a:hover{color:rgba(0,0,0,.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.5);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.5%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,.5)}.navbar-dark .navbar-text a{color:#fff}.navbar-dark .navbar-text a:focus,.navbar-dark .navbar-text a:hover{color:#fff}.card{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card>.list-group:last-child{border-bottom-width:0;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-body{-ms-flex:1 1 auto;flex:1 1 auto;min-height:1px;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-header+.list-group .list-group-item:first-child{border-top:0}.card-footer{padding:.75rem 1.25rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img,.card-img-bottom,.card-img-top{-ms-flex-negative:0;flex-shrink:0;width:100%}.card-img,.card-img-top{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-deck .card{margin-bottom:15px}@media (min-width:576px){.card-deck{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{-ms-flex:1 0 0%;flex:1 0 0%;margin-right:15px;margin-bottom:0;margin-left:15px}}.card-group>.card{margin-bottom:15px}@media (min-width:576px){.card-group{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group>.card{-ms-flex:1 0 0%;flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.card-columns .card{margin-bottom:.75rem}@media (min-width:576px){.card-columns{-webkit-column-count:3;-moz-column-count:3;column-count:3;-webkit-column-gap:1.25rem;-moz-column-gap:1.25rem;column-gap:1.25rem;orphans:1;widows:1}.card-columns .card{display:inline-block;width:100%}}.accordion>.card{overflow:hidden}.accordion>.card:not(:last-of-type){border-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.accordion>.card:not(:first-of-type){border-top-left-radius:0;border-top-right-radius:0}.accordion>.card>.card-header{border-radius:0;margin-bottom:-1px}.breadcrumb{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:.25rem}.breadcrumb-item{display:-ms-flexbox;display:flex}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;color:#6c757d;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#6c757d}.pagination{display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#007bff;background-color:#fff;border:1px solid #dee2e6}.page-link:hover{z-index:2;color:#0056b3;text-decoration:none;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:3;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.page-item.active .page-link{z-index:3;color:#fff;background-color:#007bff;border-color:#007bff}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;cursor:auto;background-color:#fff;border-color:#dee2e6}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.badge{transition:none}}a.badge:focus,a.badge:hover{text-decoration:none}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#007bff}a.badge-primary:focus,a.badge-primary:hover{color:#fff;background-color:#0062cc}a.badge-primary.focus,a.badge-primary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.badge-secondary{color:#fff;background-color:#6c757d}a.badge-secondary:focus,a.badge-secondary:hover{color:#fff;background-color:#545b62}a.badge-secondary.focus,a.badge-secondary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.badge-success{color:#fff;background-color:#28a745}a.badge-success:focus,a.badge-success:hover{color:#fff;background-color:#1e7e34}a.badge-success.focus,a.badge-success:focus{outline:0;box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.badge-info{color:#fff;background-color:#17a2b8}a.badge-info:focus,a.badge-info:hover{color:#fff;background-color:#117a8b}a.badge-info.focus,a.badge-info:focus{outline:0;box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.badge-warning{color:#212529;background-color:#ffc107}a.badge-warning:focus,a.badge-warning:hover{color:#212529;background-color:#d39e00}a.badge-warning.focus,a.badge-warning:focus{outline:0;box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.badge-danger{color:#fff;background-color:#dc3545}a.badge-danger:focus,a.badge-danger:hover{color:#fff;background-color:#bd2130}a.badge-danger.focus,a.badge-danger:focus{outline:0;box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.badge-light{color:#212529;background-color:#f8f9fa}a.badge-light:focus,a.badge-light:hover{color:#212529;background-color:#dae0e5}a.badge-light.focus,a.badge-light:focus{outline:0;box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.badge-dark{color:#fff;background-color:#343a40}a.badge-dark:focus,a.badge-dark:hover{color:#fff;background-color:#1d2124}a.badge-dark.focus,a.badge-dark:focus{outline:0;box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{position:relative;padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:4rem}.alert-dismissible .close{position:absolute;top:0;right:0;padding:.75rem 1.25rem;color:inherit}.alert-primary{color:#004085;background-color:#cce5ff;border-color:#b8daff}.alert-primary hr{border-top-color:#9fcdff}.alert-primary .alert-link{color:#002752}.alert-secondary{color:#383d41;background-color:#e2e3e5;border-color:#d6d8db}.alert-secondary hr{border-top-color:#c8cbcf}.alert-secondary .alert-link{color:#202326}.alert-success{color:#155724;background-color:#d4edda;border-color:#c3e6cb}.alert-success hr{border-top-color:#b1dfbb}.alert-success .alert-link{color:#0b2e13}.alert-info{color:#0c5460;background-color:#d1ecf1;border-color:#bee5eb}.alert-info hr{border-top-color:#abdde5}.alert-info .alert-link{color:#062c33}.alert-warning{color:#856404;background-color:#fff3cd;border-color:#ffeeba}.alert-warning hr{border-top-color:#ffe8a1}.alert-warning .alert-link{color:#533f03}.alert-danger{color:#721c24;background-color:#f8d7da;border-color:#f5c6cb}.alert-danger hr{border-top-color:#f1b0b7}.alert-danger .alert-link{color:#491217}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-ms-flexbox;display:flex;height:1rem;overflow:hidden;line-height:0;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;overflow:hidden;color:#fff;text-align:center;white-space:nowrap;background-color:#007bff;transition:width .6s ease}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}@media (prefers-reduced-motion:reduce){.progress-bar-animated{-webkit-animation:none;animation:none}}.media{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start}.media-body{-ms-flex:1;flex:1}.list-group{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;border-radius:.25rem}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.75rem 1.25rem;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.list-group-item:last-child{border-bottom-right-radius:inherit;border-bottom-left-radius:inherit}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#007bff;border-color:#007bff}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:-1px;border-top-width:1px}.list-group-horizontal{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}@media (min-width:576px){.list-group-horizontal-sm{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-sm>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:768px){.list-group-horizontal-md{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-md>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:992px){.list-group-horizontal-lg{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-lg>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1200px){.list-group-horizontal-xl{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-xl>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}.list-group-flush{border-radius:0}.list-group-flush>.list-group-item{border-width:0 0 1px}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{color:#004085;background-color:#b8daff}.list-group-item-primary.list-group-item-action:focus,.list-group-item-primary.list-group-item-action:hover{color:#004085;background-color:#9fcdff}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#004085;border-color:#004085}.list-group-item-secondary{color:#383d41;background-color:#d6d8db}.list-group-item-secondary.list-group-item-action:focus,.list-group-item-secondary.list-group-item-action:hover{color:#383d41;background-color:#c8cbcf}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#383d41;border-color:#383d41}.list-group-item-success{color:#155724;background-color:#c3e6cb}.list-group-item-success.list-group-item-action:focus,.list-group-item-success.list-group-item-action:hover{color:#155724;background-color:#b1dfbb}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#155724;border-color:#155724}.list-group-item-info{color:#0c5460;background-color:#bee5eb}.list-group-item-info.list-group-item-action:focus,.list-group-item-info.list-group-item-action:hover{color:#0c5460;background-color:#abdde5}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#0c5460;border-color:#0c5460}.list-group-item-warning{color:#856404;background-color:#ffeeba}.list-group-item-warning.list-group-item-action:focus,.list-group-item-warning.list-group-item-action:hover{color:#856404;background-color:#ffe8a1}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#856404;border-color:#856404}.list-group-item-danger{color:#721c24;background-color:#f5c6cb}.list-group-item-danger.list-group-item-action:focus,.list-group-item-danger.list-group-item-action:hover{color:#721c24;background-color:#f1b0b7}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#721c24;border-color:#721c24}.list-group-item-light{color:#818182;background-color:#fdfdfe}.list-group-item-light.list-group-item-action:focus,.list-group-item-light.list-group-item-action:hover{color:#818182;background-color:#ececf6}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}.list-group-item-dark.list-group-item-action:focus,.list-group-item-dark.list-group-item-action:hover{color:#1b1e21;background-color:#b9bbbe}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:hover{color:#000;text-decoration:none}.close:not(:disabled):not(.disabled):focus,.close:not(:disabled):not(.disabled):hover{opacity:.75}button.close{padding:0;background-color:transparent;border:0}a.close.disabled{pointer-events:none}.toast{max-width:350px;overflow:hidden;font-size:.875rem;background-color:rgba(255,255,255,.85);background-clip:padding-box;border:1px solid rgba(0,0,0,.1);box-shadow:0 .25rem .75rem rgba(0,0,0,.1);-webkit-backdrop-filter:blur(10px);backdrop-filter:blur(10px);opacity:0;border-radius:.25rem}.toast:not(:last-child){margin-bottom:.75rem}.toast.showing{opacity:1}.toast.show{display:block;opacity:1}.toast.hide{display:none}.toast-header{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.25rem .75rem;color:#6c757d;background-color:rgba(255,255,255,.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,.05)}.toast-body{padding:.75rem}.modal-open{overflow:hidden}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal{position:fixed;top:0;left:0;z-index:1050;display:none;width:100%;height:100%;overflow:hidden;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:-webkit-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out;-webkit-transform:translate(0,-50px);transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{-webkit-transform:none;transform:none}.modal.modal-static .modal-dialog{-webkit-transform:scale(1.02);transform:scale(1.02)}.modal-dialog-scrollable{display:-ms-flexbox;display:flex;max-height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 1rem);overflow:hidden}.modal-dialog-scrollable .modal-footer,.modal-dialog-scrollable .modal-header{-ms-flex-negative:0;flex-shrink:0}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;min-height:calc(100% - 1rem)}.modal-dialog-centered::before{display:block;height:calc(100vh - 1rem);height:-webkit-min-content;height:-moz-min-content;height:min-content;content:""}.modal-dialog-centered.modal-dialog-scrollable{-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;height:100%}.modal-dialog-centered.modal-dialog-scrollable .modal-content{max-height:none}.modal-dialog-centered.modal-dialog-scrollable::before{content:none}.modal-content{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:justify;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.modal-header .close{padding:1rem 1rem;margin:-1rem -1rem -1rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;padding:1rem}.modal-footer{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:end;justify-content:flex-end;padding:.75rem;border-top:1px solid #dee2e6;border-bottom-right-radius:calc(.3rem - 1px);border-bottom-left-radius:calc(.3rem - 1px)}.modal-footer>*{margin:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{max-height:calc(100% - 3.5rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-dialog-centered::before{height:calc(100vh - 3.5rem);height:-webkit-min-content;height:-moz-min-content;height:min-content}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width:1200px){.modal-xl{max-width:1140px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[x-placement^=top],.bs-tooltip-top{padding:.4rem 0}.bs-tooltip-auto[x-placement^=top] .arrow,.bs-tooltip-top .arrow{bottom:0}.bs-tooltip-auto[x-placement^=top] .arrow::before,.bs-tooltip-top .arrow::before{top:0;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-auto[x-placement^=right],.bs-tooltip-right{padding:0 .4rem}.bs-tooltip-auto[x-placement^=right] .arrow,.bs-tooltip-right .arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=right] .arrow::before,.bs-tooltip-right .arrow::before{right:0;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-auto[x-placement^=bottom],.bs-tooltip-bottom{padding:.4rem 0}.bs-tooltip-auto[x-placement^=bottom] .arrow,.bs-tooltip-bottom .arrow{top:0}.bs-tooltip-auto[x-placement^=bottom] .arrow::before,.bs-tooltip-bottom .arrow::before{bottom:0;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-auto[x-placement^=left],.bs-tooltip-left{padding:0 .4rem}.bs-tooltip-auto[x-placement^=left] .arrow,.bs-tooltip-left .arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=left] .arrow::before,.bs-tooltip-left .arrow::before{left:0;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .arrow{position:absolute;display:block;width:1rem;height:.5rem;margin:0 .3rem}.popover .arrow::after,.popover .arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-auto[x-placement^=top],.bs-popover-top{margin-bottom:.5rem}.bs-popover-auto[x-placement^=top]>.arrow,.bs-popover-top>.arrow{bottom:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=top]>.arrow::before,.bs-popover-top>.arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=top]>.arrow::after,.bs-popover-top>.arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-auto[x-placement^=right],.bs-popover-right{margin-left:.5rem}.bs-popover-auto[x-placement^=right]>.arrow,.bs-popover-right>.arrow{left:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=right]>.arrow::before,.bs-popover-right>.arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=right]>.arrow::after,.bs-popover-right>.arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-auto[x-placement^=bottom],.bs-popover-bottom{margin-top:.5rem}.bs-popover-auto[x-placement^=bottom]>.arrow,.bs-popover-bottom>.arrow{top:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=bottom]>.arrow::before,.bs-popover-bottom>.arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=bottom]>.arrow::after,.bs-popover-bottom>.arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-auto[x-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f7f7f7}.bs-popover-auto[x-placement^=left],.bs-popover-left{margin-right:.5rem}.bs-popover-auto[x-placement^=left]>.arrow,.bs-popover-left>.arrow{right:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=left]>.arrow::before,.bs-popover-left>.arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=left]>.arrow::after,.bs-popover-left>.arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:.5rem .75rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{-ms-touch-action:pan-y;touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:-webkit-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-right,.carousel-item-next:not(.carousel-item-left){-webkit-transform:translateX(100%);transform:translateX(100%)}.active.carousel-item-left,.carousel-item-prev:not(.carousel-item-right){-webkit-transform:translateX(-100%);transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;-webkit-transform:none;transform:none}.carousel-fade .carousel-item-next.carousel-item-left,.carousel-fade .carousel-item-prev.carousel-item-right,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:no-repeat 50%/100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5L4.25 4l2.5-2.5L5.25 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5L3.75 4l-2.5 2.5L2.75 8l4-4-4-4z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:15;display:-ms-flexbox;display:flex;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{box-sizing:content-box;-ms-flex:0 1 auto;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators li{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}@-webkit-keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;-webkit-animation:spinner-border .75s linear infinite;animation:spinner-border .75s linear infinite}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@-webkit-keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1;-webkit-transform:none;transform:none}}@keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1;-webkit-transform:none;transform:none}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;background-color:currentColor;border-radius:50%;opacity:0;-webkit-animation:spinner-grow .75s linear infinite;animation:spinner-grow .75s linear infinite}.spinner-grow-sm{width:1rem;height:1rem}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-primary{background-color:#007bff!important}a.bg-primary:focus,a.bg-primary:hover,button.bg-primary:focus,button.bg-primary:hover{background-color:#0062cc!important}.bg-secondary{background-color:#6c757d!important}a.bg-secondary:focus,a.bg-secondary:hover,button.bg-secondary:focus,button.bg-secondary:hover{background-color:#545b62!important}.bg-success{background-color:#28a745!important}a.bg-success:focus,a.bg-success:hover,button.bg-success:focus,button.bg-success:hover{background-color:#1e7e34!important}.bg-info{background-color:#17a2b8!important}a.bg-info:focus,a.bg-info:hover,button.bg-info:focus,button.bg-info:hover{background-color:#117a8b!important}.bg-warning{background-color:#ffc107!important}a.bg-warning:focus,a.bg-warning:hover,button.bg-warning:focus,button.bg-warning:hover{background-color:#d39e00!important}.bg-danger{background-color:#dc3545!important}a.bg-danger:focus,a.bg-danger:hover,button.bg-danger:focus,button.bg-danger:hover{background-color:#bd2130!important}.bg-light{background-color:#f8f9fa!important}a.bg-light:focus,a.bg-light:hover,button.bg-light:focus,button.bg-light:hover{background-color:#dae0e5!important}.bg-dark{background-color:#343a40!important}a.bg-dark:focus,a.bg-dark:hover,button.bg-dark:focus,button.bg-dark:hover{background-color:#1d2124!important}.bg-white{background-color:#fff!important}.bg-transparent{background-color:transparent!important}.border{border:1px solid #dee2e6!important}.border-top{border-top:1px solid #dee2e6!important}.border-right{border-right:1px solid #dee2e6!important}.border-bottom{border-bottom:1px solid #dee2e6!important}.border-left{border-left:1px solid #dee2e6!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.border-primary{border-color:#007bff!important}.border-secondary{border-color:#6c757d!important}.border-success{border-color:#28a745!important}.border-info{border-color:#17a2b8!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#343a40!important}.border-white{border-color:#fff!important}.rounded-sm{border-radius:.2rem!important}.rounded{border-radius:.25rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-right{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-left{border-top-left-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-lg{border-radius:.3rem!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:50rem!important}.rounded-0{border-radius:0!important}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media print{.d-print-none{display:none!important}.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:-ms-flexbox!important;display:flex!important}.d-print-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-sm-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-sm-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-sm-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-sm-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-sm-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-sm-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-sm-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-sm-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-md-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-md-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-md-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-md-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-md-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-md-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-md-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-md-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-lg-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-lg-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-lg-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-lg-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-lg-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-lg-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-lg-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-lg-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-xl-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-xl-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-xl-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-xl-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-xl-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-xl-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-xl-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-xl-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.user-select-all{-webkit-user-select:all!important;-moz-user-select:all!important;-ms-user-select:all!important;user-select:all!important}.user-select-auto{-webkit-user-select:auto!important;-moz-user-select:auto!important;-ms-user-select:auto!important;user-select:auto!important}.user-select-none{-webkit-user-select:none!important;-moz-user-select:none!important;-ms-user-select:none!important;user-select:none!important}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports ((position:-webkit-sticky) or (position:sticky)){.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075)!important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15)!important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175)!important}.shadow-none{box-shadow:none!important}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.min-vw-100{min-width:100vw!important}.min-vh-100{min-height:100vh!important}.vw-100{width:100vw!important}.vh-100{height:100vh!important}.m-0{margin:0!important}.mt-0,.my-0{margin-top:0!important}.mr-0,.mx-0{margin-right:0!important}.mb-0,.my-0{margin-bottom:0!important}.ml-0,.mx-0{margin-left:0!important}.m-1{margin:.25rem!important}.mt-1,.my-1{margin-top:.25rem!important}.mr-1,.mx-1{margin-right:.25rem!important}.mb-1,.my-1{margin-bottom:.25rem!important}.ml-1,.mx-1{margin-left:.25rem!important}.m-2{margin:.5rem!important}.mt-2,.my-2{margin-top:.5rem!important}.mr-2,.mx-2{margin-right:.5rem!important}.mb-2,.my-2{margin-bottom:.5rem!important}.ml-2,.mx-2{margin-left:.5rem!important}.m-3{margin:1rem!important}.mt-3,.my-3{margin-top:1rem!important}.mr-3,.mx-3{margin-right:1rem!important}.mb-3,.my-3{margin-bottom:1rem!important}.ml-3,.mx-3{margin-left:1rem!important}.m-4{margin:1.5rem!important}.mt-4,.my-4{margin-top:1.5rem!important}.mr-4,.mx-4{margin-right:1.5rem!important}.mb-4,.my-4{margin-bottom:1.5rem!important}.ml-4,.mx-4{margin-left:1.5rem!important}.m-5{margin:3rem!important}.mt-5,.my-5{margin-top:3rem!important}.mr-5,.mx-5{margin-right:3rem!important}.mb-5,.my-5{margin-bottom:3rem!important}.ml-5,.mx-5{margin-left:3rem!important}.p-0{padding:0!important}.pt-0,.py-0{padding-top:0!important}.pr-0,.px-0{padding-right:0!important}.pb-0,.py-0{padding-bottom:0!important}.pl-0,.px-0{padding-left:0!important}.p-1{padding:.25rem!important}.pt-1,.py-1{padding-top:.25rem!important}.pr-1,.px-1{padding-right:.25rem!important}.pb-1,.py-1{padding-bottom:.25rem!important}.pl-1,.px-1{padding-left:.25rem!important}.p-2{padding:.5rem!important}.pt-2,.py-2{padding-top:.5rem!important}.pr-2,.px-2{padding-right:.5rem!important}.pb-2,.py-2{padding-bottom:.5rem!important}.pl-2,.px-2{padding-left:.5rem!important}.p-3{padding:1rem!important}.pt-3,.py-3{padding-top:1rem!important}.pr-3,.px-3{padding-right:1rem!important}.pb-3,.py-3{padding-bottom:1rem!important}.pl-3,.px-3{padding-left:1rem!important}.p-4{padding:1.5rem!important}.pt-4,.py-4{padding-top:1.5rem!important}.pr-4,.px-4{padding-right:1.5rem!important}.pb-4,.py-4{padding-bottom:1.5rem!important}.pl-4,.px-4{padding-left:1.5rem!important}.p-5{padding:3rem!important}.pt-5,.py-5{padding-top:3rem!important}.pr-5,.px-5{padding-right:3rem!important}.pb-5,.py-5{padding-bottom:3rem!important}.pl-5,.px-5{padding-left:3rem!important}.m-n1{margin:-.25rem!important}.mt-n1,.my-n1{margin-top:-.25rem!important}.mr-n1,.mx-n1{margin-right:-.25rem!important}.mb-n1,.my-n1{margin-bottom:-.25rem!important}.ml-n1,.mx-n1{margin-left:-.25rem!important}.m-n2{margin:-.5rem!important}.mt-n2,.my-n2{margin-top:-.5rem!important}.mr-n2,.mx-n2{margin-right:-.5rem!important}.mb-n2,.my-n2{margin-bottom:-.5rem!important}.ml-n2,.mx-n2{margin-left:-.5rem!important}.m-n3{margin:-1rem!important}.mt-n3,.my-n3{margin-top:-1rem!important}.mr-n3,.mx-n3{margin-right:-1rem!important}.mb-n3,.my-n3{margin-bottom:-1rem!important}.ml-n3,.mx-n3{margin-left:-1rem!important}.m-n4{margin:-1.5rem!important}.mt-n4,.my-n4{margin-top:-1.5rem!important}.mr-n4,.mx-n4{margin-right:-1.5rem!important}.mb-n4,.my-n4{margin-bottom:-1.5rem!important}.ml-n4,.mx-n4{margin-left:-1.5rem!important}.m-n5{margin:-3rem!important}.mt-n5,.my-n5{margin-top:-3rem!important}.mr-n5,.mx-n5{margin-right:-3rem!important}.mb-n5,.my-n5{margin-bottom:-3rem!important}.ml-n5,.mx-n5{margin-left:-3rem!important}.m-auto{margin:auto!important}.mt-auto,.my-auto{margin-top:auto!important}.mr-auto,.mx-auto{margin-right:auto!important}.mb-auto,.my-auto{margin-bottom:auto!important}.ml-auto,.mx-auto{margin-left:auto!important}@media (min-width:576px){.m-sm-0{margin:0!important}.mt-sm-0,.my-sm-0{margin-top:0!important}.mr-sm-0,.mx-sm-0{margin-right:0!important}.mb-sm-0,.my-sm-0{margin-bottom:0!important}.ml-sm-0,.mx-sm-0{margin-left:0!important}.m-sm-1{margin:.25rem!important}.mt-sm-1,.my-sm-1{margin-top:.25rem!important}.mr-sm-1,.mx-sm-1{margin-right:.25rem!important}.mb-sm-1,.my-sm-1{margin-bottom:.25rem!important}.ml-sm-1,.mx-sm-1{margin-left:.25rem!important}.m-sm-2{margin:.5rem!important}.mt-sm-2,.my-sm-2{margin-top:.5rem!important}.mr-sm-2,.mx-sm-2{margin-right:.5rem!important}.mb-sm-2,.my-sm-2{margin-bottom:.5rem!important}.ml-sm-2,.mx-sm-2{margin-left:.5rem!important}.m-sm-3{margin:1rem!important}.mt-sm-3,.my-sm-3{margin-top:1rem!important}.mr-sm-3,.mx-sm-3{margin-right:1rem!important}.mb-sm-3,.my-sm-3{margin-bottom:1rem!important}.ml-sm-3,.mx-sm-3{margin-left:1rem!important}.m-sm-4{margin:1.5rem!important}.mt-sm-4,.my-sm-4{margin-top:1.5rem!important}.mr-sm-4,.mx-sm-4{margin-right:1.5rem!important}.mb-sm-4,.my-sm-4{margin-bottom:1.5rem!important}.ml-sm-4,.mx-sm-4{margin-left:1.5rem!important}.m-sm-5{margin:3rem!important}.mt-sm-5,.my-sm-5{margin-top:3rem!important}.mr-sm-5,.mx-sm-5{margin-right:3rem!important}.mb-sm-5,.my-sm-5{margin-bottom:3rem!important}.ml-sm-5,.mx-sm-5{margin-left:3rem!important}.p-sm-0{padding:0!important}.pt-sm-0,.py-sm-0{padding-top:0!important}.pr-sm-0,.px-sm-0{padding-right:0!important}.pb-sm-0,.py-sm-0{padding-bottom:0!important}.pl-sm-0,.px-sm-0{padding-left:0!important}.p-sm-1{padding:.25rem!important}.pt-sm-1,.py-sm-1{padding-top:.25rem!important}.pr-sm-1,.px-sm-1{padding-right:.25rem!important}.pb-sm-1,.py-sm-1{padding-bottom:.25rem!important}.pl-sm-1,.px-sm-1{padding-left:.25rem!important}.p-sm-2{padding:.5rem!important}.pt-sm-2,.py-sm-2{padding-top:.5rem!important}.pr-sm-2,.px-sm-2{padding-right:.5rem!important}.pb-sm-2,.py-sm-2{padding-bottom:.5rem!important}.pl-sm-2,.px-sm-2{padding-left:.5rem!important}.p-sm-3{padding:1rem!important}.pt-sm-3,.py-sm-3{padding-top:1rem!important}.pr-sm-3,.px-sm-3{padding-right:1rem!important}.pb-sm-3,.py-sm-3{padding-bottom:1rem!important}.pl-sm-3,.px-sm-3{padding-left:1rem!important}.p-sm-4{padding:1.5rem!important}.pt-sm-4,.py-sm-4{padding-top:1.5rem!important}.pr-sm-4,.px-sm-4{padding-right:1.5rem!important}.pb-sm-4,.py-sm-4{padding-bottom:1.5rem!important}.pl-sm-4,.px-sm-4{padding-left:1.5rem!important}.p-sm-5{padding:3rem!important}.pt-sm-5,.py-sm-5{padding-top:3rem!important}.pr-sm-5,.px-sm-5{padding-right:3rem!important}.pb-sm-5,.py-sm-5{padding-bottom:3rem!important}.pl-sm-5,.px-sm-5{padding-left:3rem!important}.m-sm-n1{margin:-.25rem!important}.mt-sm-n1,.my-sm-n1{margin-top:-.25rem!important}.mr-sm-n1,.mx-sm-n1{margin-right:-.25rem!important}.mb-sm-n1,.my-sm-n1{margin-bottom:-.25rem!important}.ml-sm-n1,.mx-sm-n1{margin-left:-.25rem!important}.m-sm-n2{margin:-.5rem!important}.mt-sm-n2,.my-sm-n2{margin-top:-.5rem!important}.mr-sm-n2,.mx-sm-n2{margin-right:-.5rem!important}.mb-sm-n2,.my-sm-n2{margin-bottom:-.5rem!important}.ml-sm-n2,.mx-sm-n2{margin-left:-.5rem!important}.m-sm-n3{margin:-1rem!important}.mt-sm-n3,.my-sm-n3{margin-top:-1rem!important}.mr-sm-n3,.mx-sm-n3{margin-right:-1rem!important}.mb-sm-n3,.my-sm-n3{margin-bottom:-1rem!important}.ml-sm-n3,.mx-sm-n3{margin-left:-1rem!important}.m-sm-n4{margin:-1.5rem!important}.mt-sm-n4,.my-sm-n4{margin-top:-1.5rem!important}.mr-sm-n4,.mx-sm-n4{margin-right:-1.5rem!important}.mb-sm-n4,.my-sm-n4{margin-bottom:-1.5rem!important}.ml-sm-n4,.mx-sm-n4{margin-left:-1.5rem!important}.m-sm-n5{margin:-3rem!important}.mt-sm-n5,.my-sm-n5{margin-top:-3rem!important}.mr-sm-n5,.mx-sm-n5{margin-right:-3rem!important}.mb-sm-n5,.my-sm-n5{margin-bottom:-3rem!important}.ml-sm-n5,.mx-sm-n5{margin-left:-3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto,.my-sm-auto{margin-top:auto!important}.mr-sm-auto,.mx-sm-auto{margin-right:auto!important}.mb-sm-auto,.my-sm-auto{margin-bottom:auto!important}.ml-sm-auto,.mx-sm-auto{margin-left:auto!important}}@media (min-width:768px){.m-md-0{margin:0!important}.mt-md-0,.my-md-0{margin-top:0!important}.mr-md-0,.mx-md-0{margin-right:0!important}.mb-md-0,.my-md-0{margin-bottom:0!important}.ml-md-0,.mx-md-0{margin-left:0!important}.m-md-1{margin:.25rem!important}.mt-md-1,.my-md-1{margin-top:.25rem!important}.mr-md-1,.mx-md-1{margin-right:.25rem!important}.mb-md-1,.my-md-1{margin-bottom:.25rem!important}.ml-md-1,.mx-md-1{margin-left:.25rem!important}.m-md-2{margin:.5rem!important}.mt-md-2,.my-md-2{margin-top:.5rem!important}.mr-md-2,.mx-md-2{margin-right:.5rem!important}.mb-md-2,.my-md-2{margin-bottom:.5rem!important}.ml-md-2,.mx-md-2{margin-left:.5rem!important}.m-md-3{margin:1rem!important}.mt-md-3,.my-md-3{margin-top:1rem!important}.mr-md-3,.mx-md-3{margin-right:1rem!important}.mb-md-3,.my-md-3{margin-bottom:1rem!important}.ml-md-3,.mx-md-3{margin-left:1rem!important}.m-md-4{margin:1.5rem!important}.mt-md-4,.my-md-4{margin-top:1.5rem!important}.mr-md-4,.mx-md-4{margin-right:1.5rem!important}.mb-md-4,.my-md-4{margin-bottom:1.5rem!important}.ml-md-4,.mx-md-4{margin-left:1.5rem!important}.m-md-5{margin:3rem!important}.mt-md-5,.my-md-5{margin-top:3rem!important}.mr-md-5,.mx-md-5{margin-right:3rem!important}.mb-md-5,.my-md-5{margin-bottom:3rem!important}.ml-md-5,.mx-md-5{margin-left:3rem!important}.p-md-0{padding:0!important}.pt-md-0,.py-md-0{padding-top:0!important}.pr-md-0,.px-md-0{padding-right:0!important}.pb-md-0,.py-md-0{padding-bottom:0!important}.pl-md-0,.px-md-0{padding-left:0!important}.p-md-1{padding:.25rem!important}.pt-md-1,.py-md-1{padding-top:.25rem!important}.pr-md-1,.px-md-1{padding-right:.25rem!important}.pb-md-1,.py-md-1{padding-bottom:.25rem!important}.pl-md-1,.px-md-1{padding-left:.25rem!important}.p-md-2{padding:.5rem!important}.pt-md-2,.py-md-2{padding-top:.5rem!important}.pr-md-2,.px-md-2{padding-right:.5rem!important}.pb-md-2,.py-md-2{padding-bottom:.5rem!important}.pl-md-2,.px-md-2{padding-left:.5rem!important}.p-md-3{padding:1rem!important}.pt-md-3,.py-md-3{padding-top:1rem!important}.pr-md-3,.px-md-3{padding-right:1rem!important}.pb-md-3,.py-md-3{padding-bottom:1rem!important}.pl-md-3,.px-md-3{padding-left:1rem!important}.p-md-4{padding:1.5rem!important}.pt-md-4,.py-md-4{padding-top:1.5rem!important}.pr-md-4,.px-md-4{padding-right:1.5rem!important}.pb-md-4,.py-md-4{padding-bottom:1.5rem!important}.pl-md-4,.px-md-4{padding-left:1.5rem!important}.p-md-5{padding:3rem!important}.pt-md-5,.py-md-5{padding-top:3rem!important}.pr-md-5,.px-md-5{padding-right:3rem!important}.pb-md-5,.py-md-5{padding-bottom:3rem!important}.pl-md-5,.px-md-5{padding-left:3rem!important}.m-md-n1{margin:-.25rem!important}.mt-md-n1,.my-md-n1{margin-top:-.25rem!important}.mr-md-n1,.mx-md-n1{margin-right:-.25rem!important}.mb-md-n1,.my-md-n1{margin-bottom:-.25rem!important}.ml-md-n1,.mx-md-n1{margin-left:-.25rem!important}.m-md-n2{margin:-.5rem!important}.mt-md-n2,.my-md-n2{margin-top:-.5rem!important}.mr-md-n2,.mx-md-n2{margin-right:-.5rem!important}.mb-md-n2,.my-md-n2{margin-bottom:-.5rem!important}.ml-md-n2,.mx-md-n2{margin-left:-.5rem!important}.m-md-n3{margin:-1rem!important}.mt-md-n3,.my-md-n3{margin-top:-1rem!important}.mr-md-n3,.mx-md-n3{margin-right:-1rem!important}.mb-md-n3,.my-md-n3{margin-bottom:-1rem!important}.ml-md-n3,.mx-md-n3{margin-left:-1rem!important}.m-md-n4{margin:-1.5rem!important}.mt-md-n4,.my-md-n4{margin-top:-1.5rem!important}.mr-md-n4,.mx-md-n4{margin-right:-1.5rem!important}.mb-md-n4,.my-md-n4{margin-bottom:-1.5rem!important}.ml-md-n4,.mx-md-n4{margin-left:-1.5rem!important}.m-md-n5{margin:-3rem!important}.mt-md-n5,.my-md-n5{margin-top:-3rem!important}.mr-md-n5,.mx-md-n5{margin-right:-3rem!important}.mb-md-n5,.my-md-n5{margin-bottom:-3rem!important}.ml-md-n5,.mx-md-n5{margin-left:-3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto,.my-md-auto{margin-top:auto!important}.mr-md-auto,.mx-md-auto{margin-right:auto!important}.mb-md-auto,.my-md-auto{margin-bottom:auto!important}.ml-md-auto,.mx-md-auto{margin-left:auto!important}}@media (min-width:992px){.m-lg-0{margin:0!important}.mt-lg-0,.my-lg-0{margin-top:0!important}.mr-lg-0,.mx-lg-0{margin-right:0!important}.mb-lg-0,.my-lg-0{margin-bottom:0!important}.ml-lg-0,.mx-lg-0{margin-left:0!important}.m-lg-1{margin:.25rem!important}.mt-lg-1,.my-lg-1{margin-top:.25rem!important}.mr-lg-1,.mx-lg-1{margin-right:.25rem!important}.mb-lg-1,.my-lg-1{margin-bottom:.25rem!important}.ml-lg-1,.mx-lg-1{margin-left:.25rem!important}.m-lg-2{margin:.5rem!important}.mt-lg-2,.my-lg-2{margin-top:.5rem!important}.mr-lg-2,.mx-lg-2{margin-right:.5rem!important}.mb-lg-2,.my-lg-2{margin-bottom:.5rem!important}.ml-lg-2,.mx-lg-2{margin-left:.5rem!important}.m-lg-3{margin:1rem!important}.mt-lg-3,.my-lg-3{margin-top:1rem!important}.mr-lg-3,.mx-lg-3{margin-right:1rem!important}.mb-lg-3,.my-lg-3{margin-bottom:1rem!important}.ml-lg-3,.mx-lg-3{margin-left:1rem!important}.m-lg-4{margin:1.5rem!important}.mt-lg-4,.my-lg-4{margin-top:1.5rem!important}.mr-lg-4,.mx-lg-4{margin-right:1.5rem!important}.mb-lg-4,.my-lg-4{margin-bottom:1.5rem!important}.ml-lg-4,.mx-lg-4{margin-left:1.5rem!important}.m-lg-5{margin:3rem!important}.mt-lg-5,.my-lg-5{margin-top:3rem!important}.mr-lg-5,.mx-lg-5{margin-right:3rem!important}.mb-lg-5,.my-lg-5{margin-bottom:3rem!important}.ml-lg-5,.mx-lg-5{margin-left:3rem!important}.p-lg-0{padding:0!important}.pt-lg-0,.py-lg-0{padding-top:0!important}.pr-lg-0,.px-lg-0{padding-right:0!important}.pb-lg-0,.py-lg-0{padding-bottom:0!important}.pl-lg-0,.px-lg-0{padding-left:0!important}.p-lg-1{padding:.25rem!important}.pt-lg-1,.py-lg-1{padding-top:.25rem!important}.pr-lg-1,.px-lg-1{padding-right:.25rem!important}.pb-lg-1,.py-lg-1{padding-bottom:.25rem!important}.pl-lg-1,.px-lg-1{padding-left:.25rem!important}.p-lg-2{padding:.5rem!important}.pt-lg-2,.py-lg-2{padding-top:.5rem!important}.pr-lg-2,.px-lg-2{padding-right:.5rem!important}.pb-lg-2,.py-lg-2{padding-bottom:.5rem!important}.pl-lg-2,.px-lg-2{padding-left:.5rem!important}.p-lg-3{padding:1rem!important}.pt-lg-3,.py-lg-3{padding-top:1rem!important}.pr-lg-3,.px-lg-3{padding-right:1rem!important}.pb-lg-3,.py-lg-3{padding-bottom:1rem!important}.pl-lg-3,.px-lg-3{padding-left:1rem!important}.p-lg-4{padding:1.5rem!important}.pt-lg-4,.py-lg-4{padding-top:1.5rem!important}.pr-lg-4,.px-lg-4{padding-right:1.5rem!important}.pb-lg-4,.py-lg-4{padding-bottom:1.5rem!important}.pl-lg-4,.px-lg-4{padding-left:1.5rem!important}.p-lg-5{padding:3rem!important}.pt-lg-5,.py-lg-5{padding-top:3rem!important}.pr-lg-5,.px-lg-5{padding-right:3rem!important}.pb-lg-5,.py-lg-5{padding-bottom:3rem!important}.pl-lg-5,.px-lg-5{padding-left:3rem!important}.m-lg-n1{margin:-.25rem!important}.mt-lg-n1,.my-lg-n1{margin-top:-.25rem!important}.mr-lg-n1,.mx-lg-n1{margin-right:-.25rem!important}.mb-lg-n1,.my-lg-n1{margin-bottom:-.25rem!important}.ml-lg-n1,.mx-lg-n1{margin-left:-.25rem!important}.m-lg-n2{margin:-.5rem!important}.mt-lg-n2,.my-lg-n2{margin-top:-.5rem!important}.mr-lg-n2,.mx-lg-n2{margin-right:-.5rem!important}.mb-lg-n2,.my-lg-n2{margin-bottom:-.5rem!important}.ml-lg-n2,.mx-lg-n2{margin-left:-.5rem!important}.m-lg-n3{margin:-1rem!important}.mt-lg-n3,.my-lg-n3{margin-top:-1rem!important}.mr-lg-n3,.mx-lg-n3{margin-right:-1rem!important}.mb-lg-n3,.my-lg-n3{margin-bottom:-1rem!important}.ml-lg-n3,.mx-lg-n3{margin-left:-1rem!important}.m-lg-n4{margin:-1.5rem!important}.mt-lg-n4,.my-lg-n4{margin-top:-1.5rem!important}.mr-lg-n4,.mx-lg-n4{margin-right:-1.5rem!important}.mb-lg-n4,.my-lg-n4{margin-bottom:-1.5rem!important}.ml-lg-n4,.mx-lg-n4{margin-left:-1.5rem!important}.m-lg-n5{margin:-3rem!important}.mt-lg-n5,.my-lg-n5{margin-top:-3rem!important}.mr-lg-n5,.mx-lg-n5{margin-right:-3rem!important}.mb-lg-n5,.my-lg-n5{margin-bottom:-3rem!important}.ml-lg-n5,.mx-lg-n5{margin-left:-3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto,.my-lg-auto{margin-top:auto!important}.mr-lg-auto,.mx-lg-auto{margin-right:auto!important}.mb-lg-auto,.my-lg-auto{margin-bottom:auto!important}.ml-lg-auto,.mx-lg-auto{margin-left:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0!important}.mt-xl-0,.my-xl-0{margin-top:0!important}.mr-xl-0,.mx-xl-0{margin-right:0!important}.mb-xl-0,.my-xl-0{margin-bottom:0!important}.ml-xl-0,.mx-xl-0{margin-left:0!important}.m-xl-1{margin:.25rem!important}.mt-xl-1,.my-xl-1{margin-top:.25rem!important}.mr-xl-1,.mx-xl-1{margin-right:.25rem!important}.mb-xl-1,.my-xl-1{margin-bottom:.25rem!important}.ml-xl-1,.mx-xl-1{margin-left:.25rem!important}.m-xl-2{margin:.5rem!important}.mt-xl-2,.my-xl-2{margin-top:.5rem!important}.mr-xl-2,.mx-xl-2{margin-right:.5rem!important}.mb-xl-2,.my-xl-2{margin-bottom:.5rem!important}.ml-xl-2,.mx-xl-2{margin-left:.5rem!important}.m-xl-3{margin:1rem!important}.mt-xl-3,.my-xl-3{margin-top:1rem!important}.mr-xl-3,.mx-xl-3{margin-right:1rem!important}.mb-xl-3,.my-xl-3{margin-bottom:1rem!important}.ml-xl-3,.mx-xl-3{margin-left:1rem!important}.m-xl-4{margin:1.5rem!important}.mt-xl-4,.my-xl-4{margin-top:1.5rem!important}.mr-xl-4,.mx-xl-4{margin-right:1.5rem!important}.mb-xl-4,.my-xl-4{margin-bottom:1.5rem!important}.ml-xl-4,.mx-xl-4{margin-left:1.5rem!important}.m-xl-5{margin:3rem!important}.mt-xl-5,.my-xl-5{margin-top:3rem!important}.mr-xl-5,.mx-xl-5{margin-right:3rem!important}.mb-xl-5,.my-xl-5{margin-bottom:3rem!important}.ml-xl-5,.mx-xl-5{margin-left:3rem!important}.p-xl-0{padding:0!important}.pt-xl-0,.py-xl-0{padding-top:0!important}.pr-xl-0,.px-xl-0{padding-right:0!important}.pb-xl-0,.py-xl-0{padding-bottom:0!important}.pl-xl-0,.px-xl-0{padding-left:0!important}.p-xl-1{padding:.25rem!important}.pt-xl-1,.py-xl-1{padding-top:.25rem!important}.pr-xl-1,.px-xl-1{padding-right:.25rem!important}.pb-xl-1,.py-xl-1{padding-bottom:.25rem!important}.pl-xl-1,.px-xl-1{padding-left:.25rem!important}.p-xl-2{padding:.5rem!important}.pt-xl-2,.py-xl-2{padding-top:.5rem!important}.pr-xl-2,.px-xl-2{padding-right:.5rem!important}.pb-xl-2,.py-xl-2{padding-bottom:.5rem!important}.pl-xl-2,.px-xl-2{padding-left:.5rem!important}.p-xl-3{padding:1rem!important}.pt-xl-3,.py-xl-3{padding-top:1rem!important}.pr-xl-3,.px-xl-3{padding-right:1rem!important}.pb-xl-3,.py-xl-3{padding-bottom:1rem!important}.pl-xl-3,.px-xl-3{padding-left:1rem!important}.p-xl-4{padding:1.5rem!important}.pt-xl-4,.py-xl-4{padding-top:1.5rem!important}.pr-xl-4,.px-xl-4{padding-right:1.5rem!important}.pb-xl-4,.py-xl-4{padding-bottom:1.5rem!important}.pl-xl-4,.px-xl-4{padding-left:1.5rem!important}.p-xl-5{padding:3rem!important}.pt-xl-5,.py-xl-5{padding-top:3rem!important}.pr-xl-5,.px-xl-5{padding-right:3rem!important}.pb-xl-5,.py-xl-5{padding-bottom:3rem!important}.pl-xl-5,.px-xl-5{padding-left:3rem!important}.m-xl-n1{margin:-.25rem!important}.mt-xl-n1,.my-xl-n1{margin-top:-.25rem!important}.mr-xl-n1,.mx-xl-n1{margin-right:-.25rem!important}.mb-xl-n1,.my-xl-n1{margin-bottom:-.25rem!important}.ml-xl-n1,.mx-xl-n1{margin-left:-.25rem!important}.m-xl-n2{margin:-.5rem!important}.mt-xl-n2,.my-xl-n2{margin-top:-.5rem!important}.mr-xl-n2,.mx-xl-n2{margin-right:-.5rem!important}.mb-xl-n2,.my-xl-n2{margin-bottom:-.5rem!important}.ml-xl-n2,.mx-xl-n2{margin-left:-.5rem!important}.m-xl-n3{margin:-1rem!important}.mt-xl-n3,.my-xl-n3{margin-top:-1rem!important}.mr-xl-n3,.mx-xl-n3{margin-right:-1rem!important}.mb-xl-n3,.my-xl-n3{margin-bottom:-1rem!important}.ml-xl-n3,.mx-xl-n3{margin-left:-1rem!important}.m-xl-n4{margin:-1.5rem!important}.mt-xl-n4,.my-xl-n4{margin-top:-1.5rem!important}.mr-xl-n4,.mx-xl-n4{margin-right:-1.5rem!important}.mb-xl-n4,.my-xl-n4{margin-bottom:-1.5rem!important}.ml-xl-n4,.mx-xl-n4{margin-left:-1.5rem!important}.m-xl-n5{margin:-3rem!important}.mt-xl-n5,.my-xl-n5{margin-top:-3rem!important}.mr-xl-n5,.mx-xl-n5{margin-right:-3rem!important}.mb-xl-n5,.my-xl-n5{margin-bottom:-3rem!important}.ml-xl-n5,.mx-xl-n5{margin-left:-3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto,.my-xl-auto{margin-top:auto!important}.mr-xl-auto,.mx-xl-auto{margin-right:auto!important}.mb-xl-auto,.my-xl-auto{margin-bottom:auto!important}.ml-xl-auto,.mx-xl-auto{margin-left:auto!important}}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;pointer-events:auto;content:"";background-color:rgba(0,0,0,0)}.text-monospace{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace!important}.text-justify{text-align:justify!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-light{font-weight:300!important}.font-weight-lighter{font-weight:lighter!important}.font-weight-normal{font-weight:400!important}.font-weight-bold{font-weight:700!important}.font-weight-bolder{font-weight:bolder!important}.font-italic{font-style:italic!important}.text-white{color:#fff!important}.text-primary{color:#007bff!important}a.text-primary:focus,a.text-primary:hover{color:#0056b3!important}.text-secondary{color:#6c757d!important}a.text-secondary:focus,a.text-secondary:hover{color:#494f54!important}.text-success{color:#28a745!important}a.text-success:focus,a.text-success:hover{color:#19692c!important}.text-info{color:#17a2b8!important}a.text-info:focus,a.text-info:hover{color:#0f6674!important}.text-warning{color:#ffc107!important}a.text-warning:focus,a.text-warning:hover{color:#ba8b00!important}.text-danger{color:#dc3545!important}a.text-danger:focus,a.text-danger:hover{color:#a71d2a!important}.text-light{color:#f8f9fa!important}a.text-light:focus,a.text-light:hover{color:#cbd3da!important}.text-dark{color:#343a40!important}a.text-dark:focus,a.text-dark:hover{color:#121416!important}.text-body{color:#212529!important}.text-muted{color:#6c757d!important}.text-black-50{color:rgba(0,0,0,.5)!important}.text-white-50{color:rgba(255,255,255,.5)!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.text-decoration-none{text-decoration:none!important}.text-break{word-wrap:break-word!important}.text-reset{color:inherit!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}@media print{*,::after,::before{text-shadow:none!important;box-shadow:none!important}a:not(.btn){text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap!important}blockquote,pre{border:1px solid #adb5bd;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}@page{size:a3}body{min-width:992px!important}.container{min-width:992px!important}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #dee2e6!important}.table-dark{color:inherit}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#dee2e6}.table .thead-dark th{color:inherit;border-color:#dee2e6}} +/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/iclient/iclientpkg/static-content/bootstrap.min.js b/iclient/iclientpkg/static-content/bootstrap.min.js new file mode 100644 index 00000000..3ecf55f2 --- /dev/null +++ b/iclient/iclientpkg/static-content/bootstrap.min.js @@ -0,0 +1,7 @@ +/*! + * Bootstrap v4.5.0 (https://getbootstrap.com/) + * Copyright 2011-2020 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports,require("jquery"),require("popper.js")):"function"==typeof define&&define.amd?define(["exports","jquery","popper.js"],e):e((t=t||self).bootstrap={},t.jQuery,t.Popper)}(this,(function(t,e,n){"use strict";function i(t,e){for(var n=0;n=4)throw new Error("Bootstrap's JavaScript requires at least jQuery v1.9.1 but less than v4.0.0")}};c.jQueryDetection(),e.fn.emulateTransitionEnd=l,e.event.special[c.TRANSITION_END]={bindType:"transitionend",delegateType:"transitionend",handle:function(t){if(e(t.target).is(this))return t.handleObj.handler.apply(this,arguments)}};var h="alert",u=e.fn[h],d=function(){function t(t){this._element=t}var n=t.prototype;return n.close=function(t){var e=this._element;t&&(e=this._getRootElement(t)),this._triggerCloseEvent(e).isDefaultPrevented()||this._removeElement(e)},n.dispose=function(){e.removeData(this._element,"bs.alert"),this._element=null},n._getRootElement=function(t){var n=c.getSelectorFromElement(t),i=!1;return n&&(i=document.querySelector(n)),i||(i=e(t).closest(".alert")[0]),i},n._triggerCloseEvent=function(t){var n=e.Event("close.bs.alert");return e(t).trigger(n),n},n._removeElement=function(t){var n=this;if(e(t).removeClass("show"),e(t).hasClass("fade")){var i=c.getTransitionDurationFromElement(t);e(t).one(c.TRANSITION_END,(function(e){return n._destroyElement(t,e)})).emulateTransitionEnd(i)}else this._destroyElement(t)},n._destroyElement=function(t){e(t).detach().trigger("closed.bs.alert").remove()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.alert");o||(o=new t(this),i.data("bs.alert",o)),"close"===n&&o[n](this)}))},t._handleDismiss=function(t){return function(e){e&&e.preventDefault(),t.close(this)}},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.alert.data-api",'[data-dismiss="alert"]',d._handleDismiss(new d)),e.fn[h]=d._jQueryInterface,e.fn[h].Constructor=d,e.fn[h].noConflict=function(){return e.fn[h]=u,d._jQueryInterface};var f=e.fn.button,g=function(){function t(t){this._element=t}var n=t.prototype;return n.toggle=function(){var t=!0,n=!0,i=e(this._element).closest('[data-toggle="buttons"]')[0];if(i){var o=this._element.querySelector('input:not([type="hidden"])');if(o){if("radio"===o.type)if(o.checked&&this._element.classList.contains("active"))t=!1;else{var s=i.querySelector(".active");s&&e(s).removeClass("active")}t&&("checkbox"!==o.type&&"radio"!==o.type||(o.checked=!this._element.classList.contains("active")),e(o).trigger("change")),o.focus(),n=!1}}this._element.hasAttribute("disabled")||this._element.classList.contains("disabled")||(n&&this._element.setAttribute("aria-pressed",!this._element.classList.contains("active")),t&&e(this._element).toggleClass("active"))},n.dispose=function(){e.removeData(this._element,"bs.button"),this._element=null},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.button");i||(i=new t(this),e(this).data("bs.button",i)),"toggle"===n&&i[n]()}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.button.data-api",'[data-toggle^="button"]',(function(t){var n=t.target,i=n;if(e(n).hasClass("btn")||(n=e(n).closest(".btn")[0]),!n||n.hasAttribute("disabled")||n.classList.contains("disabled"))t.preventDefault();else{var o=n.querySelector('input:not([type="hidden"])');if(o&&(o.hasAttribute("disabled")||o.classList.contains("disabled")))return void t.preventDefault();"LABEL"===i.tagName&&o&&"checkbox"===o.type&&t.preventDefault(),g._jQueryInterface.call(e(n),"toggle")}})).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',(function(t){var n=e(t.target).closest(".btn")[0];e(n).toggleClass("focus",/^focus(in)?$/.test(t.type))})),e(window).on("load.bs.button.data-api",(function(){for(var t=[].slice.call(document.querySelectorAll('[data-toggle="buttons"] .btn')),e=0,n=t.length;e0,this._pointerEvent=Boolean(window.PointerEvent||window.MSPointerEvent),this._addEventListeners()}var n=t.prototype;return n.next=function(){this._isSliding||this._slide("next")},n.nextWhenVisible=function(){!document.hidden&&e(this._element).is(":visible")&&"hidden"!==e(this._element).css("visibility")&&this.next()},n.prev=function(){this._isSliding||this._slide("prev")},n.pause=function(t){t||(this._isPaused=!0),this._element.querySelector(".carousel-item-next, .carousel-item-prev")&&(c.triggerTransitionEnd(this._element),this.cycle(!0)),clearInterval(this._interval),this._interval=null},n.cycle=function(t){t||(this._isPaused=!1),this._interval&&(clearInterval(this._interval),this._interval=null),this._config.interval&&!this._isPaused&&(this._interval=setInterval((document.visibilityState?this.nextWhenVisible:this.next).bind(this),this._config.interval))},n.to=function(t){var n=this;this._activeElement=this._element.querySelector(".active.carousel-item");var i=this._getItemIndex(this._activeElement);if(!(t>this._items.length-1||t<0))if(this._isSliding)e(this._element).one("slid.bs.carousel",(function(){return n.to(t)}));else{if(i===t)return this.pause(),void this.cycle();var o=t>i?"next":"prev";this._slide(o,this._items[t])}},n.dispose=function(){e(this._element).off(p),e.removeData(this._element,"bs.carousel"),this._items=null,this._config=null,this._element=null,this._interval=null,this._isPaused=null,this._isSliding=null,this._activeElement=null,this._indicatorsElement=null},n._getConfig=function(t){return t=a(a({},v),t),c.typeCheckConfig(m,t,b),t},n._handleSwipe=function(){var t=Math.abs(this.touchDeltaX);if(!(t<=40)){var e=t/this.touchDeltaX;this.touchDeltaX=0,e>0&&this.prev(),e<0&&this.next()}},n._addEventListeners=function(){var t=this;this._config.keyboard&&e(this._element).on("keydown.bs.carousel",(function(e){return t._keydown(e)})),"hover"===this._config.pause&&e(this._element).on("mouseenter.bs.carousel",(function(e){return t.pause(e)})).on("mouseleave.bs.carousel",(function(e){return t.cycle(e)})),this._config.touch&&this._addTouchEventListeners()},n._addTouchEventListeners=function(){var t=this;if(this._touchSupported){var n=function(e){t._pointerEvent&&y[e.originalEvent.pointerType.toUpperCase()]?t.touchStartX=e.originalEvent.clientX:t._pointerEvent||(t.touchStartX=e.originalEvent.touches[0].clientX)},i=function(e){t._pointerEvent&&y[e.originalEvent.pointerType.toUpperCase()]&&(t.touchDeltaX=e.originalEvent.clientX-t.touchStartX),t._handleSwipe(),"hover"===t._config.pause&&(t.pause(),t.touchTimeout&&clearTimeout(t.touchTimeout),t.touchTimeout=setTimeout((function(e){return t.cycle(e)}),500+t._config.interval))};e(this._element.querySelectorAll(".carousel-item img")).on("dragstart.bs.carousel",(function(t){return t.preventDefault()})),this._pointerEvent?(e(this._element).on("pointerdown.bs.carousel",(function(t){return n(t)})),e(this._element).on("pointerup.bs.carousel",(function(t){return i(t)})),this._element.classList.add("pointer-event")):(e(this._element).on("touchstart.bs.carousel",(function(t){return n(t)})),e(this._element).on("touchmove.bs.carousel",(function(e){return function(e){e.originalEvent.touches&&e.originalEvent.touches.length>1?t.touchDeltaX=0:t.touchDeltaX=e.originalEvent.touches[0].clientX-t.touchStartX}(e)})),e(this._element).on("touchend.bs.carousel",(function(t){return i(t)})))}},n._keydown=function(t){if(!/input|textarea/i.test(t.target.tagName))switch(t.which){case 37:t.preventDefault(),this.prev();break;case 39:t.preventDefault(),this.next()}},n._getItemIndex=function(t){return this._items=t&&t.parentNode?[].slice.call(t.parentNode.querySelectorAll(".carousel-item")):[],this._items.indexOf(t)},n._getItemByDirection=function(t,e){var n="next"===t,i="prev"===t,o=this._getItemIndex(e),s=this._items.length-1;if((i&&0===o||n&&o===s)&&!this._config.wrap)return e;var r=(o+("prev"===t?-1:1))%this._items.length;return-1===r?this._items[this._items.length-1]:this._items[r]},n._triggerSlideEvent=function(t,n){var i=this._getItemIndex(t),o=this._getItemIndex(this._element.querySelector(".active.carousel-item")),s=e.Event("slide.bs.carousel",{relatedTarget:t,direction:n,from:o,to:i});return e(this._element).trigger(s),s},n._setActiveIndicatorElement=function(t){if(this._indicatorsElement){var n=[].slice.call(this._indicatorsElement.querySelectorAll(".active"));e(n).removeClass("active");var i=this._indicatorsElement.children[this._getItemIndex(t)];i&&e(i).addClass("active")}},n._slide=function(t,n){var i,o,s,r=this,a=this._element.querySelector(".active.carousel-item"),l=this._getItemIndex(a),h=n||a&&this._getItemByDirection(t,a),u=this._getItemIndex(h),d=Boolean(this._interval);if("next"===t?(i="carousel-item-left",o="carousel-item-next",s="left"):(i="carousel-item-right",o="carousel-item-prev",s="right"),h&&e(h).hasClass("active"))this._isSliding=!1;else if(!this._triggerSlideEvent(h,s).isDefaultPrevented()&&a&&h){this._isSliding=!0,d&&this.pause(),this._setActiveIndicatorElement(h);var f=e.Event("slid.bs.carousel",{relatedTarget:h,direction:s,from:l,to:u});if(e(this._element).hasClass("slide")){e(h).addClass(o),c.reflow(h),e(a).addClass(i),e(h).addClass(i);var g=parseInt(h.getAttribute("data-interval"),10);g?(this._config.defaultInterval=this._config.defaultInterval||this._config.interval,this._config.interval=g):this._config.interval=this._config.defaultInterval||this._config.interval;var m=c.getTransitionDurationFromElement(a);e(a).one(c.TRANSITION_END,(function(){e(h).removeClass(i+" "+o).addClass("active"),e(a).removeClass("active "+o+" "+i),r._isSliding=!1,setTimeout((function(){return e(r._element).trigger(f)}),0)})).emulateTransitionEnd(m)}else e(a).removeClass("active"),e(h).addClass("active"),this._isSliding=!1,e(this._element).trigger(f);d&&this.cycle()}},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.carousel"),o=a(a({},v),e(this).data());"object"==typeof n&&(o=a(a({},o),n));var s="string"==typeof n?n:o.slide;if(i||(i=new t(this,o),e(this).data("bs.carousel",i)),"number"==typeof n)i.to(n);else if("string"==typeof s){if("undefined"==typeof i[s])throw new TypeError('No method named "'+s+'"');i[s]()}else o.interval&&o.ride&&(i.pause(),i.cycle())}))},t._dataApiClickHandler=function(n){var i=c.getSelectorFromElement(this);if(i){var o=e(i)[0];if(o&&e(o).hasClass("carousel")){var s=a(a({},e(o).data()),e(this).data()),r=this.getAttribute("data-slide-to");r&&(s.interval=!1),t._jQueryInterface.call(e(o),s),r&&e(o).data("bs.carousel").to(r),n.preventDefault()}}},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return v}}]),t}();e(document).on("click.bs.carousel.data-api","[data-slide], [data-slide-to]",E._dataApiClickHandler),e(window).on("load.bs.carousel.data-api",(function(){for(var t=[].slice.call(document.querySelectorAll('[data-ride="carousel"]')),n=0,i=t.length;n0&&(this._selector=r,this._triggerArray.push(s))}this._parent=this._config.parent?this._getParent():null,this._config.parent||this._addAriaAndCollapsedClass(this._element,this._triggerArray),this._config.toggle&&this.toggle()}var n=t.prototype;return n.toggle=function(){e(this._element).hasClass("show")?this.hide():this.show()},n.show=function(){var n,i,o=this;if(!this._isTransitioning&&!e(this._element).hasClass("show")&&(this._parent&&0===(n=[].slice.call(this._parent.querySelectorAll(".show, .collapsing")).filter((function(t){return"string"==typeof o._config.parent?t.getAttribute("data-parent")===o._config.parent:t.classList.contains("collapse")}))).length&&(n=null),!(n&&(i=e(n).not(this._selector).data("bs.collapse"))&&i._isTransitioning))){var s=e.Event("show.bs.collapse");if(e(this._element).trigger(s),!s.isDefaultPrevented()){n&&(t._jQueryInterface.call(e(n).not(this._selector),"hide"),i||e(n).data("bs.collapse",null));var r=this._getDimension();e(this._element).removeClass("collapse").addClass("collapsing"),this._element.style[r]=0,this._triggerArray.length&&e(this._triggerArray).removeClass("collapsed").attr("aria-expanded",!0),this.setTransitioning(!0);var a="scroll"+(r[0].toUpperCase()+r.slice(1)),l=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,(function(){e(o._element).removeClass("collapsing").addClass("collapse show"),o._element.style[r]="",o.setTransitioning(!1),e(o._element).trigger("shown.bs.collapse")})).emulateTransitionEnd(l),this._element.style[r]=this._element[a]+"px"}}},n.hide=function(){var t=this;if(!this._isTransitioning&&e(this._element).hasClass("show")){var n=e.Event("hide.bs.collapse");if(e(this._element).trigger(n),!n.isDefaultPrevented()){var i=this._getDimension();this._element.style[i]=this._element.getBoundingClientRect()[i]+"px",c.reflow(this._element),e(this._element).addClass("collapsing").removeClass("collapse show");var o=this._triggerArray.length;if(o>0)for(var s=0;s0},i._getOffset=function(){var t=this,e={};return"function"==typeof this._config.offset?e.fn=function(e){return e.offsets=a(a({},e.offsets),t._config.offset(e.offsets,t._element)||{}),e}:e.offset=this._config.offset,e},i._getPopperConfig=function(){var t={placement:this._getPlacement(),modifiers:{offset:this._getOffset(),flip:{enabled:this._config.flip},preventOverflow:{boundariesElement:this._config.boundary}}};return"static"===this._config.display&&(t.modifiers.applyStyle={enabled:!1}),a(a({},t),this._config.popperConfig)},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.dropdown");if(i||(i=new t(this,"object"==typeof n?n:null),e(this).data("bs.dropdown",i)),"string"==typeof n){if("undefined"==typeof i[n])throw new TypeError('No method named "'+n+'"');i[n]()}}))},t._clearMenus=function(n){if(!n||3!==n.which&&("keyup"!==n.type||9===n.which))for(var i=[].slice.call(document.querySelectorAll('[data-toggle="dropdown"]')),o=0,s=i.length;o0&&r--,40===n.which&&rdocument.documentElement.clientHeight;!this._isBodyOverflowing&&t&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!t&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},n._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},n._checkScrollbar=function(){var t=document.body.getBoundingClientRect();this._isBodyOverflowing=Math.round(t.left+t.right)
',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip",boundary:"scrollParent",sanitize:!0,sanitizeFn:null,whiteList:F,popperConfig:null},Y={HIDE:"hide.bs.tooltip",HIDDEN:"hidden.bs.tooltip",SHOW:"show.bs.tooltip",SHOWN:"shown.bs.tooltip",INSERTED:"inserted.bs.tooltip",CLICK:"click.bs.tooltip",FOCUSIN:"focusin.bs.tooltip",FOCUSOUT:"focusout.bs.tooltip",MOUSEENTER:"mouseenter.bs.tooltip",MOUSELEAVE:"mouseleave.bs.tooltip"},$=function(){function t(t,e){if("undefined"==typeof n)throw new TypeError("Bootstrap's tooltips require Popper.js (https://popper.js.org/)");this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=t,this.config=this._getConfig(e),this.tip=null,this._setListeners()}var i=t.prototype;return i.enable=function(){this._isEnabled=!0},i.disable=function(){this._isEnabled=!1},i.toggleEnabled=function(){this._isEnabled=!this._isEnabled},i.toggle=function(t){if(this._isEnabled)if(t){var n=this.constructor.DATA_KEY,i=e(t.currentTarget).data(n);i||(i=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(n,i)),i._activeTrigger.click=!i._activeTrigger.click,i._isWithActiveTrigger()?i._enter(null,i):i._leave(null,i)}else{if(e(this.getTipElement()).hasClass("show"))return void this._leave(null,this);this._enter(null,this)}},i.dispose=function(){clearTimeout(this._timeout),e.removeData(this.element,this.constructor.DATA_KEY),e(this.element).off(this.constructor.EVENT_KEY),e(this.element).closest(".modal").off("hide.bs.modal",this._hideModalHandler),this.tip&&e(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,this._activeTrigger=null,this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},i.show=function(){var t=this;if("none"===e(this.element).css("display"))throw new Error("Please use show on visible elements");var i=e.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){e(this.element).trigger(i);var o=c.findShadowRoot(this.element),s=e.contains(null!==o?o:this.element.ownerDocument.documentElement,this.element);if(i.isDefaultPrevented()||!s)return;var r=this.getTipElement(),a=c.getUID(this.constructor.NAME);r.setAttribute("id",a),this.element.setAttribute("aria-describedby",a),this.setContent(),this.config.animation&&e(r).addClass("fade");var l="function"==typeof this.config.placement?this.config.placement.call(this,r,this.element):this.config.placement,h=this._getAttachment(l);this.addAttachmentClass(h);var u=this._getContainer();e(r).data(this.constructor.DATA_KEY,this),e.contains(this.element.ownerDocument.documentElement,this.tip)||e(r).appendTo(u),e(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new n(this.element,r,this._getPopperConfig(h)),e(r).addClass("show"),"ontouchstart"in document.documentElement&&e(document.body).children().on("mouseover",null,e.noop);var d=function(){t.config.animation&&t._fixTransition();var n=t._hoverState;t._hoverState=null,e(t.element).trigger(t.constructor.Event.SHOWN),"out"===n&&t._leave(null,t)};if(e(this.tip).hasClass("fade")){var f=c.getTransitionDurationFromElement(this.tip);e(this.tip).one(c.TRANSITION_END,d).emulateTransitionEnd(f)}else d()}},i.hide=function(t){var n=this,i=this.getTipElement(),o=e.Event(this.constructor.Event.HIDE),s=function(){"show"!==n._hoverState&&i.parentNode&&i.parentNode.removeChild(i),n._cleanTipClass(),n.element.removeAttribute("aria-describedby"),e(n.element).trigger(n.constructor.Event.HIDDEN),null!==n._popper&&n._popper.destroy(),t&&t()};if(e(this.element).trigger(o),!o.isDefaultPrevented()){if(e(i).removeClass("show"),"ontouchstart"in document.documentElement&&e(document.body).children().off("mouseover",null,e.noop),this._activeTrigger.click=!1,this._activeTrigger.focus=!1,this._activeTrigger.hover=!1,e(this.tip).hasClass("fade")){var r=c.getTransitionDurationFromElement(i);e(i).one(c.TRANSITION_END,s).emulateTransitionEnd(r)}else s();this._hoverState=""}},i.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},i.isWithContent=function(){return Boolean(this.getTitle())},i.addAttachmentClass=function(t){e(this.getTipElement()).addClass("bs-tooltip-"+t)},i.getTipElement=function(){return this.tip=this.tip||e(this.config.template)[0],this.tip},i.setContent=function(){var t=this.getTipElement();this.setElementContent(e(t.querySelectorAll(".tooltip-inner")),this.getTitle()),e(t).removeClass("fade show")},i.setElementContent=function(t,n){"object"!=typeof n||!n.nodeType&&!n.jquery?this.config.html?(this.config.sanitize&&(n=H(n,this.config.whiteList,this.config.sanitizeFn)),t.html(n)):t.text(n):this.config.html?e(n).parent().is(t)||t.empty().append(n):t.text(e(n).text())},i.getTitle=function(){var t=this.element.getAttribute("data-original-title");return t||(t="function"==typeof this.config.title?this.config.title.call(this.element):this.config.title),t},i._getPopperConfig=function(t){var e=this;return a(a({},{placement:t,modifiers:{offset:this._getOffset(),flip:{behavior:this.config.fallbackPlacement},arrow:{element:".arrow"},preventOverflow:{boundariesElement:this.config.boundary}},onCreate:function(t){t.originalPlacement!==t.placement&&e._handlePopperPlacementChange(t)},onUpdate:function(t){return e._handlePopperPlacementChange(t)}}),this.config.popperConfig)},i._getOffset=function(){var t=this,e={};return"function"==typeof this.config.offset?e.fn=function(e){return e.offsets=a(a({},e.offsets),t.config.offset(e.offsets,t.element)||{}),e}:e.offset=this.config.offset,e},i._getContainer=function(){return!1===this.config.container?document.body:c.isElement(this.config.container)?e(this.config.container):e(document).find(this.config.container)},i._getAttachment=function(t){return K[t.toUpperCase()]},i._setListeners=function(){var t=this;this.config.trigger.split(" ").forEach((function(n){if("click"===n)e(t.element).on(t.constructor.Event.CLICK,t.config.selector,(function(e){return t.toggle(e)}));else if("manual"!==n){var i="hover"===n?t.constructor.Event.MOUSEENTER:t.constructor.Event.FOCUSIN,o="hover"===n?t.constructor.Event.MOUSELEAVE:t.constructor.Event.FOCUSOUT;e(t.element).on(i,t.config.selector,(function(e){return t._enter(e)})).on(o,t.config.selector,(function(e){return t._leave(e)}))}})),this._hideModalHandler=function(){t.element&&t.hide()},e(this.element).closest(".modal").on("hide.bs.modal",this._hideModalHandler),this.config.selector?this.config=a(a({},this.config),{},{trigger:"manual",selector:""}):this._fixTitle()},i._fixTitle=function(){var t=typeof this.element.getAttribute("data-original-title");(this.element.getAttribute("title")||"string"!==t)&&(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},i._enter=function(t,n){var i=this.constructor.DATA_KEY;(n=n||e(t.currentTarget).data(i))||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(i,n)),t&&(n._activeTrigger["focusin"===t.type?"focus":"hover"]=!0),e(n.getTipElement()).hasClass("show")||"show"===n._hoverState?n._hoverState="show":(clearTimeout(n._timeout),n._hoverState="show",n.config.delay&&n.config.delay.show?n._timeout=setTimeout((function(){"show"===n._hoverState&&n.show()}),n.config.delay.show):n.show())},i._leave=function(t,n){var i=this.constructor.DATA_KEY;(n=n||e(t.currentTarget).data(i))||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(i,n)),t&&(n._activeTrigger["focusout"===t.type?"focus":"hover"]=!1),n._isWithActiveTrigger()||(clearTimeout(n._timeout),n._hoverState="out",n.config.delay&&n.config.delay.hide?n._timeout=setTimeout((function(){"out"===n._hoverState&&n.hide()}),n.config.delay.hide):n.hide())},i._isWithActiveTrigger=function(){for(var t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1},i._getConfig=function(t){var n=e(this.element).data();return Object.keys(n).forEach((function(t){-1!==V.indexOf(t)&&delete n[t]})),"number"==typeof(t=a(a(a({},this.constructor.Default),n),"object"==typeof t&&t?t:{})).delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),c.typeCheckConfig(U,t,this.constructor.DefaultType),t.sanitize&&(t.template=H(t.template,t.whiteList,t.sanitizeFn)),t},i._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]!==this.config[e]&&(t[e]=this.config[e]);return t},i._cleanTipClass=function(){var t=e(this.getTipElement()),n=t.attr("class").match(W);null!==n&&n.length&&t.removeClass(n.join(""))},i._handlePopperPlacementChange=function(t){this.tip=t.instance.popper,this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(t.placement))},i._fixTransition=function(){var t=this.getTipElement(),n=this.config.animation;null===t.getAttribute("x-placement")&&(e(t).removeClass("fade"),this.config.animation=!1,this.hide(),this.show(),this.config.animation=n)},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.tooltip"),o="object"==typeof n&&n;if((i||!/dispose|hide/.test(n))&&(i||(i=new t(this,o),e(this).data("bs.tooltip",i)),"string"==typeof n)){if("undefined"==typeof i[n])throw new TypeError('No method named "'+n+'"');i[n]()}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return X}},{key:"NAME",get:function(){return U}},{key:"DATA_KEY",get:function(){return"bs.tooltip"}},{key:"Event",get:function(){return Y}},{key:"EVENT_KEY",get:function(){return".bs.tooltip"}},{key:"DefaultType",get:function(){return z}}]),t}();e.fn[U]=$._jQueryInterface,e.fn[U].Constructor=$,e.fn[U].noConflict=function(){return e.fn[U]=M,$._jQueryInterface};var J="popover",G=e.fn[J],Z=new RegExp("(^|\\s)bs-popover\\S+","g"),tt=a(a({},$.Default),{},{placement:"right",trigger:"click",content:"",template:''}),et=a(a({},$.DefaultType),{},{content:"(string|element|function)"}),nt={HIDE:"hide.bs.popover",HIDDEN:"hidden.bs.popover",SHOW:"show.bs.popover",SHOWN:"shown.bs.popover",INSERTED:"inserted.bs.popover",CLICK:"click.bs.popover",FOCUSIN:"focusin.bs.popover",FOCUSOUT:"focusout.bs.popover",MOUSEENTER:"mouseenter.bs.popover",MOUSELEAVE:"mouseleave.bs.popover"},it=function(t){var n,i;function s(){return t.apply(this,arguments)||this}i=t,(n=s).prototype=Object.create(i.prototype),n.prototype.constructor=n,n.__proto__=i;var r=s.prototype;return r.isWithContent=function(){return this.getTitle()||this._getContent()},r.addAttachmentClass=function(t){e(this.getTipElement()).addClass("bs-popover-"+t)},r.getTipElement=function(){return this.tip=this.tip||e(this.config.template)[0],this.tip},r.setContent=function(){var t=e(this.getTipElement());this.setElementContent(t.find(".popover-header"),this.getTitle());var n=this._getContent();"function"==typeof n&&(n=n.call(this.element)),this.setElementContent(t.find(".popover-body"),n),t.removeClass("fade show")},r._getContent=function(){return this.element.getAttribute("data-content")||this.config.content},r._cleanTipClass=function(){var t=e(this.getTipElement()),n=t.attr("class").match(Z);null!==n&&n.length>0&&t.removeClass(n.join(""))},s._jQueryInterface=function(t){return this.each((function(){var n=e(this).data("bs.popover"),i="object"==typeof t?t:null;if((n||!/dispose|hide/.test(t))&&(n||(n=new s(this,i),e(this).data("bs.popover",n)),"string"==typeof t)){if("undefined"==typeof n[t])throw new TypeError('No method named "'+t+'"');n[t]()}}))},o(s,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return tt}},{key:"NAME",get:function(){return J}},{key:"DATA_KEY",get:function(){return"bs.popover"}},{key:"Event",get:function(){return nt}},{key:"EVENT_KEY",get:function(){return".bs.popover"}},{key:"DefaultType",get:function(){return et}}]),s}($);e.fn[J]=it._jQueryInterface,e.fn[J].Constructor=it,e.fn[J].noConflict=function(){return e.fn[J]=G,it._jQueryInterface};var ot="scrollspy",st=e.fn[ot],rt={offset:10,method:"auto",target:""},at={offset:"number",method:"string",target:"(string|element)"},lt=function(){function t(t,n){var i=this;this._element=t,this._scrollElement="BODY"===t.tagName?window:t,this._config=this._getConfig(n),this._selector=this._config.target+" .nav-link,"+this._config.target+" .list-group-item,"+this._config.target+" .dropdown-item",this._offsets=[],this._targets=[],this._activeTarget=null,this._scrollHeight=0,e(this._scrollElement).on("scroll.bs.scrollspy",(function(t){return i._process(t)})),this.refresh(),this._process()}var n=t.prototype;return n.refresh=function(){var t=this,n=this._scrollElement===this._scrollElement.window?"offset":"position",i="auto"===this._config.method?n:this._config.method,o="position"===i?this._getScrollTop():0;this._offsets=[],this._targets=[],this._scrollHeight=this._getScrollHeight(),[].slice.call(document.querySelectorAll(this._selector)).map((function(t){var n,s=c.getSelectorFromElement(t);if(s&&(n=document.querySelector(s)),n){var r=n.getBoundingClientRect();if(r.width||r.height)return[e(n)[i]().top+o,s]}return null})).filter((function(t){return t})).sort((function(t,e){return t[0]-e[0]})).forEach((function(e){t._offsets.push(e[0]),t._targets.push(e[1])}))},n.dispose=function(){e.removeData(this._element,"bs.scrollspy"),e(this._scrollElement).off(".bs.scrollspy"),this._element=null,this._scrollElement=null,this._config=null,this._selector=null,this._offsets=null,this._targets=null,this._activeTarget=null,this._scrollHeight=null},n._getConfig=function(t){if("string"!=typeof(t=a(a({},rt),"object"==typeof t&&t?t:{})).target&&c.isElement(t.target)){var n=e(t.target).attr("id");n||(n=c.getUID(ot),e(t.target).attr("id",n)),t.target="#"+n}return c.typeCheckConfig(ot,t,at),t},n._getScrollTop=function(){return this._scrollElement===window?this._scrollElement.pageYOffset:this._scrollElement.scrollTop},n._getScrollHeight=function(){return this._scrollElement.scrollHeight||Math.max(document.body.scrollHeight,document.documentElement.scrollHeight)},n._getOffsetHeight=function(){return this._scrollElement===window?window.innerHeight:this._scrollElement.getBoundingClientRect().height},n._process=function(){var t=this._getScrollTop()+this._config.offset,e=this._getScrollHeight(),n=this._config.offset+e-this._getOffsetHeight();if(this._scrollHeight!==e&&this.refresh(),t>=n){var i=this._targets[this._targets.length-1];this._activeTarget!==i&&this._activate(i)}else{if(this._activeTarget&&t0)return this._activeTarget=null,void this._clear();for(var o=this._offsets.length;o--;){this._activeTarget!==this._targets[o]&&t>=this._offsets[o]&&("undefined"==typeof this._offsets[o+1]||t li > .active":".active";i=(i=e.makeArray(e(o).find(r)))[i.length-1]}var a=e.Event("hide.bs.tab",{relatedTarget:this._element}),l=e.Event("show.bs.tab",{relatedTarget:i});if(i&&e(i).trigger(a),e(this._element).trigger(l),!l.isDefaultPrevented()&&!a.isDefaultPrevented()){s&&(n=document.querySelector(s)),this._activate(this._element,o);var h=function(){var n=e.Event("hidden.bs.tab",{relatedTarget:t._element}),o=e.Event("shown.bs.tab",{relatedTarget:i});e(i).trigger(n),e(t._element).trigger(o)};n?this._activate(n,n.parentNode,h):h()}}},n.dispose=function(){e.removeData(this._element,"bs.tab"),this._element=null},n._activate=function(t,n,i){var o=this,s=(!n||"UL"!==n.nodeName&&"OL"!==n.nodeName?e(n).children(".active"):e(n).find("> li > .active"))[0],r=i&&s&&e(s).hasClass("fade"),a=function(){return o._transitionComplete(t,s,i)};if(s&&r){var l=c.getTransitionDurationFromElement(s);e(s).removeClass("show").one(c.TRANSITION_END,a).emulateTransitionEnd(l)}else a()},n._transitionComplete=function(t,n,i){if(n){e(n).removeClass("active");var o=e(n.parentNode).find("> .dropdown-menu .active")[0];o&&e(o).removeClass("active"),"tab"===n.getAttribute("role")&&n.setAttribute("aria-selected",!1)}if(e(t).addClass("active"),"tab"===t.getAttribute("role")&&t.setAttribute("aria-selected",!0),c.reflow(t),t.classList.contains("fade")&&t.classList.add("show"),t.parentNode&&e(t.parentNode).hasClass("dropdown-menu")){var s=e(t).closest(".dropdown")[0];if(s){var r=[].slice.call(s.querySelectorAll(".dropdown-toggle"));e(r).addClass("active")}t.setAttribute("aria-expanded",!0)}i&&i()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.tab");if(o||(o=new t(this),i.data("bs.tab",o)),"string"==typeof n){if("undefined"==typeof o[n])throw new TypeError('No method named "'+n+'"');o[n]()}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.tab.data-api",'[data-toggle="tab"], [data-toggle="pill"], [data-toggle="list"]',(function(t){t.preventDefault(),ht._jQueryInterface.call(e(this),"show")})),e.fn.tab=ht._jQueryInterface,e.fn.tab.Constructor=ht,e.fn.tab.noConflict=function(){return e.fn.tab=ct,ht._jQueryInterface};var ut=e.fn.toast,dt={animation:"boolean",autohide:"boolean",delay:"number"},ft={animation:!0,autohide:!0,delay:500},gt=function(){function t(t,e){this._element=t,this._config=this._getConfig(e),this._timeout=null,this._setListeners()}var n=t.prototype;return n.show=function(){var t=this,n=e.Event("show.bs.toast");if(e(this._element).trigger(n),!n.isDefaultPrevented()){this._config.animation&&this._element.classList.add("fade");var i=function(){t._element.classList.remove("showing"),t._element.classList.add("show"),e(t._element).trigger("shown.bs.toast"),t._config.autohide&&(t._timeout=setTimeout((function(){t.hide()}),t._config.delay))};if(this._element.classList.remove("hide"),c.reflow(this._element),this._element.classList.add("showing"),this._config.animation){var o=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,i).emulateTransitionEnd(o)}else i()}},n.hide=function(){if(this._element.classList.contains("show")){var t=e.Event("hide.bs.toast");e(this._element).trigger(t),t.isDefaultPrevented()||this._close()}},n.dispose=function(){clearTimeout(this._timeout),this._timeout=null,this._element.classList.contains("show")&&this._element.classList.remove("show"),e(this._element).off("click.dismiss.bs.toast"),e.removeData(this._element,"bs.toast"),this._element=null,this._config=null},n._getConfig=function(t){return t=a(a(a({},ft),e(this._element).data()),"object"==typeof t&&t?t:{}),c.typeCheckConfig("toast",t,this.constructor.DefaultType),t},n._setListeners=function(){var t=this;e(this._element).on("click.dismiss.bs.toast",'[data-dismiss="toast"]',(function(){return t.hide()}))},n._close=function(){var t=this,n=function(){t._element.classList.add("hide"),e(t._element).trigger("hidden.bs.toast")};if(this._element.classList.remove("show"),this._config.animation){var i=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,n).emulateTransitionEnd(i)}else n()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.toast");if(o||(o=new t(this,"object"==typeof n&&n),i.data("bs.toast",o)),"string"==typeof n){if("undefined"==typeof o[n])throw new TypeError('No method named "'+n+'"');o[n](this)}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"DefaultType",get:function(){return dt}},{key:"Default",get:function(){return ft}}]),t}();e.fn.toast=gt._jQueryInterface,e.fn.toast.Constructor=gt,e.fn.toast.noConflict=function(){return e.fn.toast=ut,gt._jQueryInterface},t.Alert=d,t.Button=g,t.Carousel=E,t.Collapse=D,t.Dropdown=j,t.Modal=R,t.Popover=it,t.Scrollspy=lt,t.Tab=ht,t.Toast=gt,t.Tooltip=$,t.Util=c,Object.defineProperty(t,"__esModule",{value:!0})})); +//# sourceMappingURL=bootstrap.min.js.map \ No newline at end of file diff --git a/iclient/iclientpkg/static-content/jquery.min.js b/iclient/iclientpkg/static-content/jquery.min.js new file mode 100644 index 00000000..36b4e1a1 --- /dev/null +++ b/iclient/iclientpkg/static-content/jquery.min.js @@ -0,0 +1,2 @@ +/*! jQuery v3.5.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-deprecated/ajax-event-alias,-effects,-effects/Tween,-effects/animatedSelector | (c) JS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(g,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,v=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,y=n.hasOwnProperty,a=y.toString,l=a.call(Object),m={},b=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},w=g.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function C(e,t,n){var r,i,o=(n=n||w).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function T(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-deprecated/ajax-event-alias,-effects,-effects/Tween,-effects/animatedSelector",E=function(e,t){return new E.fn.init(e,t)};function d(e){var t=!!e&&"length"in e&&e.length,n=T(e);return!b(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+R+")"+R+"*"),U=new RegExp(R+"|>"),V=new RegExp(W),X=new RegExp("^"+B+"$"),Q={ID:new RegExp("^#("+B+")"),CLASS:new RegExp("^\\.("+B+")"),TAG:new RegExp("^("+B+"|[*])"),ATTR:new RegExp("^"+M),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+R+"*(even|odd|(([+-]|)(\\d*)n|)"+R+"*(?:([+-]|)"+R+"*(\\d+)|))"+R+"*\\)|)","i"),bool:new RegExp("^(?:"+I+")$","i"),needsContext:new RegExp("^"+R+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+R+"*((?:-\\d)?\\d*)"+R+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,G=/^(?:input|select|textarea|button)$/i,K=/^h\d$/i,J=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+R+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){C()},ae=xe(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{O.apply(t=P.call(d.childNodes),d.childNodes),t[d.childNodes.length].nodeType}catch(e){O={apply:t.length?function(e,t){q.apply(e,P.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,d=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==d&&9!==d&&11!==d)return n;if(!r&&(C(e),e=e||T,E)){if(11!==d&&(u=Z.exec(t)))if(i=u[1]){if(9===d){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return O.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&p.getElementsByClassName&&e.getElementsByClassName)return O.apply(n,e.getElementsByClassName(i)),n}if(p.qsa&&!k[t+" "]&&(!v||!v.test(t))&&(1!==d||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===d&&(U.test(t)||_.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&p.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=A)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+be(l[o]);c=l.join(",")}try{return O.apply(n,f.querySelectorAll(c)),n}catch(e){k(t,!0)}finally{s===A&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>x.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[A]=!0,e}function ce(e){var t=T.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)x.attrHandle[n[r]]=t}function de(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function pe(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in p=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},C=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:d;return r!=T&&9===r.nodeType&&r.documentElement&&(a=(T=r).documentElement,E=!i(T),d!=T&&(n=T.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),p.scope=ce(function(e){return a.appendChild(e).appendChild(T.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),p.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),p.getElementsByTagName=ce(function(e){return e.appendChild(T.createComment("")),!e.getElementsByTagName("*").length}),p.getElementsByClassName=J.test(T.getElementsByClassName),p.getById=ce(function(e){return a.appendChild(e).id=A,!T.getElementsByName||!T.getElementsByName(A).length}),p.getById?(x.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},x.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(x.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},x.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),x.find.TAG=p.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):p.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},x.find.CLASS=p.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(p.qsa=J.test(T.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+R+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+R+"*(?:value|"+I+")"),e.querySelectorAll("[id~="+A+"-]").length||v.push("~="),(t=T.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+R+"*name"+R+"*="+R+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+A+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=T.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+R+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(p.matchesSelector=J.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){p.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",W)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=J.test(a.compareDocumentPosition),y=t||J.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!p.sortDetached&&t.compareDocumentPosition(e)===n?e==T||e.ownerDocument==d&&y(d,e)?-1:t==T||t.ownerDocument==d&&y(d,t)?1:u?H(u,e)-H(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==T?-1:t==T?1:i?-1:o?1:u?H(u,e)-H(u,t):0;if(i===o)return de(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?de(a[r],s[r]):a[r]==d?-1:s[r]==d?1:0}),T},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(C(e),p.matchesSelector&&E&&!k[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||p.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){k(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return Q.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&V.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+R+")"+e+"("+R+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return b(n)?E.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?E.grep(e,function(e){return e===n!==r}):"string"!=typeof n?E.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(E.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||L,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:j.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof E?t[0]:t,E.merge(this,E.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:w,!0)),k.test(r[1])&&E.isPlainObject(t))for(r in t)b(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=w.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):b(e)?void 0!==n.ready?n.ready(e):e(E):E.makeArray(e,this)}).prototype=E.fn,L=E(w);var q=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}E.fn.extend({has:function(e){var t=E(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,pe=/^$|^module$|\/(?:java|ecma)script/i;le=w.createDocumentFragment().appendChild(w.createElement("div")),(ce=w.createElement("input")).setAttribute("type","radio"),ce.setAttribute("checked","checked"),ce.setAttribute("name","t"),le.appendChild(ce),m.checkClone=le.cloneNode(!0).cloneNode(!0).lastChild.checked,le.innerHTML="",m.noCloneChecked=!!le.cloneNode(!0).lastChild.defaultValue,le.innerHTML="",m.option=!!le.lastChild;var he={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ge(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&S(e,t)?E.merge([e],n):n}function ve(e,t){for(var n=0,r=e.length;n",""]);var ye=/<|&#?\w+;/;function me(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),d=[],p=0,h=e.length;p\s*$/g;function Le(e,t){return S(e,"table")&&S(11!==t.nodeType?t:t.firstChild,"tr")&&E(e).children("tbody")[0]||e}function je(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n
",2===ft.childNodes.length),E.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(m.createHTMLDocument?((r=(t=w.implementation.createHTMLDocument("")).createElement("base")).href=w.location.href,t.head.appendChild(r)):t=w),o=!n&&[],(i=k.exec(e))?[t.createElement(i[1])]:(i=me([e],t,o),o&&o.length&&E(o).remove(),E.merge([],i.childNodes)));var r,i,o},E.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=E.css(e,"position"),c=E(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=E.css(e,"top"),u=E.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),b(t)&&(t=t.call(e,n,E.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},E.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){E.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===E.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===E.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=E(e).offset()).top+=E.css(e,"borderTopWidth",!0),i.left+=E.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-E.css(r,"marginTop",!0),left:t.left-i.left-E.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===E.css(e,"position"))e=e.offsetParent;return e||re})}}),E.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;E.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),E.each(["top","left"],function(e,n){E.cssHooks[n]=Fe(m.pixelPosition,function(e,t){if(t)return t=We(e,n),Ie.test(t)?E(e).position()[n]+"px":t})}),E.each({Height:"height",Width:"width"},function(a,s){E.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){E.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?E.css(e,t,i):E.style(e,t,n,i)},s,n?e:void 0,n)}})}),E.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),E.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){E.fn[n]=function(e,t){return 0': '>', + '"': '"', + '\'': ''', + '/': '/' + }; + + var defaultSettings = { + indent: 2 + }; + + var id = 0; + var instances = 0; + var current_collapse_level = null; + var ids_to_collapse = []; + + this.create = function(data, settings, collapse_depth) { + current_collapse_level = typeof collapse_depth !== 'undefined' ? collapse_depth : -1; + instances += 1; + return _span(_jsVal(data, 0, false), {class: 'jstValue'}); + }; + + this.collapse = function() { + var arrayLength = ids_to_collapse.length; + for (var i = 0; i < arrayLength; i++) { + JSONTree.toggle(ids_to_collapse[i]); + } + }; + + var _escape = function(text) { + return text.replace(/[&<>'"]/g, function(c) { + return escapeMap[c]; + }); + }; + + var _id = function() { + return instances + '_' + id++; + }; + + var _lastId = function() { + return instances + '_' + (id - 1); + }; + + var _jsVal = function(value, depth, indent) { + if (value !== null) { + var type = typeof value; + switch (type) { + case 'boolean': + return _jsBool(value, indent ? depth : 0); + case 'number': + return _jsNum(value, indent ? depth : 0); + case 'string': + return _jsStr(value, indent ? depth : 0); + default: + if (value instanceof Array) { + return _jsArr(value, depth, indent); + } else { + return _jsObj(value, depth, indent); + } + } + } else { + return _jsNull(indent ? depth : 0); + } + }; + + var _jsObj = function(object, depth, indent) { + var id = _id(); + _decrementCollapseLevel("_jsObj"); + var content = Object.keys(object).map(function(property) { + return _property(property, object[property], depth + 1, true); + }).join(_comma()); + var body = [ + _openBracket('{', indent ? depth : 0, id), + _span(content, {id: id}), + _closeBracket('}', depth) + ].join('\n'); + _incrementCollapseLevel("_jsObj"); + return _span(body, {}); + }; + + var _jsArr = function(array, depth, indent) { + var id = _id(); + _decrementCollapseLevel("_jsArr"); + var body = array.map(function(element) { + return _jsVal(element, depth + 1, true); + }).join(_comma()); + var arr = [ + _openBracket('[', indent ? depth : 0, id), + _span(body, {id: id}), + _closeBracket(']', depth) + ].join('\n'); + _incrementCollapseLevel("_jsArr"); + return arr; + }; + + var _jsStr = function(value, depth) { + var jsonString = _escape(JSON.stringify(value)); + return _span(_indent(jsonString, depth), {class: 'jstStr'}); + }; + + var _jsNum = function(value, depth) { + return _span(_indent(value, depth), {class: 'jstNum'}); + }; + + var _jsBool = function(value, depth) { + return _span(_indent(value, depth), {class: 'jstBool'}); + }; + + var _jsNull = function(depth) { + return _span(_indent('null', depth), {class: 'jstNull'}); + }; + + var _property = function(name, value, depth) { + var property = _indent(_escape(JSON.stringify(name)) + ': ', depth); + var propertyValue = _span(_jsVal(value, depth, false), {}); + return _span(property + propertyValue, {class: 'jstProperty'}); + }; + + var _comma = function() { + return _span(',\n', {class: 'jstComma'}); + }; + + var _span = function(value, attrs) { + return _tag('span', attrs, value); + }; + + var _tag = function(tag, attrs, content) { + return '<' + tag + Object.keys(attrs).map(function(attr) { + return ' ' + attr + '="' + attrs[attr] + '"'; + }).join('') + '>' + + content + + ''; + }; + + var _openBracket = function(symbol, depth, id) { + return ( + _span(_indent(symbol, depth), {class: 'jstBracket'}) + + _span('', {class: 'jstFold', onclick: 'JSONTree.toggle(\'' + id + '\')'}) + ); + }; + + this.toggle = function(id) { + var element = document.getElementById(id); + var parent = element.parentNode; + var toggleButton = element.previousElementSibling; + if (element.className === '') { + element.className = 'jstHiddenBlock'; + parent.className = 'jstFolded'; + toggleButton.className = 'jstExpand'; + } else { + element.className = ''; + parent.className = ''; + toggleButton.className = 'jstFold'; + } + }; + + var _closeBracket = function(symbol, depth) { + return _span(_indent(symbol, depth), {}); + }; + + var _indent = function(value, depth) { + return Array((depth * 2) + 1).join(' ') + value; + }; + + var _decrementCollapseLevel = function(caller) { + if (current_collapse_level <= 0) { + ids_to_collapse.push(_lastId()); + } else { + } + current_collapse_level--; + }; + + var _incrementCollapseLevel = function(caller) { + current_collapse_level++; + }; + + return this; +})(); diff --git a/iclient/iclientpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css b/iclient/iclientpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css new file mode 100755 index 00000000..4664f2e8 --- /dev/null +++ b/iclient/iclientpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css @@ -0,0 +1 @@ +@font-face{font-family:Icons;src:url(../fonts/open-iconic.eot);src:url(../fonts/open-iconic.eot?#iconic-sm) format('embedded-opentype'),url(../fonts/open-iconic.woff) format('woff'),url(../fonts/open-iconic.ttf) format('truetype'),url(../fonts/open-iconic.otf) format('opentype'),url(../fonts/open-iconic.svg#iconic-sm) format('svg');font-weight:400;font-style:normal}.oi{position:relative;top:1px;display:inline-block;speak:none;font-family:Icons;font-style:normal;font-weight:400;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.oi:empty:before{width:1em;text-align:center;box-sizing:content-box}.oi.oi-align-center:before{text-align:center}.oi.oi-align-left:before{text-align:left}.oi.oi-align-right:before{text-align:right}.oi.oi-flip-horizontal:before{-webkit-transform:scale(-1,1);-ms-transform:scale(-1,1);transform:scale(-1,1)}.oi.oi-flip-vertical:before{-webkit-transform:scale(1,-1);-ms-transform:scale(-1,1);transform:scale(1,-1)}.oi.oi-flip-horizontal-vertical:before{-webkit-transform:scale(-1,-1);-ms-transform:scale(-1,1);transform:scale(-1,-1)}.oi-account-login:before{content:'\e000'}.oi-account-logout:before{content:'\e001'}.oi-action-redo:before{content:'\e002'}.oi-action-undo:before{content:'\e003'}.oi-align-center:before{content:'\e004'}.oi-align-left:before{content:'\e005'}.oi-align-right:before{content:'\e006'}.oi-aperture:before{content:'\e007'}.oi-arrow-bottom:before{content:'\e008'}.oi-arrow-circle-bottom:before{content:'\e009'}.oi-arrow-circle-left:before{content:'\e00a'}.oi-arrow-circle-right:before{content:'\e00b'}.oi-arrow-circle-top:before{content:'\e00c'}.oi-arrow-left:before{content:'\e00d'}.oi-arrow-right:before{content:'\e00e'}.oi-arrow-thick-bottom:before{content:'\e00f'}.oi-arrow-thick-left:before{content:'\e010'}.oi-arrow-thick-right:before{content:'\e011'}.oi-arrow-thick-top:before{content:'\e012'}.oi-arrow-top:before{content:'\e013'}.oi-audio-spectrum:before{content:'\e014'}.oi-audio:before{content:'\e015'}.oi-badge:before{content:'\e016'}.oi-ban:before{content:'\e017'}.oi-bar-chart:before{content:'\e018'}.oi-basket:before{content:'\e019'}.oi-battery-empty:before{content:'\e01a'}.oi-battery-full:before{content:'\e01b'}.oi-beaker:before{content:'\e01c'}.oi-bell:before{content:'\e01d'}.oi-bluetooth:before{content:'\e01e'}.oi-bold:before{content:'\e01f'}.oi-bolt:before{content:'\e020'}.oi-book:before{content:'\e021'}.oi-bookmark:before{content:'\e022'}.oi-box:before{content:'\e023'}.oi-briefcase:before{content:'\e024'}.oi-british-pound:before{content:'\e025'}.oi-browser:before{content:'\e026'}.oi-brush:before{content:'\e027'}.oi-bug:before{content:'\e028'}.oi-bullhorn:before{content:'\e029'}.oi-calculator:before{content:'\e02a'}.oi-calendar:before{content:'\e02b'}.oi-camera-slr:before{content:'\e02c'}.oi-caret-bottom:before{content:'\e02d'}.oi-caret-left:before{content:'\e02e'}.oi-caret-right:before{content:'\e02f'}.oi-caret-top:before{content:'\e030'}.oi-cart:before{content:'\e031'}.oi-chat:before{content:'\e032'}.oi-check:before{content:'\e033'}.oi-chevron-bottom:before{content:'\e034'}.oi-chevron-left:before{content:'\e035'}.oi-chevron-right:before{content:'\e036'}.oi-chevron-top:before{content:'\e037'}.oi-circle-check:before{content:'\e038'}.oi-circle-x:before{content:'\e039'}.oi-clipboard:before{content:'\e03a'}.oi-clock:before{content:'\e03b'}.oi-cloud-download:before{content:'\e03c'}.oi-cloud-upload:before{content:'\e03d'}.oi-cloud:before{content:'\e03e'}.oi-cloudy:before{content:'\e03f'}.oi-code:before{content:'\e040'}.oi-cog:before{content:'\e041'}.oi-collapse-down:before{content:'\e042'}.oi-collapse-left:before{content:'\e043'}.oi-collapse-right:before{content:'\e044'}.oi-collapse-up:before{content:'\e045'}.oi-command:before{content:'\e046'}.oi-comment-square:before{content:'\e047'}.oi-compass:before{content:'\e048'}.oi-contrast:before{content:'\e049'}.oi-copywriting:before{content:'\e04a'}.oi-credit-card:before{content:'\e04b'}.oi-crop:before{content:'\e04c'}.oi-dashboard:before{content:'\e04d'}.oi-data-transfer-download:before{content:'\e04e'}.oi-data-transfer-upload:before{content:'\e04f'}.oi-delete:before{content:'\e050'}.oi-dial:before{content:'\e051'}.oi-document:before{content:'\e052'}.oi-dollar:before{content:'\e053'}.oi-double-quote-sans-left:before{content:'\e054'}.oi-double-quote-sans-right:before{content:'\e055'}.oi-double-quote-serif-left:before{content:'\e056'}.oi-double-quote-serif-right:before{content:'\e057'}.oi-droplet:before{content:'\e058'}.oi-eject:before{content:'\e059'}.oi-elevator:before{content:'\e05a'}.oi-ellipses:before{content:'\e05b'}.oi-envelope-closed:before{content:'\e05c'}.oi-envelope-open:before{content:'\e05d'}.oi-euro:before{content:'\e05e'}.oi-excerpt:before{content:'\e05f'}.oi-expand-down:before{content:'\e060'}.oi-expand-left:before{content:'\e061'}.oi-expand-right:before{content:'\e062'}.oi-expand-up:before{content:'\e063'}.oi-external-link:before{content:'\e064'}.oi-eye:before{content:'\e065'}.oi-eyedropper:before{content:'\e066'}.oi-file:before{content:'\e067'}.oi-fire:before{content:'\e068'}.oi-flag:before{content:'\e069'}.oi-flash:before{content:'\e06a'}.oi-folder:before{content:'\e06b'}.oi-fork:before{content:'\e06c'}.oi-fullscreen-enter:before{content:'\e06d'}.oi-fullscreen-exit:before{content:'\e06e'}.oi-globe:before{content:'\e06f'}.oi-graph:before{content:'\e070'}.oi-grid-four-up:before{content:'\e071'}.oi-grid-three-up:before{content:'\e072'}.oi-grid-two-up:before{content:'\e073'}.oi-hard-drive:before{content:'\e074'}.oi-header:before{content:'\e075'}.oi-headphones:before{content:'\e076'}.oi-heart:before{content:'\e077'}.oi-home:before{content:'\e078'}.oi-image:before{content:'\e079'}.oi-inbox:before{content:'\e07a'}.oi-infinity:before{content:'\e07b'}.oi-info:before{content:'\e07c'}.oi-italic:before{content:'\e07d'}.oi-justify-center:before{content:'\e07e'}.oi-justify-left:before{content:'\e07f'}.oi-justify-right:before{content:'\e080'}.oi-key:before{content:'\e081'}.oi-laptop:before{content:'\e082'}.oi-layers:before{content:'\e083'}.oi-lightbulb:before{content:'\e084'}.oi-link-broken:before{content:'\e085'}.oi-link-intact:before{content:'\e086'}.oi-list-rich:before{content:'\e087'}.oi-list:before{content:'\e088'}.oi-location:before{content:'\e089'}.oi-lock-locked:before{content:'\e08a'}.oi-lock-unlocked:before{content:'\e08b'}.oi-loop-circular:before{content:'\e08c'}.oi-loop-square:before{content:'\e08d'}.oi-loop:before{content:'\e08e'}.oi-magnifying-glass:before{content:'\e08f'}.oi-map-marker:before{content:'\e090'}.oi-map:before{content:'\e091'}.oi-media-pause:before{content:'\e092'}.oi-media-play:before{content:'\e093'}.oi-media-record:before{content:'\e094'}.oi-media-skip-backward:before{content:'\e095'}.oi-media-skip-forward:before{content:'\e096'}.oi-media-step-backward:before{content:'\e097'}.oi-media-step-forward:before{content:'\e098'}.oi-media-stop:before{content:'\e099'}.oi-medical-cross:before{content:'\e09a'}.oi-menu:before{content:'\e09b'}.oi-microphone:before{content:'\e09c'}.oi-minus:before{content:'\e09d'}.oi-monitor:before{content:'\e09e'}.oi-moon:before{content:'\e09f'}.oi-move:before{content:'\e0a0'}.oi-musical-note:before{content:'\e0a1'}.oi-paperclip:before{content:'\e0a2'}.oi-pencil:before{content:'\e0a3'}.oi-people:before{content:'\e0a4'}.oi-person:before{content:'\e0a5'}.oi-phone:before{content:'\e0a6'}.oi-pie-chart:before{content:'\e0a7'}.oi-pin:before{content:'\e0a8'}.oi-play-circle:before{content:'\e0a9'}.oi-plus:before{content:'\e0aa'}.oi-power-standby:before{content:'\e0ab'}.oi-print:before{content:'\e0ac'}.oi-project:before{content:'\e0ad'}.oi-pulse:before{content:'\e0ae'}.oi-puzzle-piece:before{content:'\e0af'}.oi-question-mark:before{content:'\e0b0'}.oi-rain:before{content:'\e0b1'}.oi-random:before{content:'\e0b2'}.oi-reload:before{content:'\e0b3'}.oi-resize-both:before{content:'\e0b4'}.oi-resize-height:before{content:'\e0b5'}.oi-resize-width:before{content:'\e0b6'}.oi-rss-alt:before{content:'\e0b7'}.oi-rss:before{content:'\e0b8'}.oi-script:before{content:'\e0b9'}.oi-share-boxed:before{content:'\e0ba'}.oi-share:before{content:'\e0bb'}.oi-shield:before{content:'\e0bc'}.oi-signal:before{content:'\e0bd'}.oi-signpost:before{content:'\e0be'}.oi-sort-ascending:before{content:'\e0bf'}.oi-sort-descending:before{content:'\e0c0'}.oi-spreadsheet:before{content:'\e0c1'}.oi-star:before{content:'\e0c2'}.oi-sun:before{content:'\e0c3'}.oi-tablet:before{content:'\e0c4'}.oi-tag:before{content:'\e0c5'}.oi-tags:before{content:'\e0c6'}.oi-target:before{content:'\e0c7'}.oi-task:before{content:'\e0c8'}.oi-terminal:before{content:'\e0c9'}.oi-text:before{content:'\e0ca'}.oi-thumb-down:before{content:'\e0cb'}.oi-thumb-up:before{content:'\e0cc'}.oi-timer:before{content:'\e0cd'}.oi-transfer:before{content:'\e0ce'}.oi-trash:before{content:'\e0cf'}.oi-underline:before{content:'\e0d0'}.oi-vertical-align-bottom:before{content:'\e0d1'}.oi-vertical-align-center:before{content:'\e0d2'}.oi-vertical-align-top:before{content:'\e0d3'}.oi-video:before{content:'\e0d4'}.oi-volume-high:before{content:'\e0d5'}.oi-volume-low:before{content:'\e0d6'}.oi-volume-off:before{content:'\e0d7'}.oi-warning:before{content:'\e0d8'}.oi-wifi:before{content:'\e0d9'}.oi-wrench:before{content:'\e0da'}.oi-x:before{content:'\e0db'}.oi-yen:before{content:'\e0dc'}.oi-zoom-in:before{content:'\e0dd'}.oi-zoom-out:before{content:'\e0de'} \ No newline at end of file diff --git a/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.eot b/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.eot new file mode 100755 index 0000000000000000000000000000000000000000..f98177dbf711863eff7c90f84d5d419d02d99ba8 GIT binary patch literal 28196 zcmdsfdwg8gedj&r&QluAL-W#Wq&pgEMvsv!&0Cf&+mau`20w)Dj4&8Iu59zN6=RG; z451+<)Ej~^SrrmCp$=hb!Zu?PlZ0v^rFqOYfzqruY1s`+ve{(Uv}w|M+teR4-tX_6 zJJQHDgm(Majx=-5J@?%6_?_SRz0Ykss3^zpP!y(cg+5#{t0IGvlZlxgLVa!|Pwg%0HwaAkJPsR_7CkF z{hz=5BS2$bQO4>H%uMR+@Bes%qU=0}`qqrY1!(P0t>lnf>u?>hCHF7DiD%jIRLs_gA0(b1L}rzgltYVrt?gc2Y5;9UDjQ z%B)P;{Yp$h?WOgkCosju&-Q&Abmg0GDQ~^0YA77V?+nuN;!-_LToFFdx5>D-3RhIC zNim@Y28=&kzxC#&OZZhTUDD)z++voc1{on3eJelI&j0@(PPn1`HTMH@R>gMK0^H#} z-APZ<6H9s`4L|t$XFtpR3vV~DpGXL)8ZghQI8nFC#;Gm~d%|gaTbMPC42!c1B?miM zn$?TN(kwg4=NH!N?1DZwr|Va=QM0@at3QmtSVbGuP_f*EuIqDh*>o`umty&fMPWVN zwOSy=lGa!#OKqKlS=4KL6^YiDEHv;MA!Dj|%KqdbXOLRkVPgo+>xM z`tdLxr03~jdXO4;l(4}>Kca7fS2gy1&DtubqsnG6amCcr?ZNni_*#ur)!una=lO+a z(W#N+^Oy#G-fw#XCIlD!Q7hD3IjwB$Uoy5LHCCk7M6R+q+PRlLC+2F#Og&0KX;fTm z9gRV6t=nO-P_Az=CG4l*~#0dwv=AFvG8)~&n&z! z>wcqjdUo&ccd;$(NdM=j`265c&L?J1yxG?F>}_{_wry>?^aan|yPK}R#cpg(b^$xz zf;Gl2?&aw=%jBtFht&{S}(z)fW6^mCJSIuQ@i4|p+ zx3$z#v51krkNGj$t;x!E@Z?f6a(ZZoC>r5@Ucl5$FlAy4?Q*}B&hb1!m&U%lE*Euc z#N62h7Dtl~c7f-y5Wr$VDS7_#wX$QaKmmSK`iqLyDz`g-`54&Z80Kl-ofTt{b;TI$ zT#%ThARiNAa&`dV8`oF>zV?w_b1QPe8_mRA%fyml9N}zE z_-m(6zyG|m?j+Mnf7=xbb%mHqB&x=o>~}ut(o3hDKA)2v)LFgfzUPV|zwQq${}Jm! zdvqS0#f$auxa~yCyx|1clRx73VPI)bD(DG&?EH&%UAHgnwu8I!`Kp(SFWc>Wqg^Ma zTe*j+Ez4Kzf`(q!&Qco{4bZc|i%U<6aYU6B7)Lx7;53d@W>5_ia)5Ny1_i;Fuu5e! z-gKnZ5^0T^BYvyJ8eYL}Z1AdPGrK^uOnkDgwNvdLC@Di@t#zMFFbngC*yBaZnjCxO zZVNwAs{vvUm;SyZn;h!w92-hzJ6O%btT}YL>chAEtV)iFcrVtkM#9EvCDS2-twqu&y5y= zw;q?%OgQCDn!(c|X=^MS%LcRltks{LOR&8^`AO+?V#}7fxh-2D&&;XX#mAnwc+n^T z?I3bku^;?ONNGpAEzQ9|wZK)t4otF{`3c3+*b1IhG!ph>Qy^76GG!OWj>gw*J9S{; z4GguD#dS*bxuJZ1h^DeJ+j4C4fm1qeo$MT>2@;LZAJ13vO*7V9&^G2tG7zXZ?FfUm z#SMB%w5<{KY9(%XvO$a>;P-@EExte!yNWhJc8Fzlj6qNMLkn-vTJq?^8$)^3(jB7q zK=I-s|H2zsK0QCgqux+AWHJJLC*aI54Qv=}8o8CR zZwEnEGeI;95)@8khtt_i7IdVSr-7d=zV}u=kyugRRIfhw zeDDVL_QJF74|wmnm%D6ymv^z?^V}7hzydG+3&|d1l55zYhOj3av4&o`Cs_*%Sec7K6kNmX1R1PD zYix+tfd4N`+-xrWgR9=NE#s(Rcb7VHTc13*dDZG`u2Vy5+-xoVUX3HO%~S7URi&d_ za|fSnjU2xwx0TQZaKH4&{58k8C}uC~%bS*!t{HKh8i(U_G87Y4V6Mbq6(WCwXB8|!8EMz7QHK&Z*mcFpc< z+RRN&4^&tAL+^tIcvp=oXtiyp&{<>WDx_onB*c$TJG+1&G7a-fJb(lhUsyZ?n4aYuiGF!~%5BNht zkLp&(Oy-jvTIYsHHM$C!I<(f1-`DJlUJRPI*qqTW+kTY1z~}7?FWT8-kChzvs)6UdU2dnB zx$Q4tyPa>#r3G#wn2l*V56=aR2F{ncODvttVSQ>#9gal)dghYmi{bh)=H+FHv=R)hRtN(5RM_@E0? z5kM8i9$Uerye_+vY3w_3_P#}l!_lo1O@m<2iy=ee^_*n$LO%GqY8Q0?Zgjgfu%~GcgW`lM%ck$vJ0hs4ShNL&iUr07ttjmJdpcTs@YpWWi zLeN`YSMXY|ok4QJ?b0l&5gLe$Y$tuGLVQ^KYqd>=*0HTNl+kS35%>Tm0`e`E!ED_IcN2j(%)=h7jWUMUO0+h zRRdK=F-j8tO~s;7T+L5ZJE`9#xx)%NSO@&}!yd9s-zo3*_M|@$v_@C3vckh1zbO=c zQz)I*Tce|GeeMd4hi+VZwk!ITF`O4lyst z4Y9otCo>pme1^Sp;8gd3{bk67rC&829rHZ0Sv4^W_lM?+#W|mfdf9!dfV9s|K;O|StI2k1ficm_+HH-M&Az?i*JgaZ@5^* zE(GBy_gO3&{S94&SP6KeFT!J~`_y882z_O7zCy_m6O~Qphe|_ZM`==gUbZ=u2Swa{ zc-fe%m1d0D?+|)|HxUHK2lEHO%w;$(wR`cy*WG%iYh_pcDb`1TTj~Ka=bd}qEvd|b zQ^m{sB3zJTR-u==fD1KM#C|~QSdzg!U=2oM?a81uk|lZ~xEUA=&kOD%%>%Gb(5GU} zTOiHa&bDc8$;Tnw1g$O1?*a*kxmaWcc5HS9ORvEu4`$0U9^0!Yn(iJ=IPSjNkr=(Z zDY5+W^zl3}LDjB$vt0K9RLLL5oR)B01*NRQyg(`CyrhZKYKCkpBzcJRl8dOC)PO3V zwaRCOc~t7^!d#+yVgv-}OF|o3m8R8-X8{D#>>(A*N?k%eEp2Xp{Og1~APhL#`%a==_CxDO?0Cstm3 z30%#eV0U(fut|VC7qL}fR)`ZvgHV2zC*{}rc8UrQR$o+3OBx1mZ zBw=TjS?FXCbR;9PLY)=VCY?28(R%*NYUev|5yJtCsjYSrP2lsA^AtqzGR9J<&#=SZlzmY*a6=bs1jPR3mA)Spy%lFF5 zROWpz3sBDaoT_RIIQP`UxG^?pxxq~=8DPB}F$ARVc7;st8!RO5cGmB4ZoCptXt$F* zCv5*@5{La6dkp?4(js8{AS3-dZwU(s)Cst!XwFM`ri$l@b{jSbv$P3IT0yOVSP=dS zw*x&V*WCoyCHggs=e+QPsqGa4jr6auy%nO1Ao}q)D@u%U$o8tSy3nH?Dvbl+CYu7R zr;${9Fe_A8p_~#-b)dOUM&F@rV13*8{M%o^J~;k`hJ4<8%LsADky~hvVqJxtWL9i& zd%G1Mt!u5vSyM$+o%}ek3E&T+d^?dS@rBYBXD1idLoy_TzhGTt(IHuqpa=xQPQX9) z0h)5@Nist!gP>qOtZ~ zMv}`QE9zVNwYYBcTms~PKGwK=(ESy}0lC<7k|w5-tgTAbC1>SlGFV{0;z+^k=% zP^`6tvGjFXO#;T4IOYvy2(y&V4OomZUoa&6Vs1-oEuS+>A1T9w;)~}99&%k-92Wn0 z#WQ5b|rc;Pr&qX~%&%}F#z(-avRX_b{G<+PY*7c;v8*q~hfsmb>XW+&kft>v*aLckMzT1J z?H52T$v0c|wF=q6AAu|`zT{OizHk$e;I$04CdhHNvo^$$PQGVNwOorbI=H7r;%%PvE>$cds9X%hLl`MJ6ID0UQ$ zMeHT$iSw|nEZP>KML>Fm^x}gE6TyOH{baI=g|o?MIs%(H=}Lgtd<{kFSU|8gs^G;wS0(6~;HoUQld?%1QRZPOq4L+V$^Kce3< zza;Al%6f$Xs zJ(ifhc0+%g-EIkP+x_5%O&`B;lgFbvI(tX2(;pCqr(#uYQ^?=!6x^22htq48xpO$v_M&$&HhkRZI$5SG*{TDTls&4?T2*ow$^%;=-wcMati4n z1CHQ>9wQCHD;N>p7-?idNGxoNs;bt2YwvLPeckc+x|?c4{(9F?>4DPUv%A;0{U0rT z_kOmD&oj?W>$p&VVcQqtdrO##R}$gZvxB^K55{&58Yt zJxOe?lC{aLO=P4@bLhDSp?60bYv?&Ikwm8{*lPk&G^LoJkdZLui?+rM>F(~;>w2o| zMK;_&(66yNkzdnZIw!7G&E(FlJ&^0YY17!o8++wN$M&_u>xQ?M7Ubo=DWd@UWC>?f zaBRpICMlP|)$9eavi2=$}kiDm__jweO@3rN;(HfCW16c9Drzu=v&AdeV|?K z)Hl>6;GWe_22rqia&JR(5=A5kv`TN7kZQ7Nx(gj9+tU~<`a?Zgk%=6%J-S;Vf)l z0Lt7Py8yV%l2=b$%8RSCQEe5x!D~D$o5J(-tk}HN7&Sr#rE{V&8p{&>vO=@mh5fr@ zQ*622sGaQeFjBNykn}REr5UPzt2F@U1^%tXhqD=YE_!)(NR36wpAto)W}`tTHWeJ$ z>Kc}gmd$AFZ|-gi@CbSTFbq6RJAy4%%b{gEY$%uTDdmFttp;N%I-l% z_DCo&{xE-elH$n7{aCg!AftazXDcW*!Ul!TUdgkhUm~V-!*`ujvXDvFDD7)ohgPl3 zWm1X0-gs9>w5?TZZfdBjTAsney4@_8{!`-jJF=) z!Ih4dvLfo`b6!xSXZ<1gZ}Sax-i2Gee9%xRy`{56px72K`EN^adc9{21=65bkhPMa zR}Dn3Al|?mA(VFLEopIu&Y`6UD>6tJS#HW#Rgp`MU*q7S=7Roe3s? zbg=ZL(wEq2hzDcPE1w=LJ;!!djFtF|h&6!Q0rm&jArNo?F@_L_;&0BWr8|IO@M|p5 zV^z@OMSa^7_Ik3gs==b^kpd(=UXG#yyApH&grKsGYS>(CXI*eP5|0)*5;5XqlEGv) z>GAT5Uhjg%i|r)ZqCAxW=_qVL;vCo@d{ur$1HGvFS~T1cs1i7rfLDhc3FNwt#^9_X z`3W{;p$@^_j3^24E}?yX_{*-JGFZvcEqWTGQ3FhTSQW5DIvH?aGyF zk3DtFNc2_PSEc&;QuIYu!pDfmBKavGX=2$iW)X~27!K12bis%qj}Q|O76PUUm*Ff- zh(K=yW32f=f-Gtf8ik+mT7n?g`{Fb;KX*699YJse1^RPncoAwWVN!L?8DcsO|&<8t7Kdq z`Q9J`nkB+!vSBC#S1)l1?-teTmXcyN2z!u8TG~Z)8QW1+P4O3{b27q$os{tyrP<}z zx7OA-`w?YU^oCs3PI!_{W{^hEMU?qN`~?|#F(>0GzkJ~2VzhR7p{k1)r2?m6sBWH{_0ElUbM_IgNLK-IGf3H)siHZ*NlW8BqDLfvrrdWs4Q)9dtse@ zdgUjCVS;eqtTrRor(4+x+}wGcodNd|HfhW?)@zo&Kqz^^fH7$!vL>6cBDm6s!HHpl z#=MPK9r)$MtSMq*b3{&d=aeH*<1sr~L&)!RxEiuaV}1e(iF*QComGb3c$)@#%l813 zpfU5g?P{nz=baV?-BPtdTWz*ha}(MUGZoWM{SRhCnFzkYoX}SJUdUO7!Q6JDaqr(o zLb8vfcTx_Lc_9mdGtxeS>Lq@OQ_38%N{X~2GqXscyW%7GGs(zgkD-Vgl572IYkT7z zkYbx4!@3a-Yf@}N*%Eqw7JY+R{MNh>gF=GJk+TUtTB4p;&mta7RDt|*^%O%D@{~bW zj5rfJQ`?DTU`|A(F)!2;bd*BO#H?&*-40?SRIJPwWee=&%AG603XhI~c)|FF{nSOFGh!?# z$5_gC)e2iJoat~E2P2Di)sxrX1@%rZu%q~ai52n-sVc2aS;J)k-@p zd;{Wy3fO83T!q5&L-ERaY7XE@%u(n#W=fLr#fwEffiJ}Ja(e<+LE<| zAKks(g4^Amu2r=T-DK~?6Q#RO-ipICub*04fAsAZ{tmxK*q(*0z{wFf2t!Mmg~HS< z>`uZ0#bj`lsuhmsPTqG=(;VIR-t}1S__ab%HRvO3wh`Qv~V zG&_H|9c+aQBq1r93w9*CE!)muNoGLTzeVug92sfn5XkrE$Maj-qZVJPLz8<%)fWDT zYO|`pyy$C&v*cMl#O}-w#qaIxfR$|J=B6QX#Ts!(SZYHyqH|Va4G|3|{NW@V%W!qt zet-|{BU!&P7E4MthFhYdjup5s;)wu1vE>0W{6qMs6irp&xM52#`!HY%^9b?-BDCbe zxT3yEmE)D3l9RN7s6GvaZ1A$ap@)-g-y;2CG(Ru%Kn)<@5P3$(YF{3Ys4sm1mF*`z zWJN{{f4O};u>=p;jThsI!xA9IeMQin>M|XGoeaHWV?;bj0bXenCTp2cMTEYoihVET z)k=SXLAtLHE$8)bgCWbk^CZ^uo50^ynC}X|!3)9CL!8!NHBV)%i$OWY;Q<)FNR5Mo z4G0$|PZum+RFegqHeo^SJ!b+lN01IFab2NDZcAX#&JK1aZhOSX=S_p1CPXYFPML>S z{t1QZBuJ+dieKX3Gqtx4c6JWlTKmkwgbd#yxGnlb7U3qvWdPWihk${mv|%2t;aZ_f zErt@qWwkU`(l?~sxh#bEA_&UDvxt>Oe1dPg3>+>wAcoRtAd+J3N%#cL(0DFAuU26n zES^bVhJ{)vSfFOi9XS8Yx-}iIfApF2kMsF8>z+9uIQIDYXFmEm@P_a}#%Khw&JNO3 z7{ZQ{X%IssbOJEqkCBHx!uFCK4rEXK<44fI@&%>k_5|L9(4Jeg2hEx^JvcAZChO9L zXUGK8BgJV18%zJ^ca5CMmp}G1PyqzQqs0E2t*dmW%(5p;&en#281ton$6v&pbEmcw=4n?au4S-Sy0OJ!_)R437?}-km!s`%H9AALC89lE}Q4u=a{lsF?svCed+$tOaa z7j01y!_E-)lp}n->@^&SN_b&c_#Gi1sao0GfB+13L7b4F;FcvjFxlAyXuB3Cz*OnS zLFh&Xup&LLHOAWIaWJ;Gp|13!8P;+CbFV)7;c4bB?f;u|8Jq=COLwx){kM8wdEn7k zcQE%~oIlrf&ql+pbLmMzUxg2m>^jTN?ub3@vBo@-2+8o<8-?zdFfJ=@giXjUz22DTppvsdH%LW6F|Deg9C$UdSM+ zp7x>W(CDkBH(v!RK|E#3)|M^z&|%-f{gIZfE&V6Q9)0!IN5@WzQ~pb9rV1&%>T3ZX z`D6q>&~aZGYfl21IG+XS6HKNw`!b@b?0XiT-D4M*6e4FY{oGzG+F64gv%yqkd`1Ny zq8KZR&sg-iQhbIXD9|A=I$A3-(&ZcZ!(Y^Fjs_FH{2%G9mVVYK`jKbF20-6h3|u3L3WtCZ?%+>khd2<9P#On9qR?tn zD3Q`R#3ncc!J<>KUS1s7Jz#gM>M!5}2?cAq2L`%pf+4FV@C#LS+sik_1<$|B-OC^4 zc~K&91~DqX1|25-$#%9k?h?EXv{($)X`)ya*weB@HV~>Po#eq8OdMbMCb%Whq zt->d?0gkZ?msD9O$U4ug~o53-O@Y zXY)D(L1$-uYkOUfV_X05!g^AJDrjj7EYO>jJw!`)Ub{9IZ>u7C6|__a{914>6a(r- zAdQtqM)(Y;zq%x0Tq$!HCGA(#kukJu`aN5E8$&hQ_ie8UH4b#7DV(;!5I-P$_+G5Y zv(FmA!*rt@$D7<<)0J}cuUXUYXkB@&h#z*4P$JCDMPmANCCx6lGA+BR*!x7Igsq!& zng~K&B|pbm9V?97=_G<(fuzEJJcu|49L9g*%a%Z~Sl_EX^8~_w^k+V=>UyvC#KSEs z5Zw;m{_<-o@%`vaFGcm&URL$!^UuTMWXKPK-uM^!eL^_$094|_*&whq>dvr}r|-VI zbncGvV~A$?O@8#qvtM}oZA8yf*&c}1D4`gv zO6G7O=P!87;&V8M?59KS=?E0SB7G~Uo{)jDpY!ktmHUC9gJandKaOyhDJ8*2JWXR; zqFYsXfeG=kfY(_q&NzA!ra&#WB5#Wz{F=hdkYX#IW}QF$Nb#xCUqAgCix$6p@7Pfc z;v+vS{pj@5%=eUDdgHZwzpNjH=DZ{aRDohqOagFMYYO@(FbTNpO_-?tUXFIb(H1*E zM`hE5{t_FW*KdC6zu)uF&mYv!KO+?APQyexUwY}Kd;a@VH|r1n{Gn&gOJ%!kC>3&` zSjRA6;Sq9MnD&ZP`jJv3l(dveW`K|@a{7}r4HRZ4Ni8Pn6tPJ#k9QV@o%CYqoRF@? z1&?-$bD~@TlI#PuIM0a~cyE=U8=wl{QDu`X+%lOkp)WQl+y+~I0)nr{TS`MM@i?dG z!Hu`OJ#Re$k`3kjUKFk-)zFzjPXGpqjQ0<5BRHvT`n68n1WDt$)8LXx794u=Jl9inhOTl zy4*tU3>eu#sT3Fv|_Nmk$>MddiLLcl?ftEQR)K?w&D2nwZuD7ZAh`NI%oX?s8k zMEAs_A-z8f?rCt%O1ysWHp@C9+BVuO+wo}IE^kwuTNAvv^5k5M&d#;BEuEgT8fWL0 z9aW)2tK^1}=hl|eE&K$b(ZW&u=HSjE^TXmVpU0gy%4kL=MS`L6Q%MJjmI&Jc^M!YV0ahT)5@ za9#<`svH+wRt?I;;PUeFb@@K~un?<%EPlC1B&DB=kR@r1F@m%gzFk>ER!6uB6>bv0 zWamU)Sd3)3EctQeU6GgcQ{XzSTRrG!5QiMChEIC=GQpYzT>vrtt^61r^j~-gzuVb` zAFm8Gt!h#=l(bPf|8ICxfYb;QiA3f8HDUKtEU^)LXy>qjibDbva|2t8qkJY%y!_+> zo&3h>Kcexv;0qLkSc@^b5Q8Z62^{^lvUdE$vSn);tt0S$=Tk_x-d*aFu!0Ro-Y9Op zM;sS`p0Y&W%WI9jRbE%@t+Ie$Zn?Z(pg^bE9+ zJX1I?X2i=u$_Bkf#13LZ;3nn>0eJ#+fP`L91YozIt)D|_xuBB&(Hm_1fDOI8MxOB( zGCOz#C^sFg!x=PeGCKZ1Co<gp2|!4jrbaSO6X!>?9ULbX+xTXvAmyQl}9%v~VI= z3!M8u(_J*DN5n14CUSX+?wpH_?oUJJiCINd(OXJh+ks_BR}#7t1V)I&!e15kkn~O@ot<>Ic)hij70o`d z$5cbTGh8|yZ?ffvN{0daPq(P5rQP=gIt%$7Pi?-Yg`I4&9r$qRpXgL5=4R-lEwC5Z z&PKGL;Guw-I3Xv6FR~bjNJXixr6V{?EQ}zK$$_4FBGB5oLYR=u#~x_PWUkePBgr`}zS=;U4%-t?Dj4?Q=CpUG}+675F7%!W>pkV-far zsGNdN2rIgXFUF}%kaB517sm6;&K|lz0Wlx9i0PzofhBucDgzcs`!|g>Tuce$Fc-)k zK!Nqpt_MFS-1Q(hI@u3M8X?0O+3IDm2HU%sVg<_U2YyKyZ9D6$#d$%&>K6MTM2V(V za47Nq3y5op{f}XPEUYJ0mqZ+5Rbxjf%)C+$0ZvpyN{nDm*z3`@P@M;xMetFn;L>IZ z8wblNZ?4Fbzl#nlzhLK+A}Re?Cc^K7lh&nXoMQed0&rwnBu$v~U^qVr|Ce~Aq&Fl{ zc0(%yk6aOtwY4-g7(9i}m(#l)psZmmBE>jlN=z9d8Rnlx%+s>8>a4xUr|?sHlYYdg ziWn^jq5W)?{KY6=#%omY)$MzrwCg%u(OG$<7^6WG0VjHA1-*3wa0)m1-DC^^oXB*6 zcMc$4h(@p+R+VrgF-XFSr3H|T1Q-khK^aaGJmqVG5z!q<>q&nRbO&)SkbB{)kHpAo z1eq88W)k$;6=L{^0e~qsM8N=XGo90gXe+{vmUIJpZ$KMpV;hdp3Y!M)_ZXCNyrKj& z0S4;`oiNA_(IJf}y-Idn{9nm!^>p9}5`n8g}>V zUrayz^{+gV{$l?8bb55puFaX}3@zx6u|0dn?kJrb+O=ZEu3wh*9|1d+{9F_%XFJ>6 zAZ!`*IyQe&kWexolH3mqGT90gLz3Vz%{5t^R3F>l)mM6}Dc=;rzVSX*dQr#$(5P?| z5hVt(sSYrJlWqR{?Xxg96*D6-wK{Y7L#b~VfIer zzOlAP7Mk|$iayeI{Y>M+!^!Xd6GQO!KQ+xrrT&F?_WiQxm?Z??tp^etdbtAaLlWc)xcYL#)OVvH1n*7eUFBOS(lA7c~Y z2IQT6?~!HXyAD|W6W!IHsK42@>i;O!z%+c8z28&0^cmqjR^UAl_=pNvLsh%<8D&)c z7}Zx><*HKN`22)XY&|}#it4`i7q*Ufty6iA@|D*VYWQAlm+O|(%KGK9_j;b{S3Xl& zm!5w=ZB#zQ&Z#x4Blyo$o9;7x(e%Ge z@0jD}A@g4Ilja{g{GwTJL#a3tQvK_O{*O0kr>aOb1>I2meR$p|~I<9pbbUfuaS7WJ}sJXx9$(nD~{GGGS zdDMBz`JD5I&XOzR+UnZp`k3n}*Ppp9?wotK`>6XQP) z-Rt!o^{eV9>OWfl#rhxAml{?z9BBAz!}lBBY`D7XE3jegVp>?=*qV+`US6knS)J0B4UWxp)&DplOZMN;nw(qoEY)`e{)Ba@p8&Okq zWAyRpUq(x@q1aUHSnS!@f9t60*w``K@k%EJ-V)#Zsd5032=w9NmwcF+>f1$LfnDs6 z7U}S?@}QAt@I3t&BTrEn|J%r`N*h~g=j5;%tTT#VU)}> zSRnqBk>{{x{8uBdDx=D;jJ!#yWj7mnv(m)wHS!iEz`m%A;1%36$|PR0O|RJ2lquyy z_}z|3p3V4bcq79>yq^0oUc;>^cZ-*CA3$!ScxCqyksijo!DdjFK>a?X9e~Xd{LLyW zVXIo9>@(_8D(m**rQiEd`yie>f_D}vBZp@ukId-W)Q7a~y_zD2wHmLmtW zjfV~%*?8#i{uwRN+oyFLIC5lm<%$*iP`Zywd+*%WdvN9m+NgNf_%+jq4q`=?y>I*$ zl-)9|yywVQV)R$ObX>zcG`v@-2X?m}%(4&p6dGDKu$9`bgGX*Ta{G+ludUSjd$K)= zzJAoYvN>h3qVnEvK;J!c_|97n9n|`J@uw+(-YnpC5Mx+2u|u;n2Ybr1lh~+SdI00R z+UKVz#3^9LnaWIfqmu>pDjVJySH-H8^~wf7XA>~z8s=a%piM63Mzm5b^D-avvjFTs zb*!E>uttV}2*j(kFb(lct$6=T8*67#7GoWF{c9KNhW)Gu@x&`wAKvbapb3^@X_kSM zpJM}TB~B-)0?GVe8ojwvlaOqwE^C880lpmR-lTvTbZT+rh@z^=v2G z#dfm~usj=QH?TeIMs^e1%Wh^9Y!dWyn(1tY?PL4d0d@=2t}A7qEw zo$Ls^iydWmvt#T->>l=EcAVYI?qeTe_p{$&A4R=}~ryJ;px8{wBWs(+ak*ctXb`wIIiJIh{RUt?cq-(WAYKW6jnKeCtD%j}!%PuMH$ zPuaKFx7l~tcUh7BC-!ITd+ht{RrVVDbM`v>3-E^j%+9g@!hXnp#Qu`~m2xFed4C_r zX@~v(8>f@ z^K^!%vpk*S=>eXemG|%WfGs83cc(#vc`*}9Ovq_#!@obuBGd!E+*&NRf@a!bd zPVwwC&+0ro!?XK%u8-&Xc`m_oNuEpbT$<-HJeTFU9M28#+$7IU@!T}e={z^XbNl!} zA0O!F0|`Emkm zHOZ%@_|!C?()rX3pW4T#`}lM}pHA@UB%e<4=`^3t@aZg{&hhC1K0V2&r}*?VpVs;G z44>Y|^**lmb3MWJB-c}1PjfxP^(@zOTp!>FWY?#-KFwiu)Mto(FudR2RY_h7N?a=_ zyYd^xHEqk+73YpE1TKJCP=e1W%5egj8?mFeloRAV??P{s?&NM!x< zXm4a005N+Y6@X4bOM5s*w%T8^-qJ!;x^~iM&?WzC9lcfYveKkp=s=Nir4{<3RTUKQmsl*>#sPK=L_ zHx^j;_;{qCY|qb(kM|VRxVAwnnA#^XAoIxfe8C(UE?6SN82)&HP4pB@@d(DH>1WJS z!y4U@ofoP`3d+QWg4z{E>4Y?vVhesuxa#NFn9G7tZ|J7SUocRb(1oMDj4G0iE*kj zv0e<&7JuGat&D6K?g}pg+8$pH_$t{7>&6g9Fxv@j!->cwErNiO(nydjXpIFdYa3NKRZDLrPK=)_eZU*Udc=*J`nOaMC z;c$0jE5PK#+`QdA1%Lbuqci|GQyPq)Q7Ns9pD|HdA3tNJv>|@RLTO|CjFr-+_!%3e zq4*g)rOk1rP}BV{7)T2S(u@W)4204!2102o2102B1EI7H1EI7X1EDmEflwO5Kq&3N zKq&2uYpVpFcf~P(_k=crMVO#Pn?zdZB&6z&7rMF&UDz&hVCp8I)K&LOWHJ{aI`y74 zfG<6Tp2am_fkM2i!2Epz%Dt6PS$=CpTuX~__Mr~jaOHLd6}alKs9XtrRnXe?Ly_E> z70i#B^kd!_=v5z?0M<_CdJ2hnZ*WylA^F>?0>h?JJ%y!E0_|F_wuyEoKzPlG6PqHN zKne1o*PwUUu1SVSN%Wrv2?+rE@h_?r>?7SXCwe2Aw(11h$}HX1dSx306WT;AtuR5G zdF_t;SGcBXjbFhF!5hYhiNM)FDA6B!jBLc#!YVG`C)m`iTT*d8GNDHb>d2%H8pB5> z8~6r`3`8wzXbaTZbVmBMRJYd ziuDeU8)Fc$e~xpta2BEhJE9 zQ@oHuGD=X}0Jv%!!L!P6x+YHOSQrIZH^-k>ly%5#L55N0+W7NKlw605DA`JNhH+~f z)uGIGszaF_REIKSRA&g8>!}W9c2XV6?4ml9*-drUBJ%;NLzz6)q0Bhdq09|bX9Sr& zREIJ*QXR_NM0F^$m+GuR=4PrxnF*>xnMtZcnW=aoy9nlKx+n~ySQoif$ju0RLh))` z?28w2i?#RDg{XZ%vdqYRqR@Tr+G9AMsVLf0GmB@H{k&9( z$MeMEdX%D4)$7*{jm=ME&&yC9P z5Iif6Z;~z1Ves>XqTo5s;51bGZ?#U*(Z8WluQScPTCKR04^gV`*3_0;xaw6`H2dQAVS%Dq4X|gY2a8zpT7?rYl=nrE^r*8M62n6<51-) zbynb5S0dELz_CRMSC3!?)zGWZ6^+q6Rmd)Y*8ZBUCJ<}6r;#h%J5x)=g(6r@tvg%QbyuGN*SfhP>NBf2*-2qU8YRMQ6|b} z;F$KM%Hy~<3adCsiN(GjYLsD{siZ5nVVe@DOMA2KAY~Rx2cd;R)a$P(!%7Qt%L)sk z@+zaU28|pPHEKq2X;IXiqOz$`nZ+~8GK)(eFN}&G6dToVYFXLL^xJNmg3>8eI%w9E zK{E==(8dTQUv@MLhxx@buqz6b&|WD*SrPXC?#a{f^yB2XXq?mKjKrag%Hx!QN(%nt zF~&G05e;>Du=J>LGs=p}rWY2(MWsi@4NMsr9~*~Smp7+esHiC8(M2gHqewnEbuuXM zABBsBrL&5PXGFyf!iMu=%xEE=ZeZ7e70)c3F)%nfq6_oCcYtzkr`1MTZzU9?0QF*CfW*)7K1+6`zJgVd<6P3we@&Yj6RAm~7d6y!czsZgF& zo>Jy1)yhJMn59aMvO;-UaVvGov&t%^L0PM;S2ie{lr73OrAgVTJg4k}8rZA6r0iE( zl>^Ev%3XlkfxQ4KXr?WRVk*Q!0#o@%6eoqB`XTXm>W>P>32 z+E?wT#;CWdgVb0xUQJY!)l@ZIyIlaY3g)!hB{L%Rm;@bYK8iw`jk3PtyUMRi`AuSjk-d8T6L>+>a*%9 zwLx90u2(mxo764pHnmCJslK58mwHYWaq$U>Ny#axX>qY}adGi+32}*WNpZ<>DRHTB zX>qx6d2#u11#yLOQ{rReWO4N=iyn=sX$fhGX-R3xX(?%`X=!P> zX?bb+X$5J8X;X4zbK`R3a}#nCbCYtDb5n9tbJKEjbMtcZa|?2(lt(<>luU@)VRFGVdQjl7ZR*+keSCC&&P*5m^=>NN#xgfg(Dn?P4flQWzP#8$% z84yb?u*F@_s&^~*fCcYWSAuxzK|ZTNKx;rk>p(<}Aft^Sq|G3utstiDAg3K5sAly! z^?7v{2y3^xN8PKwsJ^7`Q}?SaYODIPdO$s>zM>vd538@Luc>Y7Z`9XSkNSpsL_Mm$ zsUB0`Qr}kJQQuYHQ{PuVP>-u8)DP8@>TlKGsi)MB)ZeQgtA9}csD7e;s{Tp+O#NIv zt$v}NQU9#|Mg3C!O8r{>M*XY$t@@q%H}&soJ4pKxB9cDXsV`ZAzG-WYZlE4Bz2V*riE+Ww5zoU?HcV`t-IDkvuQmwyB4YS z(yr64*KW{m)Ou^b(j1yoi_-dNH)%I((b_FqU(KcU)B0;M+5qiVZJ;(tsnc%LVzoFe zUQ5stwInTBOVLubG%Z~ltlh3dEbSp}v^GW?tBupfYY%IWXxZAM+GARdHbI-HoFTb;Go)k{B$pqOQiQUI{pWUN>k4Jhe?yuQ9y1MILy6)TSM_%7{{hw|abi?Qy z=H2k}jrZO-{>I09NA}L>eYm&(S2zD^!LR_Y|9CP@b8P0uCiBZ3fs*P%i`a_?% zK1=)TxoO?a%cJK;ABz6*maA^L_m+jXeAxH;zLWcY?YhzRtZS#M#r37@d_Q}?n11*4 z%kHlsJ}nvp_nZLZXJ*{fZuxmt!r=nao__3rwyzhCR}d2C)`j zc8l85!WXxMv_$fce9w!IEG_;8c3(DM?9aAFFfY%cKeZ#v8`AR(_jF|0qr&{rBFFCX zN4tE{E-TOBG5Rl6Y)3_rBVsuInb#N1nAac8^ax+OSM}BKoDhB%EsAj>4%;~H;Gx(Y zv=^bm;moGyMGm^iaWU4Wb5!K0=#UNI!9slFJKcYI{Yx6Wct7)+9}FzCPuTe^Jm*d3 z?!p|ryKlZG4Equu8(^0 z?rlSuA(};~{m#1{?aPFPl|EBeJImnj@lxGq@a}dI;Sc9Cm|p)v{cg6Gotymk%u|Mc zy7<^GhKcU_5uyJpiT5ls4)XE#cSW|&uV2IUKfKRXBjVha*(#PUgy(d$+Wj>m$I4d< z4`Z7;5EM zsp7?2%zL4^P*jl{qh=Ytxrf@jykoN_o{btrMf%nwxW}tKq7JM~CNHu}0 zz8bok{tiZ;8fKh2rH^}~=nw2PJH6-B8*doC z#ivk3e`DO9VJwxU7Tq~+oN;QHe(Kc0vy5x_oAi%iprZ^CWq#m9}4 zr}WB=3wE$(*1US##*GFq`kg)VZhd3r>M~Z$iWihrRvIUV=`X&x&BKncBW15W{-O~v zXv=J0v@cp^zG!o{`-Zvv<#r}c;c;DzpVEI_J#EocHkB3CPj4_V6k>n*Z4TTO<_bN| z-k$y1RKuU*Ptm8oHv4UMobhyi1GaQ#@EXzGzW32Bqu2;0(!~wf(s4Ly%cFa#Ihsc) zr$WHZ=d(Imz2~zqhrZ}YS`lB3l~xanOr$4e8b~TIogqC_eSNS%^H$7Tys+93^TZy} zlQ9>T$*<{^ja3^RzUM3(8yhz|eVW%RdRk}h7E^iM@@J}7EvTEf!f=b8b{;K;h*qXA zK`;HnxF@n-ScDhS&f5cn#1mi%ZQrf}9WAM;S>p76YF*;4S?TDw!?M!tUg_jxthVp* z{1)4{EASMn^oQx;R2^bgI}c34*6?`!(P0# ztl9Alt9|+zX0(YumW5A>5HW2+Mpa2=5u3mY))($5*-^6Zsr}6Gt+MQ6FE;LIGTfFO zJJ#=G``Ig%d#iR#_(X*8X$vunL@#K{Y zbjIEj*Brgc@Q=3~{oy@+4P(a2)r=<-&(m0>^blHHoY0)?=7$HS-J4fb`WSoI=xDXD z*Gpf`+mrU;!{4!g8C;9|T4)Z}`7Ha`S0)}g^2#em9424KfD2-{cH+db4wvt+HK>`K%$s#4xy7*gcJA45kR1*_qsVdDy%xHSZgILS)QiRT z!|4;lQ&WczPj!kIi}~mtk_H}AQh*{oBvb<85VYbA@#1<#jb5;5`t(HwMok6tAJ$V( z3_tDg9rpSUTZ+pu{a6C0@38N%g%-k*Ej$*N*9As{00u8gKEyEC`BrmW=%Axjk04o( z;(+e*e;J^{Z6+1^z7%cIV$xag2T_m5dx44|AzSU{u*4XvBw?|{TD-Nq+0l_@kq^U{ zfd1S|9AXS6Vd5)e9W)=9P(ez>e z|D(Mp*1c_@1u+C`u;{}%N7--K{)Rmpwrtq4dG%h<_15ZjbJxvnC}#zR*TRlfy*}k7 zW6DbpH$KFS2p4fKhEEa~M=7nV-AAt!w8;O=${bg&8;w<)CKsg8Y+5B_kmY2H)wOZ8J_ zN5*a&W;Cr?zm{+Eh3oFxr)!th8j}v{{tCatKJ=kcL!GSOxWvH|_Lm=?|0-mpi-%)# z{eINjL!A*z|M4Rb)ECV#^?*H7CgD+Nh1?as~4BgDxtwR>sTAp zS=lq?wX=vkQC8CR^Y>Au}aih*=HkItHXx+ZAW&0uHgQ+9ESW*Zn?U<=ujnkCB& z(Q8EUR{fLH8GNt^XZXty8K0&bGs;D;hSJ^DO$|*A4cHk&c&6@Nx4M2kGngA=*XH0v3OCrvg+U32OFpu^X_o z$mz%eO991t?Ed*(JM+!A`r9F#E^Qv?0PtPPsddTw0z4>t!kO3R^$nzvuw~1ZFEs{= zk-F`RTLR?T$0CKB|ADUT9h}uP3+}32US|yCxXZh|ZdonvvVGxy01p~u4Ppx? zNfC$5%g;t~?Q19oQ$67OYpyv_gq_0`8WV;k4E06(fi`^6rm&OR1gwMtf1t>eeP$JW zx7+D*2lTTXpoe*T@ONmSwpV*QhjIY&Xk?0hV75F^BU)`L+M$| zI<{d=?ONkAXcF5iwQHBInTuik(VxW%PoZG(`Z;T##BAh%|4oHB2MUq@e$JmDOA*W7xUFP+GDlEWOyOfdHL#%VFtLHk0aL>oqb=3`X9YY`oNX3ayTy}Zsyu&)T zp?aO8!(mz1(6G+g;RsYDE&_zY3Y*xHyS?}$bVpVV0nCA6*)9Nv(#HAvb2FM}?0kYi zbLrMu+sd{Ze1sKC1gPdAYY6LNT9%lVt686%g%6+rwJYzzsyFxXZMQJg`i zjEA>1&&LJb%i4H&^BP<^bt;>OuW7~==EZ&Un{i>-Dco1QM#mLBTe$5(CenhV#3OHp=L5aC?6+aMr34S)3pyq!n`I|KN;uEi=E{~*l}_Y? zw|TRz!IRU&Pk`XO0qVnvl)u@oHmkhi3YDriJKK5zY+wQ+@I4jPA1vm%*N78@?CxR8cq+BKU#(3LsX4^f) zG>K-4;n-%1nH+mQ6WefXGo2h4P&5-7aA25i;}BP9To@>_pPkKrwrbTP!0L9vNd-&N`?Qt~w@PCkx#I#DJdxMt8^pU`x z@YlfjlAJ--gRCp(UU~q*8q%p@e$z#AngELs$>U5wF2LIX*)TqXM87GSr6LUJITK?> z#lV=IUQ5v053aofMZtk*i9&mN>8LwdoFRY@xE6o}?CVi~NN+N-62Nvu9}qQib}^|N z@SNvcJF=iqZ6ALbVPt^NDw_;Snu&(u8e+Y7 z^yqt?*;aP%fzijS48D4#zHZs(QudUQE%g=H$ugfUbT4xo-=Q&9w551k)wZhUCC@YC zV-U#4mJi>2^FwEwm3=t*%@K`;Sp9)Mw{}hwTMtb^TFk-SmNjfuO>K=a(Cf9bJ+qt3 z8p|4sS3bdvAztV-npz-vpoRppD-y79fgN`x4K{!awaQ!&U3>*v8(r$ziCR6G;Vc zQo%dPn7DG9HG&5wB^4Fv)zzY2tYKn?A=3Db;zpi^?M7^A4#sDQdcLN*!4UWRM@k$> zgc}q&Cg_u9CCO3~V~{6=5Zw7zDMO`iEkLtGWRR`kSsE@T09G(fgTz`=5fQP~gr@sDLbk-_3w#{RMI7`&7 zBvd7|MP|ZB-I-|OTbZxBulu_r z_4?{f3)cos-nEN1ET}gIefPm}{n#<~_lJ&+ezQLtJ=z#Ca^Sa++fUZdhscIQVTDm+ z;kqcc^IoEtIEk$%zYg+_9Ihl3f@03J9l)66a42P%NZZQumxE8sAwUIsEIAcI&+ zfBq={%|F3k63}^>gP6x|+j60z0q;f2+ijQ{lB&#UF0l!WypaTU(7F|^WkX<0qS*w| z55g)-$DCw~95w>o-T;gy*^;m?O))r5;v~o)*>(>bI5`x$$F>EYTNuMOj~C$tJdS^S zS2q*%EFJ?$K}tBnnA993lR)4~whvZqT{AcT+}2I_L#(=L*&DN7Jw3Ejhh%9)?)jhj!j`R za~D4U#NMg>9#}r1Cgm^lPBP&3-OU#ng{Z_R|cOV%&mcy#+d>77?Q#$W&f(GnMyP8Tf4RaEVX>j3uFRiR3V)hy+ysmzPK&k!bBIG|ja0!VOiJ~lMb%F6g-Mpa_JH^E3v0uo`fA7d4F7z) zIAE==U)12}h_N)(*Ecx%fuO4s-oAjV({~u_Ai=LW4ggDnzdcFQ0?JDa5AU<2yllAi zy#&$WC6VkCb9p%!(KPL_TrLy5!{JPdDOgTsCB^{0$szZqG*{H)ak2>6Z{1Rj8BJ6C~CDa}~hN7;aFXc0O;4N=;fPz08;5m@5i ziEsIL{96hgwXq}6Rk7a)q(j8U3M5BdJeKT4jE#*L2EIDjP!x?JRgK4|Z<1k9#V#-0 zBv()h9j#Doh@Zg5la6s3ErWlYB&3Tx6R>8`8rgcCm-W0muySs5YU6b z9-iPi{v*!@f*}Yi(U7#>f|gsrfWyuV zzW@6=R}8lY;_R1%+et$ZotX9t_94E*B+o8*H>wbDc*=l$J4%#9I6%^q*X`EV*EF(5 zEZK#;0n?8IquhQwp>9+Unt}WVtog;bfH(`SDq^|@2M}oj>qyR!;j(2===ysgP0%#a zk~iqmHKV6ANhFDgP{GsC#rBLa^E=|43vSC0{yD8WwT`)xuO7pX>EbCj z0bpnE+B;2-_iJaZQT{Zz4%tz|n_7`81?p9m|ifZNpOY2LQ2 z*~zw7Y@JnW{CGt#y={xwkFZ7OXrxJwG&xR}3=&W%kvyl6Ri?eoA0r+M;g4bYU~$tj zS$Rv1eN0XMoL^5fCQs7mEvlZwo-!j9>)ED;`nATvgZiF5C!cN2+h6eX$ozZ*f-vTi zdYh>pglUZa$tR3=&-kRcdD_Ou>nm&Lu*wyN{~GbObcgC08BBElB;)9q&#Hdgv~%^2 z^;@?Z2M+3M>l-$+^=1&_DOORvXr3`?l3rAlxj3)2VE>8_T3XD;>+4rGvIeu>a<**6 zat0{3h%KmI1{iTr900zh6}Lw4Re$^L9~s^rwrbyLM1joVbsZW#^5w&tH0klBCC`*R z^Hc+4W~c+`lp^&{HdL%%w0_a1xotH@Tg`7bz5DJJ#%om8&ZYrlZE{4FJ^Pt^D@Tno z=j#e1Ut7QW(otVNvdKM9EDi#{r%E;4da z3rYY@xgnv*r*jx80S&pKRZSO-vdI!|FO{y|V5S#xy^!(6$2s3($JW2L!@aC-3A`T&8#Gq! zp1X}5Wrq&oYunu2RgH$rt1qivT({J{^R*3cGQ@R*Nnrl=P~k*sLI`(ayRb)ogHzlj z6l^y+DZoLlD+~p$JE<&#PDPUa(h4N&B!?rd1Ww0vrzXydpIEiL>fqi5z<`>#~JpNFmqun z5f=~?X&jw3Bp+;5TpT$&nBm?2@BdxH!gW|N#p(ao!8fo zLXo&N#*3-4{ls^HJ0~xgI*Co9a6FtfK`R}Or5skPOV|VDwS4h%Lr~t&MID{3+s-l3 zkE_Q|yDvF7_&PAPz;&-ug=a3-DyJwz6a8zG7U(d`Gp)B*{y&pcqwc{rZ zzKb{OEiE6c*k7=}VEF@6fCSuv=?fNAvIVObtY#ZmuQr}_fBjwN$pJC?V~?@hUw!P= z$3A7RzG}dER1-u71^XY_{0N{ojC{yJf*}%jdv!mO%iyCjZ4onAO45_~%NLD|BFZd6 zU5YW|wnx~c$7eqL%DA0FSqhs`Q?jIFQ}xD0TbXhCgc;!;{xzHqCxHqf9c29bL>!_& z7q9t>#Yy|*M@CH_vD~nIw6k!-1eR@#AhBg-uTMWXX{&MG;j&LEpFRnRR3hDKTMI@_ zM?Mu@n>hZ#>6t8(J-BP42bz~2v&Q63$Oj-}Esnx|!tpiGF1gmt9NaiWFg2$rggM-2 zX>uYHis6ET#>%*o{Fgp;;~pGZkj~QC(Ea1yq2!%5ZySU?S(s2f#N==t|Lua!95k+c zd0mYwe|IDbAsq^)8js1g+kSu)BqtKZ1!GuZ!Tt9cybbUN6x*b1RVf>=nr8e=LRKt&Am7KttP~DM?F&vG2p-}FU}x!0mZE{a z0y+pCnED4ZCH0T#x0AVyBoiq#K2xfzTf#(zh_)9_*VFGC4;NmD5mcTWN)+2T2)>Yq zy=m_og}WZecxk$RY{LG#*D;U19%UCIrnHz#6Cc$r_{%5T7Ti|E-ZdhQeU zec!zF*O&fktS#nM@IZ2G~apy$t%;kLyig^3mVL6kMkbky1 z8j_tAZ=ADwmU{_Xz~&pa=R_51Raw{?xO`VG*j~9AxlV5$IPm712PThpu;R)&3ue`r zb$J!)p&DCRW7vjoU$D8dnVD559~kW{W^*cMEm%^6Rzb2=qRL85x>p*uy4Bk^%2rX$ zF?#ak(awlx;gf-98;X#k!3?vI%pA&zvzHbc-uZg%j{5DJ@Y%KTI2`;hR&B1_ zTv=bnN?GdEvg}FOlSbah#8pPAx5>&*@7mUOu+!_^JXZmQeN-eaDEtz+Nc@ai#Kxhxw(7?33w)iF4OAd_@m(VASU zPsLh+d7rat}dTRi8YyGAhNs4ca*Owf`7*4 zwYY0|iWmdLm

=q+oq7+tRRgr-9Vc(Lh=j6D4m!A>yC8%GnaP7{>EZ zX-pf@FJa{XJP#(u2LqqMU@wxK*gp@RI%Nz)Cil1@MXAUql8E#os&k%ZryhS}tU+!w z>9z16Hz-^mcBo!f4A~8e2ds3 z&cO2VMT!&rgg+8S7IJraDbK`0mQqOhIZ?*T#B+fQ(sxP4LH{J`Bc%*8f;>BtVQ{e! z?6*NAV;&_i^dFY)R`P{8C~r8&YP#5-_90GjzqEF28zgpiOJ6Iw)*QB5DSygpgG{yB zZk5V|mftjmV1|4Q4$mtp%5$Riygfy&4&Qi7>z+NWPTpM_oIu;KH$9OqtH`B%_d#Xi zu`OSI`oVV)B~VecE;QLvrv%j>=h`zIF8faA!5Dkq8bRA2Xw7wp0| zUi26%dOmDSx1!w>qVJ!gTE-uk^z!tVr?-?JVux7E)|Yp^yz9Wh7SEr4Jb@@APd9d1 zMbFnok0Zk7F)CK+=d(hWu^G=!+dgf3VawD*_npb+S1sZ_41SnL1mdRViczLztKEF3 z!Ib}`@_+&{5ft7b#Q~Tk6R%(tfJ=IS(rhouxu=P?orJU2_7X)O=+z1^A9<{4N?-DN zaSYpC5~(>AvQrsrm5OW#xf5s_i8M`jg6vbe806et>4vWU2lEDM1T$!UNMA}z^0FmF zMw(ngB#XBe?a6bT*Doel#v@(hm(K|ANF0XD7}#52DdbEM6XwW6EFlhYf!2`_IsGAr zvGa+ozam?R3$rCC!tFwC2Qrgvan%FD=*%{&x^Eb=P-5)1Ta*D|9a)jKK0^kC+42=> z!JCzHQQ5XNa5v3R4B*o!1RQRh)*&ul)~p~hEY13>QZ8uFw9K*bA{r46zR1YGilP8F_Xw6bMUB{ z4;CDs1S?3Q6;{|NA_2}?dW}b5wRPSHF;xI_I5h~`2B1DD1<8UKP{`$JzJZMTV4ClF zdxo74!5bpjhT)YM_%rYZ7~V(lV3~t%8|1dh1#d&%i4>h}cnJaTJMb8p^betuO{5zL z1o;jlv?E_qKrldh*U40Gw^d^tw}c^n3fsim%$gQ%s(^QIQ^nuJxOFA#N_NcKQNN>p z?Q@HEEZR}PuV+n0)7B=EYY4fL7H*E_2bpux#>%y`<$94cG#jQ+(IETWl3T^N3N(49 zqM~$RF*9J(pS5mb8`suvG}u{wuvtQ5yz5Y0-qhqoEVgMszaCxgnD<;sy;0%TE0$Nz zTTp@f#3sDn1S{EB)9wx~0vMMN3Z%mwvqYr8Lfm}?tb4Hfz}$UC>=eDBxNZiUei_US zx`G_fv*(vKR~vi2)645iYfEd5l`=~}7kXD>N5rI9LaEHfJoi!C%B8pj=uHj9}Wg(wmndeUV#b|UDAV)Y&Z zfRy$@;tUobDOdRinxhwthKBi)BZr3hXG3D%73QCBCPktaP@{Cg$kd|1Jw2_ql-0Ot z$udfp9|N957A(C3;!BBKy7ZDV+im`GmsvHI=OFiW*NVsS4-%vC_eJy zTTzdDBV(;_45D;|S^ACD*6fX>x}8hWbuh2E(~wM`(hKNhXc!NRyo zCB2kHNuPxO&1q73Gmx4u91RKw6Fm!rdXM2r)4zR-YcKF{#=9{dI{n*GhUar#sJ|7x z_M@5s_;x!RR{lV~@kX+K`1#j2yv^Xnee%!~hUbj_!2Ub8Wym^|tUtgMYbt+(`gv9M z6U;IGHQog*HpD^Eq8Ajf5&H`^&w*HC*y=ZLHh3#Ps5e(Xk0d7!`xe>Mv`28RX1x&u zoK5JoyBiRUV%38yvizpm2 z(`yYEB?A6Pd)Dw<1@@8ZPlS>dUZ6=L}CXP~r@~)LaVY#s)J) zo#8U3?Yby7y=LlzEGJec1TR@UoFsD4XG~Jq87{8}EK#Y!!h`-!ywnizg$~0Jm5P{Q zr-HsuJ)Au5ofDNWv)RHg7}T8y=LF!F;r7dI=pdSgO2fvhukr{I zF&schP6Qb_z)6U2Ai|0#Fgpvr1W9T~+DG!)KqOE>;pBorgdm(U5`tM-PLz^82;3`? zE_fROig4+E^3U$76@0Tz-CYxG})-B(dRFjKX-BUq$#7z9)MuHBw*zX$1g|K;fJT9{{6r9$S+^-e2tDf zpZ{-d2kQp+o$Ck7{@t@t{m%Dvu1oj-Cv9}T=l|mPN__^)g8TotAN*om=eoZ%*3NbQ zljHxbonLxRD!=R+o>7(s_E)R}`s#dN=i|=LtG(8ByuVbh^F4H|{?PS4D*I3Gy|k_W f%X4~$E_2;^J#ifP;CI~=<%5iE_!YyhznS + + + + +Created by FontForge 20120731 at Tue Jul 1 20:39:22 2014 + By P.J. Onori +Created by P.J. Onori with FontForge 2.0 (http://fontforge.sf.net) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.ttf b/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.ttf new file mode 100755 index 0000000000000000000000000000000000000000..fab604866cd5e55ef4525ea22e420c411f510b01 GIT binary patch literal 28028 zcmdtKd3;;feJ6U)xmZaM3$bwn2@oW}1>CTclqiYRLQA$5Y6)oBGM7s&UL;16CB=~) zH%=W_6UVZgVeQ0|xQgR(6Hfyvk(0O_J9V>Qn%H&oG)e0>@i>{hWS-onNvmm7eN1S+ zzjH1~P?8h3Z~l59fphM;=bq(ve&@HJt1v}T9Lj@=s?4rmzvGs>e#M?e$-DSAY}wuu zPnxz(DGIB>^~Cf&le3EBXMcd}6Zj5KA3GXEIX;t*;HP5m?7n+mKJ@c`Tz^VYD(~Jm zd1MylPFz2T)UxmH5A7ZOe}4HVio)j=WvpiZ%%sNt@lV$&%8rY;pWcrG(tJLATS5ef5?>;m=`T3U~TdOF!ucQC(+%tJ%mOWkhLq)lj+7BL_yl3W< z|K$8OuAf04Cua{GIr?|bL{U+0Z%`D&^z7l8*&pAf{=TBzgX+qM@uk@--(Pw5FDd=Y zzv;PiF*WcaJFOVej)kLlWmcx_K_#l7Hdl-))s-Jiaq+Wt?>bHS=G)5KZ>d2Pj^cL) zspv_s6cktVJbfGVdn<57wHg$I5=3giAFkhi>*`hfDp#)t<$c^@rlkfMM*)4yKjpoZ zm;e7O&j~k_zvW&)&a7B2n1DOHt25zBxS|PHxb6pE|LkYEcj28n_7e#qH3-ZzD|Xba zuyCr&LatB>-zH{GA;V(qa?!?47iYCXp*YJ<^ZA9f8oR8`&1u?oZB#99!|V;=FIv_H zHB=}yp=sKjTsBRN!=aeIVp3RFXLZmQUKG&EInIE&niKmm!2v$!20ko9;D~#VS11nc$`+=KtG~yf>$N>ebwp;yRE`v zGH}Jv)#<|c{rH;oR1LoSw#IV{&!ba4$LBE(`n=!v1WX7n_@h>+xl&r**uQ0L1!}B7 zt%+QDbF_1>eooBQh?%++pHi_R?rNvaVp0_&7C-Jcx2Da0VHnH(`yji@Q4AK*~y%C}@R$UciWpw&Fz=BN&REs|Hb5 z;$@}9KzIq9aGHV#O5h8E}wr4JV`QcE{(tKyortc-Ac zv8~hc$>PQ3trZG48duddZHX0S*S59PQlWs6zK{7a+O3K5cJSm-tA>$kafivtXzwF&by768I+`}rql(K|3%uZ`sLDML~eis`agzI^b!&%^)q#exy z{uPQ>X;RvWcC-W=e9lS}(GIuYlzx?4YHksgUImQXzoMzdf+Q*$Kg_9fyOSJZs$*<<+E(%oGdnwYpO{(HB(_-7zv zf{W|>&!PC0imz2WsU5X!4}vIr{4C;UXb`h{hi!c4o#Kn{u+t~=S@!wOPZV$8Jb5y& z2B{D?Kb}81xtV=Fdw=ovEV7czOS)@RtV$L75Hy$i0P=${%0+O6L9*X{n_ULtT`Uma zcpe2nR-kN&c4Mx7aJ`5UC-`?oL-n;aHU{{!w7-%2v5+p0DI98!q+H=t!kzY;Lk8jw z9$!4Yk|kTp^6XKUi`{*~_MqmmFZ`|Dqdj=ZUUQlSi+|q{2y_IPLnLaD+1c-X(xDa4 z*gYOQJE*Z**8?vU0$$A%qWMuB6`;a#{Ho zt(sfqBHoMjtCFy>n+Y~b9K*m+LKs3S=}r*hvY}^>Jv{vG+rtlQg~72wVC>ju4rR7% z$sGF3*uqQggM&0jfww#&+H;~s;H}GHHxf>{6Grf~aLOFbL^J-3H)Hl@=HhJ6PkvH7 z8{f2PZf?^i$TM?l@X8ZUUAdwcfOZf$EZYxWC7`sT-KIvruTtPDUw=L zK&%PU2IwJhOkYnG7;3ptY2dV;w43plfJ`Z{ovO3g_gK62-G8vEK~3AYZ{eI3GQtww z@naTIz&YGdTO;7iFb!-NY#O#Y?0Lu^g&BK5+2eYB9kt&Chy zfn`Q4M6*FP82LQSjArinLqVwK=$geu>6<*q=jB~2_&j$6Ca}PZ|3b3InB*GPsR8WC zdaR*a?n&0fd}iig5CvB;D?tY9&>S72HQ@i#6f+u&|KzB3ZAsgz*zsapcJtE*H?CND z(=BR1jTz0wKd7>$x43E@tfF{qbN1lV&EbE1ts7D9GGDu?OG5h7FYwkgf$VxLUl*#P#m;wC zHy9Wj9BCPLIK2U%W3wr4q*}&xM$b{3ll^&h&^+u5hcn=JN7hh-m1 zUgY!Eg_o@Ci6@G-`&Hk0cZbvNW=`vi*luVYA0ZEs-s1)rt%np7R@|$dpbgX{mqGDrvr8pyH$VUJ#p{eOwmGZp&nc8YPIm z*Gqe^tGyMQPwYJa8z?`>2;_3sX zzCdyw-DiScxfm(eg1j!u3zB9pwPDrk6lbXw+0Ifwq8%#>vD54{>7}xcq{~ehO9(P< zALw#-N2Ix$ldJ~$!4UT~G4MeLq#}SSf<4y5q~rirF2v3jJ*|iQU?^1886#}I!lG_d zy_LnY6<*bzuBw=0M&@l~+a$}X0^=JH6Hh1O9908c; zM24g{$zMn|S**+aX1^KBA#1BaN`;`eysqH2ZYzW2g4@MeR3kJH8QJdA7^F_c%u#cc zmXKPcMWmFrIxV;^*H-~nwrliPJmz0iUom!V^aVD&sCQ=N^)>B~OnXf`8B7acfS?sM zmz3BmqjPhm|D_g7CAdXH6XO%~$OS3Oav@MHWMv=`v3~r7K+uWp8xx>F#1a-+V=~Qv zF`Fvw#f$dJO~t?4#4h8)Ub%1#ziJRv9mOb#dp8scdT}K`RcWVwm*fsJ=wJ=-+Y5Wh zGJU7C+glS}pWhtmVI_r!+kTVJ|0Z8Nt2IYPTY8;k8V}vL`9e!*w5``x2K!p@dCP@J zqnH~wX@C(UGlzwx3v(o{l^9}fkQ-uq0ZwKx(D*cab^n>pe(Nic3yZ&MI5y^bY@=#m zChiT)6$*16H3+kob7x;&O`PP)cwb`d*sjCS9UuZw1#tWlj0FyOKb%#EBWezp zhTw;O0^xfl3+sJ9S}43FdcO5a0lN@{qts`ip!YX)1!5)OjlKwvrS4OW{UP*~#rX;) zLrhdQof|3+jUA&&@p;+iP!1Gv*WqPju2dQ^X0J`?3GTQb93RXd05g{0xYX{I58ra< zxsHL3+B2+|0JqcwWX>adoK4B}{xgMZ`yyPBV^*P;I)DpR6~ul(>sW%pJYe>Rqpbslp0X^vu63MFpo-IU6@N$SCoJNeMx8o)D97z!m@tlv(mI$ z_AG!vnmwd~S*c6Nr=`uUyzkPujZ5P;`h{gy@;nS%@0}F40_I7`LvmCU{JmdUsjOGF zD6ZA^jT?rC1_x4ou{Mulf>DEz2bSiv6fL2=39bdS7w9i&4y4JXSQw%|!el_I9Z4Q$ zDG01&A!rFgAP3Afg8NXMc4GO(m%!D$adxC5fK3AAxq__%vqFqG8iev2JRu*qp@Q62 zfsQZ1C?)F0siXs&TJQ_8rz^0}Objx#D+!&*3+C6HBEhQw1xxi?E8e|SfZ(UwmBEXM z-nk+5LH4QfkP#RTmL(%kiReXDqq~HZ*U&u@<+Kk8UVSa)6Kpn4BkiDNptUIDJ=SY@ zkBcBzYMiV{WwxV*=RsldIPBMY8zuXlUxEGF<1E?hVZYXuO{sF?wJ0zat_j%kx*L8!tfj+p%JQRk~3}w^rf?yJY zV*aWYrv`*%%l5>JXW1UopyOI`2*sdC8Wo|OnqPt!t+O9|CrR+?>x$HS#99MhC8K(2 ztxNDSC)1fhPHLFk45>^sQo2`KrV{UaMSyb7V^>v+&%V1B#*MK-)2&Wo$pGuMh#??- z+z~K1Z#9v)+g`idzW#bVq1{gMoUr|qNgVcP>@oPGNQ;2&gN*d=zAY>uP$%G?qB$?& znJS(q+O69ljM647X$7?cVnO&T+z#}dTz3P!v*_0-o^!(wrnZ&|G}6Dq_LPY(g6PNI zDl5^)A=|6O>OzmUsWc9Nn`{cOo`#dH{)|vzg>p(T)qv(28GVPgfc0(R^Y45C`{3jk z>T)^vff3@4BL`@XVqJxtWK=AQ4deCDx>mdFRTV_l$&Uk@0RAA#w-SjGUnp%cc6wng zBttUz3)V#z9g-ypia;Rj1pHGUpea|MCNrcm2%6F;>`Bn~;(lO%I2D0PEi9;hV_O|{aD zG1j=HZ0Bz@2u7Al4yhUFui#VCE=icjV$D@;{Qkf@_DBwYjSE z@S!s+2@6-AIdr(Qs<<)W9Xp22I@sW81Nda{lRBinMQvcmvc4D} zLItj=PwpZ>n%0P559kRR$zm|JUk0@#-)zO#%47#`7_zwdl2=Xt!c9Pe*D}}|AjerQ zSP+{a>434-Yiz}?7I-fQ38W)|0rEo`T{eJzko;$_w15_n{Aa|Ner3bK;auwcn7 zxeVbVCyG*_N#y3{=jP@k*ikeVv6rAH&cn8{Xj_C90qGUeiw7c17z>i|lF2F>$|NGG zFl^?G=caFSZhrNtCbr30Jnv@h&bMy;*x_A!?!5cO^i{?EZD*nOm1baR{Lbv5ag7`~ zoA1lsvs+u;qCND-)US|#M873|N!As}KR)pK63>MEvy5i~s2TlB_7w8{(;Aj&1IcNN zAM~-r$Nn{PC0fHWl|TF5vZ0hKf0u0d-g2pwEq|L_`u^ogj2cV2#AB?2SJ*2o0=ED* zL{5Nvli2|hJ;Dug8es@&;u^Geaw7soNFmp*NZ3jGRS(Qa0oVHAJ**PA7H>2(F}oq$ zOy-CoQ%U@a#>sm~*h2PD$fRlZM11<@b$u;XtI5A**Td^JeEhZzE|+R+?;gEHdq^0b z3Ki820dJ#Sa9chfO08aR_L^Y{2RpcEEkB)iT#W{No=m1waKkbWTZrM=(#$fcZch%=s7o$M7zP?Z2(a; zB$=R);Sl8umil$6&d!xy{U7 zTUQUS8Qxr6ke7R>^aAXYC7e;gu_0d=q+9}5vm3<^{F*cC(ti4K+YnD2cX6hz4P z!uKNNd&!H<2{pmgL?(!72E_9eo zSG~XB4RmEhJ~vdTc1F5Iz6)NG+)&>wj$`oJ3_5Pd}~f^(Nh*@hrj7 z1gjn9B;`XFAPDnS$e(eAGO&FCD06e{GT<^xUOjOsFK*CArCIO>xBjqf3eVHCV)IgC z)Cd(6FN(%!EKBsu49#*U_V2b0(dBldRNYQLU(#_1KMyUGDW*?jv_%{gXX~s6RWmv zu4+v?2YNR>)Xx2Z#@@bq#+n*kRaHjMTE^5$lUwb7HQaAh(-zfgc3OR~RF&doVs1y+ zYOwn~7HDPFBkNgnMPpjER{0JDeIo;&8ne5-(Gd%^RaRHkR(Sm;V`Y`On!E3*XtG(D zN%d5jDt&6Cd~JwZQ#_fJ-TjR0kx*c~A^yrF#gUQwv1DUFM*E(|dMFi}xyUNZGLT0Id4ixx*U!xSYmhON8Q9@Isb_MOI zQfk3JD!$fO=e3)Nzajpi%y{b(9$e{YDJi0EKIaBSdfpp=|29`w<6gMa%?EXb(p|hj z1d45PlmE8(mfL+nS0HtI1^h{XUeyu3f_MXOgizX{x1_`sI)|1btjHi?WVtC_kpmw- zwit{nag?!sX^y-0lUF8{0{=MR_U%(oxug#5u4*_^P~05cHzr zYmrc$uR`El99|uAB#`Sm5{0vh#o}=cSo9X ziN3x>U{y!QDt1I90Tl4u>VbjPC!RT>C)$dwE0VpvN%|ry;iJc6k^JP7G_m9uGYQ5i z42LNMx?n_*M~Dds3jtGw%WxJZM4&fb^Xc-Z&@90ZE#n}xH|H^K?F2PgiU8cPzG*X;t<{~s@Ewc#f%^JAcM5Di|8`8 zt)i0RFNzmsgatb-<1vb}%dhXOu5I)p%B$7pyVM&>MF{e|PB~fa2F@KDSj3l;*s{#GqTM7HF%D=1OirTVkeS`pN&nEGQGf zH<%OJD%}g%OE8$*N;K~M+ek?Ek@QZ=K{797A#g_8M^L@QFL6qlBUVX~c4TH2DRftS z1b-$Ond~tXaYJ&gcXf4ltPN6Z17uhyqG1h+MJQWB&(EN5FpJ-r7h+IAP&slo!ADEf z^Tt`kgNZ7TUv8XYs6w97>53j_Vr6P8kqpd!*b?5bt9S~%0;F7}5P?W(7@-wX9l%d=znfr%CJ4UDvf z0&J@Ey?1+whJ!}P_Nt|w7QO*-LIrHK39dq6`Js5_95n~<#OEk<95W@!_{x=n7RMK2 zd8s`CD?jlZ8z-IvKWGYV0Z@q$6U`BC@J7k43WpDZLn-k5GBQOQAcsyg#4r*Ipio9c zP+$$N7F9%~gOi2PZd0A$HRN;fm=U9+Z&pMvM508voY3C|NIgC}UlXe^X}0PW9j;EB zW;EY2{`hNb&z+~i*UqTH*B;-s)r8xfu8tMeHqBsd#}mbSPv42dG;f?)T7UHI6#fpc zOW2-;t-#I^I0!>aiG{+{EbLCg0>xx-lp4&R%$|PWU@&Owy#L-OvL|mAf~roRAr4^Y z_z~mXO}wZx+En9mn8_apw4m8}L#<#dTp$Ta(Oj@2*=@;o21_yny8b=XdlV?<*`^&veDfVWp&KJeGyLt_=znKkl`P~Kc#4@ z499g_ddY_YQ55{%%4XPZk^pu>Y4Mg>6C}e||^>sa*Z2KnZ52N|HnG0$F z`G&|dLRS0Ictm~a3n*_t;UX(CV)#q#-_~f>Ap_1oY%e$hAj8a(^$`M0)JOvzCB)@7lNe+IIY1- zo=lq;gL3r412BA%8V3g(5H3WXE?B&%CiB@X!h+g;(Ew(SARSWTIs%W~6~~^P9c+)^ z^_Yjx8wT4Ah*(CPG7k;>8HMV^Nv9KvU;N;6)priIw-4S~{oKL04BsKRE&4jp z09c=gfI(1c!91En)k2qA3?+ukYH6&bZ%DawSqSkJ5R`@I5i5=O1kY9(I9#+r45iUP zB*og3@Clru@mxKxR$w12o=IT3g<2?Bpk~bJyY$?eRc&v4^tnq<^7&P3p1b5b@#LlF zKKcgmhVVezd;C~u8|f(wVMmD+h#?X>0T}j1$-^FId&mw4vM2uWBWPghg3?lZ0&fCn z&neo2W=)zNoR=wsdFjG6WPs_B;xzpA#sBsDdd}d?wo2 zxy~oXeDy!@moVoT`iN2=iZp{$KdYD@q7d+772=l>3u#7Jq#sw@4>KUdK*s*)*};K< zD=qs*TPD`sYBt+z%vTy%Ah5Hscqz^j$umjo(RKH4{n;~HnGa{`Ag*0*8Qs@1xo!{K z>rTr*H*RZ0%vka7lBW~Nr0s*K`pnO^GN+^oa?hy3My}H&3Nk`qUpOUBgK5&b3{E6+ z1b$sN1C6!8lia9u5RHvA)p}i3A|8Yh5rQ&ArxZ2i&@$Pmg~)GS)XhrwQ{d@{8!^!554>LAvO5K>rXuKdhv6bW;n7<)3zPK z9EB}PoDri~XFAj55uweCwy3afX9&4U5x#ErIu1m|-LNbCo{*2!V9DHo01S3noRFa4 zmL)qd+1Y()yBa6JRO!b-=tdf_B0aA;%39@dFt(?zrud^7*7o2FuRZ?ZY33~M`@4&2 zoCQ&fM_Bv5JKe87^!RJrnDehLUF^7Ty>8dJ`m~_0!iPw9on>ct#GZDUqb^B=WcclE zLQ5i36wFmZR>(p~#lDuOb@Vej1qc+vdV-@T(1@19Uc_KX*q1^@T3xM+_Gpm*MLTjc z2(jGH%jq^$TTovd-6P$T4r}T*LK2IFu@GcS@Ed6>R7H$mjpV0v3QWbukrt99M3;=z zIfCS4%8*R`;85Eh$RNqC)}hGI=xfEdUIQvYJY~w}rcL+JVc)@h;ik<^eW%ABf9X5yRtP?g%n=#HJ^ukG6EmyxUY=0CxJ|y&w}&`CR3b!1<_R2-3!m}wu(y%k+T+m zZY>n7tj>zrP}_RkjV>F=*m{c3SoFD4e1=87T0&n67J{Z=6Q)_163G85zB0H_ z(Au8}+P-+khxyz%%_9z{L=g$8nz%U7zo^<6@lATSdmFMx z=dG$^7oYz?@vE($YK=UsHGF;dO)NW7{HKxJpJ>gdK2|UKk!QvFLEoBmTqB7Jhkz08 z;EiX7I1r9d8V5om&}x$?k_S_^Uem`#Y=r0kg^X z3srSmOE<*@&%MXpYait~Q35z~@=dZ|1J0yBSuS+P9D>(@7K@?U4HT;ads=450zws` zlRP+siGytb_CG(cX0WrP*tznTr1iQwGKO|lpKDWheV}UV-mO)E z`u?^Qh11sQ;s<08&r4-__E|l6m~NEfcoSQzI+C`&Rjc}J%>y@!_+c9fCBocXAf``O z((HmO!?LTgy-zes*t$ul2_w{1@^hTkF~i86N+8%3NGkltgNSp$Vf?4QZ1NQfwcWwz zoJS=im`4^#ef% z$Fjp-9N{ieN`jAgn#Q)oYbum#!N+`Vd!;zz=!zSB)!2%>C5-TE3Nu5Bt$3ET|L`M) zXNrIO?CUI2`11W@$1sSG{IK|=v(GZmGg|S@*YE$bb_|;Hk{nP0nn*DTz};Yj-$Q{( zz+HFTK<#&Pvt}$20%^zDIukuy*M=p+L9mCer!h%P-&e-=Dcd zd-&&%Ja*|rBpHlgj|u+pQLG^Fgs0ZF-fP0 zO@ev6y&&wQSBe*fbS*A;q+Og71>FE3$v#kx^PGr*cUK6y0jdBVRWixKEt3ur`eK8^ zZLsMlAoyCWsW{XWi*bq`Tz|LI_4ZRB*-*~!M`06>G@)GEH8S_T(q2FxHq1xZ-*MKR z+Dd|UN{^ZLE``^G0$t{$BoUA^*&jm(}czG*v{jdvpQ*XlUZ*!1?F zZ|g~=dbWN0t)|8!3%Btt_g#2mV@s1UYkEa`}7TW_;u$D?h#yiIX# zP2f=Z$+;+Ci{KMi885SW&_!riG61xao5WJRr(K1GuPAc@k!@df< z3%=;Jt5;-`y)a9{Dk)=z;fpSFUJ1>r6c=1l4NAn|+VawM=|20g5UYPIez{8|#h;6i zC25S&gR~dEU0y?0N4N?VZVr2W9e@7{jA2)adP41?rJgqjDNB!`AOM`^3=%+y;A7fL%L+^HAY0{O1?gW7mBC+sS zg;MolS0cwW+7k1NNA#tF?!UXJZYP>`?JAVE^eRRW-GGoGzksjj8MI7=*yAdty{o?6`3 z+}LcNSuA^;WQ5+|)84wapH#SqzEiC_i_dx- zjS+`+ZbKP<$(S&knbTN=Jsm2i;1j}%F5-)EDifq!+RugY{F<|e4p2bM$0=euDO_O5 zUY1OQ1=9XaVGS2k!Z^$YvIkILEwt;w&k1)u2#!Yf1CmC_a7MOz8LYwfET&k2()xj4 z5=L7tc&c$;P_VkiJ_u1FDHR+_y#E5?T72IV*dGgPN!2A0hgj9vF$yy;*F&)9Dj_9? zF(>TxNK2r`h0P-Ps8n!ivxM}6<&-y;<;mYghm~Kn@=1{te=HN>_rXc)Vk1s5{}cf@ zGA)oMOnNY!AB6u)JW|pdk|;Z&6@f?g#G)-t4RtzCq4VYRZU-o97>h_T4w({DhDe6_ zrx5eBEUma;E$}J)6yKsBF{%Pa3qokUP$7RY%2)6j6?`@8ZYb@VMptxJ9x2AC(?r0D z-dRC!odBFd4PGZ10{|y7UErMqh!>&}EQeJ&+(-^8dK4Ji1iVaXO0NhL$H6hxHaHA#NfZiL> z0@~PuBecS%LHj)lr5vv)0Zo9xI!q@FGDCDoBSNoIAmYF_4-Y>~azSfk>LVYSQkx@n zHEVY6TvJn58|vr`*3ukF2(GC8qc_ghS~ZjFu20P^kE00*-yN+t;&?1_ zAL@M@ukB`etEERI*cM*gv-V3slWmsB; z*hOEK8nYN!M5Px6s4QY&04kWm!Y=nVt96?jFEJqLh)Ba?`@hECw1N}Yp?$x*s-k4u z6PkN8U5%Hfkq#gA>FyeK{EaWB9{u`P9!q^OcWF8`x_jrw^b5KcbkErC-DCF@FAnYO z>Dl?qlKvxLr;?wGBIPU>8ta5DgI>qxO$ZW7=0lSEVL>Kafuc(iJQ{RN7ADmv_I30Y z-)_h?1h8-1PZVDgasV_c+(bmm88%cvxwm2AvEJ{#OL$FRY15;&?SiL5a(5$gS(n{$yiNQiv|mJiq2XmbB6LtV%ZnFb z>e8>l6tQsyO~HCE`Z%MYC3qJ>TO<6Ou-m=2pHm1lh?%FL47`gAx(K)w!rD>^;rFx{ z_bvK84O?!7-}5`fZ*JRQcd04CA_RuK_IPd^Vor1)=su$*hNlmJHLdVl)RFQ1-KbT< znX)lb3|hy(c8qiw_kD~_gd31|_P38LE#Gy(YM<(?_)+Q($BO@@R07lRS@wQUc^A=0St)(r{b2RV>%P}q%j>+K{O@Y# zy~au9*WJSyMVX%7unzF6{JHXc`FO$4m(BOR>Xko3d7L#{_8gVH-)FCF>;L36jbRzA z%hwZm{o{l8$){wMTa^>algc-hpTqZfGn-lxVE@EzyqRbDX0Gx3_$T>`U}Med z4)vH?P=9H#8Fm>SFnrPQKMn61W5yxl9^=!-ADV)uoav`#pE+m#l=)}o%NCQR#?oOq zVVSeMX!*Y7rqtF@l3^cDs7b=m7|sWD<7`BVym{@Y&&Rs z#&)sFR5elcVAa!A->UitdyD;;{fzwu`w#6!N7}L3vDfi2$1{$-f2db8eJy$^Z|K7%jf zyV-Zx_oT1jd)MFWf3n6`^JL8%wQaR4YA0$xTKmP?AJi7>R@CjU`)b|y>)xunTyLvy zsb5jQqh70jp#JIlUo|KVS#Zz?8_qWr19br{@QJ`nfxm5RZd~1XTjQr1Uv2zlQ*+a? zrf&v^f+vD!gD(ev82nYJF?3t#Oz2yopElPu4>wOVpKAVU^Sj}i@agcY;h(nHTQ;`L zwmjYPot7)D$=3T?pKg6KVu-AdJQ?}xNHIDTor<1_J|F#WZ8dG{+h*HdZKuFn;+sEJ z_9GI3K3x2g4>MhPx5z87i~Y$W9UfL5*7FRWr~j(wDGKBN)$^*-!Ups_PD8RIdfuqm z*=O`T-k!r=g*3$sBoz}z$vlGv;=ky54r|8$t>;x`RQZ*jHz?KY4n1#F8rc1M-lX{0 z7nKp^Fy8h&sT{?xrUaEK)H#6sar_>|%!4>ja|q=}MS2+T z2Ae@y9QAvVwxPyR{LLx@uvPUad-b}M%DUak5tMeLg&EX?GCp#6X7cEa7M%J}aBKI* z?%4w(UQ9batSpXD>?kQfc>*z1;_Aj-rj5 zlxfismg1)ALkE!@&`T&)4xsD+(%&}n0gQg9m>13SZUK=#lu>z~(gnL)7iQUud=d>U z8`wZ_=fR@~j@~_^^#uoleO;NZcyAwSUEiFtSW!`Sp^L)+#sM*M>ZDu$261!d@R0+D z4hH+W@rUa}fanZH*R_0Nhh}FEc9mu)u~E7D5XO0<&reZ^Q^1Tfl^O6xCll;d7Q8X8 zf>kPOm34s524K!j%*Lufn;guEXr*fAW*+8cKG=b3SS_n#^$Y>PA9Iw!Sf-uimhgA*f1Mm zYuP%so^4>G>?XDmFD$;9-NH7rEo>{>#>Uuowu9|tyVwU{IODvpM#M>`C?% z`!xFudz$?R_F48h_6++Yc9wmfJUnc=!^5d1n*1oz7+3E^S%u4%ksW{ z-Z#nnrg+~p@6&kS4DZ{^$5T9>=J5=VXL-Dz$0vDwipQsUT;uT> z9^cCoy*$weuQE?0cp}LYDV|94M207_Jkie+lRPoS6Vp7Q@x%;I?B&T`p6uhvI8P>c zGRc!E1YPlDh9|Q;+0T=cJUPXa(>$s1f@<6PbJ`~=BX4XgXW~4Q;F%=PqgQ9Fd}@kMP4g*@PtEYDy?nZtPxtZZ zIG;}N=_H>{@#!?5&hY6hpYG?=lYDxLPfzn{jZe?;>AhU*w`~4l|1WJN*uYz)E%B3gjC&tIe>+`I0d_0_2w&rHW$Gh@sEVwS1 zH?&S-K*o`+xx6tvoHvDsG5qm7o9N0LVquIcsGT!T4F~Ct>^xsFl2<0y<<*W5N=JgH zf~U~(xn5)IscpH5t@V>*@|#un=G|;W9iN26)56 zlXFPd2MoSSKc1O1cJf5ZDb?O3z_inc)p6R#&A`I ztFF8Q%{T=}f`Gs@hMl*MOaxC&1oL(Ptt;=0ZQ7ALXVBJ;x8$p4!Y8`&uGpq+xlP+; zVSNbYZc$zxJEu5CcIM7G93y!)Ih=QN5`qG4htJvQrwTuL=EF*;ty^>F2x|eX;Zs;# z>b4^k#$%;?y}VD40PpGUIA*c|aRt$vF2nIrF6a%5O4FjRHJr-Oc@Vq02`8y|qBUpq9 zTC_=|`F298&RD*qGv9&j5(B1g07~6(zl0~VVWLyNwFdB|E8n%a2F#a_b>x}1S3tSD z94gCi^~8cHG0tApVe78nuAl-p92S);zOM>eyLKp?J=ep$m`NYzje*|qkqKb!WVS0G zk9GT3bmbGjt12*T8r73n3dPqN><(_Aoe2=$bn4WG@CHzV9OyOZ9ky$NAyN|kr$9n{ zz<&ITDtYTj=gg_@a4@*y6xvEJ-41rkHu46viCV$@1a0Qk+j3vwK{Z(a6}%9?P=mY~HN@&3D2JDSMB;$3hqQyx(+$sivU$77&VM~1hOELt5AbK}O zbQpwJ05n-qoVQ^227~Lv8>ll{t$qPAnt%>bWk;?%xB^U%Mywa2u_ch3T5)v~ZY{D^ zxlq?5*F;!f8H}+jKcJ6bq_i{>#CNX+Txlr>W8q*oL2W&#?uzm5bDhkCjkjX47^}Hd zymGNv)Gj@`tjPYLas1& zMK?By9OD`g3lQiEz|xCYmQXO-Y| zQ;g6tKMJsJjGb4MHOOp2hEe9`*m)*OZb3$rY^FNHxV44qP-ZLDq0Ba_LzywEGla}` zszaF_REIJ3CWBKf2?R|71YVQ|0s(nD@ zsOp`ueE(wAyXZnxy<6m{>OCSyRS(AU1B+D;(S@iwD{@rzgCa*&568X&|7J-t8t%+n zX7Xyw))T~Px)cc5g)s;q?2{nMQly?erx=GJFm%Y&vMl`uxQA7g=s8tcd#;5&vJJxG tBe`>`w)R|vu3oY{2>a6NN2Vb$p$g>T@pFo;#)kMsZl literal 0 HcmV?d00001 diff --git a/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.woff b/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.woff new file mode 100755 index 0000000000000000000000000000000000000000..f9309988aeab3868040d3b322658902098eba27f GIT binary patch literal 14984 zcmZ8|b8seK(C!=Cwr#($lZ~BhY}>Y-jcwc5*vZBlYh&9^ZhqhW{ZvpRobEY2 zRim2jc2|&)0Du6#g(m`l^xtUf0|3Fv_;2t37YPYfIRF6U=Qof04SefskYWWDCf0Ax zvBgA?Sg zQ{3X4{N{ANb;56uL&kuESlGIFd~-hEx-kF%7M7U{z_qbA{?BgvJGPPkQ1m-q%+}E3 zdtHw2HU7t!7$h5R$XB`1U|?VZ2x4oEo(?{~<9cW^U`%1|L<`O49o%ya3Cchk?TQjvHN{6At8vTKtqH+gT24Lz@);yzA(}YXmPMtu?=J) zB`AsehXP=+al-fk06b49&+lmeAMwbpQMYtnkU%E5*g+%ehk}td81f)!!euyQg~T*2 z)@9npKco9a9KNs1`!r1D7wjizEmb+j<)@`LL%3o_S^DOxFhSl--hj14 zM#H5aHC`i!yXJ}d7a=RP@L93co8&-xe2dITtXa!y%MBkDB~oaSX8=|B+}p%5@uonM zn_)dskE5dgxwy$B7UDtO_s#N{dQ@IiYRc?**2_dj%d{C+ob@a*k&~f+QCmvu@MvPv zXAzzv=m(mV@f35IWRg%#BWNS#Yb*+XqhW64orn;jVCARAp6(CT+dJl6*AU;? zM*P*yjc8Zknkp&+s)x#G((ur2&&kDr+QHf9@3~dEGc~r>L7*Gzy1Zi26w8WWema4O9nUHF1Ay`VkG|KN;jIkW!y|Iqm z_{%A18!12g;hLL=>v$cmr4i55J7qcYXU=B~yAkp<@s~C6tv|V{8@vThN7>Ar*+kUT zG#R!Mo!W$4Nb=yBdJDs4I&6_7L__a`awb5B)C3Ey=!p>9V1OES1_-UBB15l>gAY6! zgAcgD1lD&~n=am~Xzs0?{DhP>B#)UnBu6*&eKAo@JpMbD(YyVmvxqj z&@&kK=UwrH$rMA@KCPr0_vdj`DwkaL#P-jJHm=bJ?i!1 z8}!q?ktnS3m!tlo1#^A;Kj@_YSVeWK>j|c&ToS7G_GF@PG48OmO z9f5EK30J^t+iqJy*#ApP50`b1Itps9p(Y}?<(r0xM8Llb@Vv_bC)p7#QQo3mf&A%)o+*0URgNCG za4$QHzx$SKgZ`gRt#R0@*1!twSlSHhsoh;QsLMm8r|!LTG;ZrmyWdoHUi$My zm|}07P^J|LaHp^NgRiGf&NR(l5NXAon_%#8@W<{J!y{jdzW4$&DU}1qKxKQX)8XSL z?2mV_=`AIG5HC-7@$7A6{NO&-ydr#n74Uj&pF-Z$8y{E$zC4yusOM~M_{>Se`eA&?^+`>z6+^^e z-9zRTW5i&l^d`h>3TNz)Nke3o@P4#IaDYO_;5OYM^K&LQe2?L@Z-9NqAh8)@a0oa2 zBgZE0*v2lzCWIB9Dg+PnN60WgJt9X9;>y;|Kz%P)#Ht|n&;k+1CZVGLZfL=$4YG(l)XI zh)7x3yd;LHCXIWu%}triolkzfz}&Mv;H7!jBuw@gw*s$C$eu=Qa`1sc z5B}ui$H!Ce4T7GYUs-(D)QtlbRq-=L`#jXs?`*z*GJpGBAOxgH)eXYY$Hg~AG4DOq z=I=cl`sYCiMJzXE)U-~?69#ZqtZ&+AQf<3#MTmlm%g{%Umm_j2vh91ay zqv1Eg^xKZrziV{;&zZQAcXh9BJ$2;6V~=dAB!U$EAp{B=FqE%)N^YkP%oiRBdy5yc}^m({p@zFIc>%w~m)m9mf}!-OfW5B#m6e+P`6X=P7dmh0oT$%qeiyr_JA?e>=;4&-SO=&B8d&53>ph7P{!2UjA~-<}+y zPd{`k0wz%CSu^`360$||g)I7cO(uA+j+wedG2^l`$+y$zR;9Uh)P|Z7YDCGkDr?Emz*2pk z=&{N3d}iyDCb5)=dbZCriD^F425+7nvY$^RexMM&Y@~fu_8dox`Rv=J+(Qc9 zWn-qPasT@eA02E~FvN~G5E{6FE|YOYXW<6Lr~;=-HsGPY*-BMa)A~nN0YuSZvNR`; z?3GZSJ9gTT=B1hQ>?q8Z$4Lc+-+cJDeA2{i2Y;$GDd|}~D%QeStOPVz3q!BG*3_3< zsN9j}+#54rC}E;sx!5Odt+_wQl@-R;EOL%rm7PhG84}(HzEmEj=aMrK zIbG|+mgHB(oqX}A(s99tu1a)pigk_tAoUw~m?aQ&b3GAeI>XD0@EuIa$5l*WS1n*g zVJzBC98rNH+I+s$#v@W|d9@)RcYCycT4=Se+q`R8J-~u{;9-d3WS5+P6N)5m6Yiaf zW5r-x?=Ll_GwMmLqv7bF{L`WyIobWu>Q~t8YF*XhO1GVnn(*7@JyIqu1`U@KGOlS7 zDkIuCSkaEPKx|W0eg3B=i?9iL1FUT5wishps-be9I&>pL2hh8|-SBPq^WaW#5tOE~ zT}eCEtSL~gqcqjWVd7I9gOLIKbVX?4W{OO%%C0HvcP#h>_@M-fc}T%}R9KJL<`U9V zXu1u!HS7X0Ez~@YB)L|YW@u9W5-|tHX@2Vd^Q|Yoj6j=D&m1~FnIk%im7$;J?kgN=T59<}6@^cfW2XSeDIy;+ z;ETOlaWdwo5OPoV_ct=W{O6{#XMgMJ$9oeE-~m`CjpUZsw{hJ#0gvO&c?Cy}%w9Ms zF1qLs5n#X6OVn!u32_b_qY`#EKw4CB&te~7XZY(jWdCXUQ92kuUn~8)qF)SI2<%X% z$*37c99~#|tO)1lveW3!TBbb0&BE?sJ2VN2b`;e?d02KJA-GD}T=1K%plNHtYUYXp zgJD%O29qwCKm_~M0K>`K8^SP{D*2gCTZu`SM9S}-Ykw9zDoswD2oi?2TS?0j|YT&|8hjXaQoPL@9w`)i%-M<8&28g z`*F!&y{zlqjf@rLrt~FRSN5BK<&28)W4m>{vp08~u*1zMt6=`$Tiv_$EYw^6mW-W< zt8zy&d5h9t;u3Jj2lY=`hj8Cq$z7Jwz83FVg8EUT_;y_|+qcUF=C!0ITJ*U22Lx;V! zcKoPS=n8#~`Z=P6J*6*B$?-V%RjyUCCvVVwdl4E(WA=YtevNLvY$%)5Bc}Fw#;j-I z0#n6dHjW;Da&pE??)2+d3EbXdopfMeK@6A7^s%KeI88UNE8A_UQz9pRg$VLmUKJVl z4I&pPU<9*3OS$nt9-xj5K$8UbcV(lbl*jMiig1b^fo^TkNqIjEk~>Q^*t@Y56IUj>ezm7Kz-yTs!n(QG%R6u)`W@o3~fE4rr$BH|lu!66Zt>E+mol2P_*O ziCJ0f=UY}ApdzPxn7#+JwBo&4_`u(lc$Y5=bBVwn<&r;>yAaRJ-31VEoTj>*61yyd zp3YVTLPv?QW5862ulNZ1OgO37-b6gtqu(;CiQAmQ# zCr+Ycyg+WEcZ!?X&fSUptp-8 zOKi8O!M8Q-*Qu1ps0AggluG*V^1Nk{%4)ki%nw(VY+snRW|#=(2QwJB9_$3%HZg&v zGierEtLuJ=$|~f4f4fwK5=?TPAjUyj8Yew=i=kkkgavOh6g$X3)xPOz)zymuI+`8M zw>dd|>IZAe!R{&|(y{JJk1V~blgfVPyc@hkWl%sl(2&%1_ zBayVylj>~>f=ABwi~c<+Iw4?r-Y>*Ha5S^04!G0F`%{@_*=~3GPH#N7wy(VW#9K~% z^A}g?O}_Q?lKt*@WTk_H-hSSv3-$^pR130pW(KZ(yEogRXYxqJ=3(mI^u9}QZvQ-a z((-M|R_NJHj9Leb)GgW74j^HIe+xHZ9kE0~@bpOQ{p$rbO7MWSD}JS|^sjCkYlGuC zUORP_Sk^=&Xl>}jo)cc3(U8>A$EKMhU3Op5&q?!5bIRWKQy#{mHJe~z zpD_@@wKexPN7*mrUJtXFETM6Et`^w$d}C!Oti(ItQxZ<}ac+wqpcwP31>V3Xy^R=>z5USMBZKK+o&=70h3Nk7J|rhq`+&2=kGz zbKt(1>sMjxt*%JtH0X1QUjjrO+!WGqJ~>^oI7Jo_J)Kc&*z0~air!w9jp!g4?wfgq zJL+up-MtWP-#IVzI~_ZIvZ7?AAS3Z;mPEnwP_cT! z*JJkw8oBTf-J3$s=O1WSr-_ar>?Lq(5SfWB(V-~fojAhaKW3_-Gv)6Cs%N6kHOpSA zcS_*;`P_me1{t2on+Vr1a$ReDFnK`uz3Z3nG7l^pUjIFTxC`QjIs zw*4v<4CwC+ww4{v+O69!bR4?vCk|s{UsX-Jfap8;>_AXh$l|f<;E74Cz!jC7G9IXy zRd53A1wnR`fLa1lq+bZjJc+3|#A70PRV!DqsMBI+{Y`^Fjxpas$8>UHzBCi7^C*i6 zK(hW0jN5kPJk|E<^L0~z;qgZas_$AoR&%@#wjhOvWDm=21DL3NucshN z&4&0NC>nxBdAUC#X!+LbzQ^kjjbhE1k1OVX7~$`<-c{$9+pA7>tr~|B)r7k3PQii)1bP3cLR~PA43g zv4&593)87tEg~Q62W|9|3QnF4m?e!IAcZS5Ibl^1YcsARB`ADY4@045znu~7a01Rh z>+l$JuFC|4z7hK3+kCD|DCv!`W2+C<_BhK-N=Y> zl~TeiuMqwCt^g2?J(W(R_x%hzZ2vT01(hBOkf{W6GNbOatvp{|VWfZ@Gaj%s85B1e z{1-eVWEKKhhEWhGjoh&iS!ze1fT3o7ow#1s4uhlLS<=;VminN4iuf0PSxB_tM4{Q*zUBpS#fqtC8M||{+PW- z5(wRsj(WEBgf#w`o)_kNV2gkk)eH-#tUQ@!r1^IZh&ZD0`?tbafwU1|CVhznf zNcNSz+~+>zhi)M#9b%<-D2l7HP?UKitR+ZD(RSuH;DtL1{iZh<2ucun!sawL z`=q-fJdKD;G+Bv51liqQ+tU(A>7MJhhOnA&5qu5Rl=-K7=a^Bc5AfVym}bjN8}a31 zSC+FQ2;YpbwsQh&KyheTK+B>WMu-W!SdTKbq+HdKtis?NxkRxZ$qSeOCGaBhz|Z(DEp*18 z1VY0=kluAfiGjwwj;QdjMMGCGU*OjKSx<7Ei}Qj)i@i@!ss5pK%B8wKW43@}FZc$1 z-YoNXL5^b2WSlRy4ve@Z5jq~L&dXc<&fA`H7{ix;`+e}9bh&Hz9biU!LH$`ro>n{E z60{dR1cz+zB{R$pgoATCvTD1<7#BtK@y^5If#X$}l~ytQCQx-!#mp8tbkW2!!BzcyD)40=2|*Yu0mzK2QhCp1h#(R@$2;3wHfiXgEyLjy>&XZ{&M zX|0LbwAC69Uagm>U>z2#~Po-F%98OE1a8pWC?$^=_E$3P3gIXP#XRT!S%HmE3Nof?Q8}oXNel$6zZ6o5zeox?V*DP z#;gc)w7}{?5S6x8>d);zSK@Bkb2cjyb4fpGEQY8yvG{d=<)f#aeV&c7cz}dINU$Mi z(%?!S-H5nn;V;BHL`q}2RFUQG#`yzUbSbPC|xe%Okxc%);L zG_IfQ50^C{^A+S3h12axEIV`>eqL^5>t|45rId@hnBdprP!y7Z)cQ%p(8ARJ5fkIp zsXBB>UB(p=2!Bb&w+Ydbzv(Zoq=hleRCOX?9E-CqQnFv*KyBvL5g10fl#6st3l1r^ z{nu}0VD+#h3EPFLP)&G6MVtXL zojBMIJEED*owWecK9Axcvs^)EyxTG6kCj#khg~RI92J@%q-I~YswpGSNItHCSVz-Z z$aI%XJe@qt>YU7K`DFEY%(uxUQNk=Y1!MdKB!^j3lDhl& zB*r^qUR%{ANk;qd1q6@ttEMdwk?leq$2=`&Sl6|!Y!1R}KfWg7%;x6J6}JEmGNXFm zg|_y^m62>BRdyx`Y%_8b#P`(XCq2~>tsGTcLL!`UA*V>h`1J*&%T zdIHFYXJMi^OA7M~hfB<*ZueY+JM&>+Qfs#=kiLtfx0Ft)66%I_u?evJL21EhB1K~o z`y+e<;GfX>bBQsII2~e7232`QBzVq9t<1BI9gB&3v^Ec(tsL>=LHPD(3RZhi>+eHu zd|8z;=K=UNDEvmBsN1(=_6jNRl;dDjM9kO}*MC(c^F3lY{V&6y`f`AQZw?~-MqNy@ zTjAUYNJv+3iVw0y+J$1+cV)GLRf00|eV_EtDGG}ZM`MgKy1E3@Y68%4IWb*yvmw;1 zW4+u|$L@h*3@+;&b&FewrGx#rG#a-Y6k`B#0lUWXJ{=|geA4hq+^u1speQWAISOkxN6G2HT#(@9Tx^dB9XN_J?3OOn|~ zl$aAWj7%vg4nFC>fH5@o+O&Bq=Yw0FizVKxE{rDu<>BtzXAf=xem*|A%c3k`_IB1; zS?QAC^M3G%gl?zt#n9;@+H;`p^q*0YcXU&pIoTNQ@}1(qL22#*r= zZZi_}Yy%6t5zSkDn-$(McjvFXR9jx!dN;Or+L1<0IbO;R%_-O(w+5pxh#!$=qJ4Y4 zYD|XROqif~U`MF-?cxEZyv;j173tj z-YY(e%y5_KiS|+MCa32c^uh!YtRyu#U+7JX-2>9+vtNsXrX)PoX~9gbOv0o7fgfj} zB`?g8I*)BLm-MV-8F|9RS6zfd%mWs5oU49T_0Hc?R!?L211om!o0F5?OCs*R=6-{c#%b^7GQ}uK~jPH z!qWw1S0j(t4IW+yW|v#OYAN)jCMFo4AluBz$FX=j+Sk*9N}jv6sek`8*blveRYyK6 z@$$QlJR0o@v$S+f-zsLw0nh#kUV&fD{$c1Ky*FirKmqzg+)FWg)*qYr#!&xh)r5FM zyIhdtLDGe=z-F!B!f`gKQ;5@DmkA~JFJ)}&q2vWU*3SVpi6R6uxf)tZkEGzFa5#xh zgxWZZW?URJ?Z)bcPP-?uZsE@O`(e|((Jc)+yo;i4MIL;)hlm(2w741^jymCajG}`Y z0+9`yJ4PswEoFzGwoK&Bt{R)>WKNgeyhyZZrCWq%%VuYWOSZTCmc7B@AINXaIYw>g zD(_7~W$3#FFPFybE@REcF<7d=>Bl!Qs|)m~SLEeCXQD;JBti`=eSRQFLEkCdcI{wy zZh^j@{zDOlr}L}zgS3@RiQBzf2Jwro|}z zp(8`DShFcww4*$ph=`Zv&Qf;2lWqEvw#uf03PUx5*6Zt_ixy%t9Lsse#_!)n3$--l zOf$;2nUJKM8%rIVj%qU1>XT_ym2MR4aaD{P*8oOSZgIqcWfWlkoR%D~ll0=66q}CTgR^m^OW6AzkH7eH)iozB+LoEQPHk( z#`+MS)QEj`X~>v7ZPYe^*p)Xt3}Ja0T^Df?O^X*F|EApS<~55@Q05SkK0sF+UD=#y zt7#A&M)vf*n^sI0F~cOr_VJvOH0Xd?%4c zS9%8jMQZ#au03wIpvh_4m~jGGx}6aI{d!htmWrf+Ec501JY=~N`(k@SGWn!aRsfxN){B8UN2djrCZY-c;VfAmwKt~0mYbZs}* zN)bzhWb*t}1j2|hWp6O^-@hIy=snZ+vUl(7haLy(cRSqP)j6yC>k9j)-0U_2f`oC* zDq6$j2-(gxSw{;!Dp96XDiCcn<=s}RfXP?}T|Y2spwLwsB6ETb1}TfF=R{7Hzpnh5 zA8mde1`9$mIOIAp6)$HGzWUmv@fqHkz82Ew-Q~St6-GJ%T zoE#?-c3l0~iaA9*ZHhlS4{FA<9Xf40OlkBmvD;}@=7o63Ay)&<*d*Y$1s;!ljpE;>z#T%*x>L7ZnjI45Ij{?bC*!?k!+qG ztdZ3sm+s_sl6t;4RC2XWn51!HZA6K~SFd{_-)wmP_l?z2qE~E~<2OIQ+O+`I`?nv4 zTY=XT@qB)6R50(?106eq%h-+tvkEe1h`*@lmM&+x3DEC^osEhDdqcgXu%ke2MH&Xk z1C-O3ZCc_QBqYIvgg?eabiv}wJFj##c2D8mmh`lixXcu@YxCQrG8!B!t|Fs3VzCQ; z9hr_t$>&PsMb)7~T9Gy2%f@h*+#5)SQ1_;4J^h9y10)bshZ z;l2nhm_6Q$h;b}ZWEkFj``_4Ccc@<0bZ^yIU;nEXlUv%4ty-&3ERH>Fs*hBk2V4(@zX=>s`_S;> znv9FMT_}=x6fgK5Eocs51k=oLfx-1*kl`Xt-`Wy>}^8>`FDC3BHmx0tiP7SUAm<*Y2o55|>ORCS?h9s0JBXbw;#Cph$cb&794ji= z+q>GiW^0_In6F@|`Go$PG?<~CdAy08(5Tw{%|4#eF}0z$P|{heEvSj_fb)BSxH5<| z05&!eJ_hd`J6pRTn3-`De*kX~6ob6;5$76=(raIQ zLf|D#m~aFvX;k~)4ngj9jDkYEH>=9Bl0Y4lFbo2hwZ;8SM5yle*pjPB#+xSFQmlZS zx-6>M44W~rAali^78Y#mRKbxFx=eMiUEa9z(ucTGd4XT}DvL>5sH(2)4?_+6KO;-8 zrn@NfBWJqrmF0aeV)74j{RNieoN=x1WWDtZBl&cYz_p4>6*bDFG3D`jit{?pN}=Kb zA$HRnUz77!U1Y__9o>Mc9eAhu-xJAe)|vDDd>|D0$V1~)51#MF`!ucYiH0PDBh7hd zP@~9L9U6_>0ITN)i|*;n^J#Cuv4^nl9;%&+iqY3>S?5D)G#pDe#$!hX0bHuh9I~vq zA2D4T@VATH2!##Rj~ya`D*lSE^NQsk@^8~~tHFwqGoQhqMQ94Y#*!-iK3j^ml#r&i zOqazq3pA5ARb?ZISzwF}DezJS|A=-F4_sjNEx`+yGyRH{IhD+PA05?2fF70oRRvbTyn=GafV{2>-SOR5)yp}dOVJQnupdB__2H{ zi%Re7Q-_+nW%M@Y$ImbA3k6IhfhQs^_th%;8QPSFoVu@2dYLVA7&B7wEV3z3DWY|4`dJ^1W>(H5b9w2ewH26TeK*KTVdYH@0yhXow`Vt zEiQb%wNti%zh@KY^!l}LTgdz&+oC$>Osld`vBzQUXWP=M-9c}NQL_(n4;71kn5XGo zmVOZ3ksQkzy(!yLlj|9MYY%lc=Ah@ZOz?K%F2w`tdy65K9JF()4*MSTo^&Wn?TB3P zh4PYQtzNI2laZ^V1u@2%VYXofo#$f9?} z{g5ky{arkjo0YZngdjFBkKC`Vo`@ZkWNC`C_ZF7g_;LQ^=gJK60isc0nfD||;QbLh zqm?XPW>-Ds0dZJbpO zb}am_%z^ldSG0U6@a*@mqlI3hkR}r6(>VCjfiSOI46I~*s;(97Ro)8+>zQ@jlv$49PArKvxkxgwBdB;#)2(4-!CdDVF!4L+<>%U)0rggTDio~bmuS8 z*DD7#>a9n~qz&fVQ)Srb$Y8w@3@3OW!=V6HjEqk8@ilHta1dF<-HO!0i~(!}5~#<= z!n4PX!FG>le~I^w5dGJxZstqGGH1pB;o}eE(Eh6Be7L8vtB>x7O+Oo_hROX4XeF%iNrNuDbMF%%Fj5&tjH zZ7s_!M;$vi4iUxIB2MrA(l$%5jD^&&(JiBh?Iq~B=emhrk`8_i{Ffx(xx%$@JBb4$SlNt~?WQ(N zrbFis>F-n+Ewf$L%LDR}95)U!ev7AlHLtPc>%(EeK6Xt72Nfmhq@VH#)l!BvMwO(w<36$uo$fW(#UmwvEP`o}J zPq{_b+bON@JG)PrK_|W_HmDM^PA|s$o1Y4khOl?^I?z#%nE! z{XC7pZ{9)DmQ?j7%D20V@pyT&Qdj#Tq9{+FAHx6pAWx)0Eu9L z5P*=4FobZ6NRH@+n21=7xPVTSv+KMKCW`On=9T!~!Jpg?S1Asw@0mRV42*4P_1jnSrl*M$yOvfC< ze8(ciO2@{;PRE|bp~m6EF~AAJsl@q<^NGucYk}L0JBj-b_Z|-(j~tH=PZiGu&krvf z?;0O~55)h8AAsM8|4D#LU_uZ>@SEVAkd#n}P=_#?aDecVh?K~UsE=5H*n_x`xQBR& z_?m=}M294iWQb&!6qi(l)POXKw3+ms44W*0Y=CT+9Fbg_+<`ose1!a!f}O&PBAa53 z5}Zw{%81H?s+?+r8k<^z+JSn2=DS1cf3GEvp@e?oJ^-k!K_hm=RJ*f~ zEPy^8)bGD}--KRiQ5NiBg;%7?zy1B=B*CHtc5B`!uGQRYFqnRBRXcLS z5pE{wla8bepSRui&#pNdE4gXH30(*{{GCl_2&(6MoneF?{$&T+Oa5g?MnXO=2THwJ zNyu0l{80#UvlT~tQNytW?0(Xc(S$a90`+1L4jIB^YnjWGh~q2PwiAbQyrJWIs()GM z-LTx|QI(~BF!yZyu3jYOyxi)d6q1}%F&nsTiNOoMg)@>4DswO zd7&f@=3|L%Ce-$h8rp+jmYY_uB#UFDQ4=Lb^GwKDnU=3`E4&nCwr*b=o=B|s^hs1R#V!agd6;mD@GGo*1m^2txCCYJ=jET}Lb#)NzldN#7*)#TZtJX7)bZh()DN<&DULB-z4J%ASOCDOS zi0&0yIg1V%+Atv2pu!%dK1bsWTZ|X)or9^6BWGs)3I=Y28W_*KeR-jvY4B^gK*h{y^sAn)+SUTnDOF`orBX|!{9+a4 zVtJ-&laFDBi^D=mo7d6d<;Dz!8i#DF~u*T d`d@*P)=+z2O9=Gccp2C_0H}G=_V0V@{{Zm~b;kez literal 0 HcmV?d00001 diff --git a/iclient/iclientpkg/static-content/popper.min.js b/iclient/iclientpkg/static-content/popper.min.js new file mode 100644 index 00000000..8a17212f --- /dev/null +++ b/iclient/iclientpkg/static-content/popper.min.js @@ -0,0 +1,5 @@ +/* + Copyright (C) Federico Zivolo 2019 + Distributed under the MIT License (license terms are at http://opensource.org/licenses/MIT). + */(function(e,t){'object'==typeof exports&&'undefined'!=typeof module?module.exports=t():'function'==typeof define&&define.amd?define(t):e.Popper=t()})(this,function(){'use strict';function e(e){return e&&'[object Function]'==={}.toString.call(e)}function t(e,t){if(1!==e.nodeType)return[];var o=e.ownerDocument.defaultView,n=o.getComputedStyle(e,null);return t?n[t]:n}function o(e){return'HTML'===e.nodeName?e:e.parentNode||e.host}function n(e){if(!e)return document.body;switch(e.nodeName){case'HTML':case'BODY':return e.ownerDocument.body;case'#document':return e.body;}var i=t(e),r=i.overflow,p=i.overflowX,s=i.overflowY;return /(auto|scroll|overlay)/.test(r+s+p)?e:n(o(e))}function i(e){return e&&e.referenceNode?e.referenceNode:e}function r(e){return 11===e?re:10===e?pe:re||pe}function p(e){if(!e)return document.documentElement;for(var o=r(10)?document.body:null,n=e.offsetParent||null;n===o&&e.nextElementSibling;)n=(e=e.nextElementSibling).offsetParent;var i=n&&n.nodeName;return i&&'BODY'!==i&&'HTML'!==i?-1!==['TH','TD','TABLE'].indexOf(n.nodeName)&&'static'===t(n,'position')?p(n):n:e?e.ownerDocument.documentElement:document.documentElement}function s(e){var t=e.nodeName;return'BODY'!==t&&('HTML'===t||p(e.firstElementChild)===e)}function d(e){return null===e.parentNode?e:d(e.parentNode)}function a(e,t){if(!e||!e.nodeType||!t||!t.nodeType)return document.documentElement;var o=e.compareDocumentPosition(t)&Node.DOCUMENT_POSITION_FOLLOWING,n=o?e:t,i=o?t:e,r=document.createRange();r.setStart(n,0),r.setEnd(i,0);var l=r.commonAncestorContainer;if(e!==l&&t!==l||n.contains(i))return s(l)?l:p(l);var f=d(e);return f.host?a(f.host,t):a(e,d(t).host)}function l(e){var t=1=o.clientWidth&&n>=o.clientHeight}),l=0a[e]&&!t.escapeWithReference&&(n=Q(f[o],a[e]-('right'===e?f.width:f.height))),ae({},o,n)}};return l.forEach(function(e){var t=-1===['left','top'].indexOf(e)?'secondary':'primary';f=le({},f,m[t](e))}),e.offsets.popper=f,e},priority:['left','right','top','bottom'],padding:5,boundariesElement:'scrollParent'},keepTogether:{order:400,enabled:!0,fn:function(e){var t=e.offsets,o=t.popper,n=t.reference,i=e.placement.split('-')[0],r=Z,p=-1!==['top','bottom'].indexOf(i),s=p?'right':'bottom',d=p?'left':'top',a=p?'width':'height';return o[s]r(n[s])&&(e.offsets.popper[d]=r(n[s])),e}},arrow:{order:500,enabled:!0,fn:function(e,o){var n;if(!K(e.instance.modifiers,'arrow','keepTogether'))return e;var i=o.element;if('string'==typeof i){if(i=e.instance.popper.querySelector(i),!i)return e;}else if(!e.instance.popper.contains(i))return console.warn('WARNING: `arrow.element` must be child of its popper element!'),e;var r=e.placement.split('-')[0],p=e.offsets,s=p.popper,d=p.reference,a=-1!==['left','right'].indexOf(r),l=a?'height':'width',f=a?'Top':'Left',m=f.toLowerCase(),h=a?'left':'top',c=a?'bottom':'right',u=S(i)[l];d[c]-us[c]&&(e.offsets.popper[m]+=d[m]+u-s[c]),e.offsets.popper=g(e.offsets.popper);var b=d[m]+d[l]/2-u/2,w=t(e.instance.popper),y=parseFloat(w['margin'+f],10),E=parseFloat(w['border'+f+'Width'],10),v=b-e.offsets.popper[m]-y-E;return v=ee(Q(s[l]-u,v),0),e.arrowElement=i,e.offsets.arrow=(n={},ae(n,m,$(v)),ae(n,h,''),n),e},element:'[x-arrow]'},flip:{order:600,enabled:!0,fn:function(e,t){if(W(e.instance.modifiers,'inner'))return e;if(e.flipped&&e.placement===e.originalPlacement)return e;var o=v(e.instance.popper,e.instance.reference,t.padding,t.boundariesElement,e.positionFixed),n=e.placement.split('-')[0],i=T(n),r=e.placement.split('-')[1]||'',p=[];switch(t.behavior){case ce.FLIP:p=[n,i];break;case ce.CLOCKWISE:p=G(n);break;case ce.COUNTERCLOCKWISE:p=G(n,!0);break;default:p=t.behavior;}return p.forEach(function(s,d){if(n!==s||p.length===d+1)return e;n=e.placement.split('-')[0],i=T(n);var a=e.offsets.popper,l=e.offsets.reference,f=Z,m='left'===n&&f(a.right)>f(l.left)||'right'===n&&f(a.left)f(l.top)||'bottom'===n&&f(a.top)f(o.right),g=f(a.top)f(o.bottom),b='left'===n&&h||'right'===n&&c||'top'===n&&g||'bottom'===n&&u,w=-1!==['top','bottom'].indexOf(n),y=!!t.flipVariations&&(w&&'start'===r&&h||w&&'end'===r&&c||!w&&'start'===r&&g||!w&&'end'===r&&u),E=!!t.flipVariationsByContent&&(w&&'start'===r&&c||w&&'end'===r&&h||!w&&'start'===r&&u||!w&&'end'===r&&g),v=y||E;(m||b||v)&&(e.flipped=!0,(m||b)&&(n=p[d+1]),v&&(r=z(r)),e.placement=n+(r?'-'+r:''),e.offsets.popper=le({},e.offsets.popper,C(e.instance.popper,e.offsets.reference,e.placement)),e=P(e.instance.modifiers,e,'flip'))}),e},behavior:'flip',padding:5,boundariesElement:'viewport',flipVariations:!1,flipVariationsByContent:!1},inner:{order:700,enabled:!1,fn:function(e){var t=e.placement,o=t.split('-')[0],n=e.offsets,i=n.popper,r=n.reference,p=-1!==['left','right'].indexOf(o),s=-1===['top','left'].indexOf(o);return i[p?'left':'top']=r[o]-(s?i[p?'width':'height']:0),e.placement=T(t),e.offsets.popper=g(i),e}},hide:{order:800,enabled:!0,fn:function(e){if(!K(e.instance.modifiers,'hide','preventOverflow'))return e;var t=e.offsets.reference,o=D(e.instance.modifiers,function(e){return'preventOverflow'===e.name}).boundaries;if(t.bottomo.right||t.top>o.bottom||t.rightwindow.devicePixelRatio||!fe),c='bottom'===o?'top':'bottom',g='right'===n?'left':'right',b=B('transform');if(d='bottom'==c?'HTML'===l.nodeName?-l.clientHeight+h.bottom:-f.height+h.bottom:h.top,s='right'==g?'HTML'===l.nodeName?-l.clientWidth+h.right:-f.width+h.right:h.left,a&&b)m[b]='translate3d('+s+'px, '+d+'px, 0)',m[c]=0,m[g]=0,m.willChange='transform';else{var w='bottom'==c?-1:1,y='right'==g?-1:1;m[c]=d*w,m[g]=s*y,m.willChange=c+', '+g}var E={"x-placement":e.placement};return e.attributes=le({},E,e.attributes),e.styles=le({},m,e.styles),e.arrowStyles=le({},e.offsets.arrow,e.arrowStyles),e},gpuAcceleration:!0,x:'bottom',y:'right'},applyStyle:{order:900,enabled:!0,fn:function(e){return V(e.instance.popper,e.styles),j(e.instance.popper,e.attributes),e.arrowElement&&Object.keys(e.arrowStyles).length&&V(e.arrowElement,e.arrowStyles),e},onLoad:function(e,t,o,n,i){var r=L(i,t,e,o.positionFixed),p=O(o.placement,r,t,e,o.modifiers.flip.boundariesElement,o.modifiers.flip.padding);return t.setAttribute('x-placement',p),V(t,{position:o.positionFixed?'fixed':'absolute'}),o},gpuAcceleration:void 0}}},ge}); +//# sourceMappingURL=popper.min.js.map diff --git a/iclient/iclientpkg/static-content/styles.css b/iclient/iclientpkg/static-content/styles.css new file mode 100644 index 00000000..b39be2f0 --- /dev/null +++ b/iclient/iclientpkg/static-content/styles.css @@ -0,0 +1,85 @@ +/* + Copyright (c) 2015-2021, NVIDIA CORPORATION. + SPDX-License-Identifier: Apache-2.0 +*/ + +.table td.fit, +.table th.fit { + white-space: nowrap; + width: 1%; +} + +body { padding-top: 70px; } + +.no-margin { margin: 0; } + +pre.code { + background-color: #e9ecef; + border-radius: .25rem; + padding: 20px; +} + +.clickable { + cursor: pointer; +} + +span.jstExpand, span.jstFold { + cursor: pointer; +} + +#btn-back-to-top { + position: fixed; + bottom: 20px; + right: 20px; + display: none; +} + +/* jsontree.css */ + +.jstValue { + white-space: pre-wrap; + /*font-size: 10px; + font-weight: 400; + font-family: "Lucida Console", Monaco, monospace;*/ +} +.jstComma { + white-space: pre-wrap; +} +.jstProperty { + color: #666; + word-wrap: break-word; +} +.jstBracket { + white-space: pre-wrap;; +} +.jstBool { + color: #2525CC; +} +.jstNum { + color: #D036D0; +} +.jstNull { + color: gray; +} +.jstStr { + color: #2DB669; +} +.jstFold:after { + content: ' -'; + cursor: pointer; +} +.jstExpand { + white-space: normal; +} +.jstExpand:after { + content: ' +'; + cursor: pointer; +} +.jstFolded { + white-space: normal !important; +} +.jstHiddenBlock { + display: none; +} + +/* End of jsontree.css */ diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index b141d81b..92833e65 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -70,7 +70,7 @@ // the RPCs will be available via :. // // The RESTful API is provided by an embedded HTTP Server -// (at URL http://:) responsing to the following: +// (at URL http://:) responds to the following: // // DELETE /volume/ // @@ -86,6 +86,10 @@ // // This will return a raw bucketstats dump. // +// GET /version +// +// This will return the imgr version. +// // GET /volume // // This will return a JSON document containing an array of volumes currently diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index be752fea..2061fa77 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -62,7 +62,7 @@ func startHTTPServer() (err error) { }() for startHTTPServerUpCheckRetries = 0; startHTTPServerUpCheckRetries < startHTTPServerUpCheckMaxRetries; startHTTPServerUpCheckRetries++ { - _, err = http.Get("http://" + ipAddrTCPPort + "/config") + _, err = http.Get("http://" + ipAddrTCPPort + "/version") if nil == err { return } @@ -132,11 +132,9 @@ func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.R var ( err error pathSplit []string - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - pathSplit = strings.Split(requestPath, "/") switch len(pathSplit) { @@ -235,10 +233,9 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ var ( confMapJSON []byte err error - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() defer func() { globals.stats.GetConfigUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -271,11 +268,10 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { var ( err error - startTime time.Time + startTime time.Time = time.Now() statsAsString string ) - startTime = time.Now() defer func() { globals.stats.GetStatsUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -714,7 +710,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ pathSplit []string pendingDeleteObjectNameArrayIndex int requestAuthToken string - startTime time.Time + startTime time.Time = time.Now() superBlockV1 *ilayout.SuperBlockV1Struct volumeAsStruct *volumeStruct volumeAsValue sortedmap.Value @@ -730,8 +726,6 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeName string ) - startTime = time.Now() - pathSplit = strings.Split(requestPath, "/") switch len(pathSplit) { @@ -1354,11 +1348,9 @@ func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, request *http.Req var ( err error requestBodyAsJSON serveHTTPPostOfVolumeRequestBodyAsJSONStruct - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - defer func() { globals.stats.PostVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -1396,11 +1388,9 @@ func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Requ err error pathSplit []string requestBodyAsJSON serveHTTPPutOfVolumeRequestBodyAsJSONStruct - startTime time.Time + startTime time.Time = time.Now() ) - startTime = time.Now() - pathSplit = strings.Split(requestPath, "/") switch len(pathSplit) { From 1241409ad9371264f8933c8a5a5c1fcffba1c8ac Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 16 Sep 2021 14:36:08 -0700 Subject: [PATCH 092/258] Refactored common HTML content serving out of imgr & iclient (into ihtml) Also made HTML serving in iclient optional --- Makefile | 3 +- go.mod | 2 +- go.sum | 2 + iclient/iclientpkg/.gitignore | 13 - iclient/iclientpkg/http-server.go | 70 +-- iclient/iclientpkg/static-content.go | 22 - {imgr/imgrpkg => ihtml}/.gitignore | 0 ihtml/Makefile | 22 + ihtml/dummy_test.go | 11 + ihtml/http-server.go | 71 +++ ihtml/make-static-content/.gitignore | 1 + ihtml/make-static-content/Makefile | 9 + ihtml/make-static-content/dummy_test.go | 11 + ihtml/make-static-content/main.go | 128 +++++ ihtml/static-content.go | 24 + .../static-content/bootstrap.min.css | 0 .../static-content/bootstrap.min.js | 0 .../static-content/jquery.min.js | 0 .../static-content/jsontree.js | 0 .../font/css/open-iconic-bootstrap.min.css | 0 .../open-iconic/font/fonts/open-iconic.eot | Bin .../open-iconic/font/fonts/open-iconic.otf | Bin .../open-iconic/font/fonts/open-iconic.svg | 0 .../open-iconic/font/fonts/open-iconic.ttf | Bin .../open-iconic/font/fonts/open-iconic.woff | Bin .../static-content/popper.min.js | 0 .../static-content/styles.css | 0 .../imgrpkg => ihtml}/static-content/utils.js | 0 imgr/imgrpkg/Makefile | 16 - imgr/imgrpkg/http-server.go | 65 +-- imgr/imgrpkg/static-content.go | 24 - imgr/imgrpkg/static-content/bootstrap.min.css | 7 - imgr/imgrpkg/static-content/bootstrap.min.js | 7 - imgr/imgrpkg/static-content/jquery.min.js | 2 - imgr/imgrpkg/static-content/jsontree.js | 182 ------ .../font/css/open-iconic-bootstrap.min.css | 1 - .../open-iconic/font/fonts/open-iconic.eot | Bin 28196 -> 0 bytes .../open-iconic/font/fonts/open-iconic.otf | Bin 20996 -> 0 bytes .../open-iconic/font/fonts/open-iconic.svg | 543 ------------------ .../open-iconic/font/fonts/open-iconic.ttf | Bin 28028 -> 0 bytes .../open-iconic/font/fonts/open-iconic.woff | Bin 14984 -> 0 bytes imgr/imgrpkg/static-content/popper.min.js | 5 - imgr/imgrpkg/static-content/styles.css | 85 --- 43 files changed, 314 insertions(+), 1012 deletions(-) delete mode 100644 iclient/iclientpkg/.gitignore delete mode 100644 iclient/iclientpkg/static-content.go rename {imgr/imgrpkg => ihtml}/.gitignore (100%) create mode 100644 ihtml/Makefile create mode 100644 ihtml/dummy_test.go create mode 100644 ihtml/http-server.go create mode 100644 ihtml/make-static-content/.gitignore create mode 100644 ihtml/make-static-content/Makefile create mode 100644 ihtml/make-static-content/dummy_test.go create mode 100644 ihtml/make-static-content/main.go create mode 100644 ihtml/static-content.go rename {iclient/iclientpkg => ihtml}/static-content/bootstrap.min.css (100%) rename {iclient/iclientpkg => ihtml}/static-content/bootstrap.min.js (100%) rename {iclient/iclientpkg => ihtml}/static-content/jquery.min.js (100%) rename {iclient/iclientpkg => ihtml}/static-content/jsontree.js (100%) rename {iclient/iclientpkg => ihtml}/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css (100%) rename {iclient/iclientpkg => ihtml}/static-content/open-iconic/font/fonts/open-iconic.eot (100%) rename {iclient/iclientpkg => ihtml}/static-content/open-iconic/font/fonts/open-iconic.otf (100%) rename {iclient/iclientpkg => ihtml}/static-content/open-iconic/font/fonts/open-iconic.svg (100%) rename {iclient/iclientpkg => ihtml}/static-content/open-iconic/font/fonts/open-iconic.ttf (100%) rename {iclient/iclientpkg => ihtml}/static-content/open-iconic/font/fonts/open-iconic.woff (100%) rename {iclient/iclientpkg => ihtml}/static-content/popper.min.js (100%) rename {iclient/iclientpkg => ihtml}/static-content/styles.css (100%) rename {imgr/imgrpkg => ihtml}/static-content/utils.js (100%) delete mode 100644 imgr/imgrpkg/static-content.go delete mode 100644 imgr/imgrpkg/static-content/bootstrap.min.css delete mode 100644 imgr/imgrpkg/static-content/bootstrap.min.js delete mode 100644 imgr/imgrpkg/static-content/jquery.min.js delete mode 100644 imgr/imgrpkg/static-content/jsontree.js delete mode 100755 imgr/imgrpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css delete mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.eot delete mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.otf delete mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.svg delete mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.ttf delete mode 100755 imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.woff delete mode 100644 imgr/imgrpkg/static-content/popper.min.js delete mode 100644 imgr/imgrpkg/static-content/styles.css diff --git a/Makefile b/Makefile index db4675f4..3dadbd9b 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 gopregeneratedirs = \ - make-static-content + ihtml/make-static-content gopkgdirs = \ bucketstats \ conf \ iauth \ + ihtml \ ilayout \ retryrpc \ utils \ diff --git a/go.mod b/go.mod index 4a7de236..422992a8 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,7 @@ require ( google.golang.org/grpc v1.40.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/ini.v1 v1.63.1 // indirect + gopkg.in/ini.v1 v1.63.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/go.sum b/go.sum index 0a8829e1..65e380c4 100644 --- a/go.sum +++ b/go.sum @@ -801,6 +801,8 @@ gopkg.in/ini.v1 v1.63.0 h1:2t0h8NA59dpVQpa5Yh8cIcR6nHAeBIEk0zlLVqfw4N4= gopkg.in/ini.v1 v1.63.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.63.1 h1:WlmD2fPTg4maPpRITalGs62TK7VMMtP5E9CHH7aFy6Y= gopkg.in/ini.v1 v1.63.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= +gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/iclient/iclientpkg/.gitignore b/iclient/iclientpkg/.gitignore deleted file mode 100644 index 3a694af3..00000000 --- a/iclient/iclientpkg/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -bootstrap_dot_min_dot_css_.go -bootstrap_dot_min_dot_js_.go -jquery_dot_min_dot_js_.go -jquery_underline_3_dot_2_dot_1_dot_min_dot_js_.go -jsontree_dot_js_.go -open_iconic_bootstrap_dot_css_.go -open_iconic_dot_eot_.go -open_iconic_dot_otf_.go -open_iconic_dot_svg_.go -open_iconic_dot_ttf_.go -open_iconic_dot_woff_.go -popper_dot_min_dot_js_.go -styles_dot_css_.go diff --git a/iclient/iclientpkg/http-server.go b/iclient/iclientpkg/http-server.go index 8f2880b6..a865c8e2 100644 --- a/iclient/iclientpkg/http-server.go +++ b/iclient/iclientpkg/http-server.go @@ -16,6 +16,7 @@ import ( "time" "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/ihtml" "github.com/NVIDIA/proxyfs/version" ) @@ -30,6 +31,12 @@ func startHTTPServer() (err error) { startHTTPServerUpCheckRetries uint32 ) + if globals.config.HTTPServerPort == 0 { + globals.httpServer = nil + err = nil + return + } + ipAddrTCPPort = net.JoinHostPort(globals.config.HTTPServerIPAddr, strconv.Itoa(int(globals.config.HTTPServerPort))) globals.httpServer = &http.Server{ @@ -66,6 +73,12 @@ func startHTTPServer() (err error) { } func stopHTTPServer() (err error) { + if globals.config.HTTPServerPort == 0 { + globals.httpServer = nil + err = nil + return + } + err = globals.httpServer.Shutdown(context.TODO()) if nil == err { globals.httpServerWG.Wait() @@ -104,71 +117,30 @@ func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, reques } func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { + var ( + ok bool + ) + switch { case "" == requestPath: responseWriter.Header().Set("Content-Type", "text/html") responseWriter.WriteHeader(http.StatusOK) _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) - case "/bootstrap.min.css" == requestPath: - responseWriter.Header().Set("Content-Type", bootstrapDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(bootstrapDotCSSContent)) - case "/bootstrap.min.js" == requestPath: - responseWriter.Header().Set("Content-Type", bootstrapDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(bootstrapDotJSContent)) case "/config" == requestPath: serveHTTPGetOfConfig(responseWriter, request) case "/index.html" == requestPath: responseWriter.Header().Set("Content-Type", "text/html") responseWriter.WriteHeader(http.StatusOK) _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) - case "/jquery.min.js" == requestPath: - responseWriter.Header().Set("Content-Type", jqueryDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(jqueryDotJSContent)) - case "/jsontree.js" == requestPath: - responseWriter.Header().Set("Content-Type", jsontreeDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(jsontreeDotJSContent)) - case "/open-iconic/font/css/open-iconic-bootstrap.min.css" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicBootstrapDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(openIconicBootstrapDotCSSContent)) - case "/open-iconic/font/fonts/open-iconic.eot" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotEOTContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotEOTContent) - case "/open-iconic/font/fonts/open-iconic.otf" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotOTFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotOTFContent) - case "/open-iconic/font/fonts/open-iconic.svg" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotSVGContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(openIconicDotSVGContent)) - case "/open-iconic/font/fonts/open-iconic.ttf" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotTTFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotTTFContent) - case "/open-iconic/font/fonts/open-iconic.woff" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotWOFFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotWOFFContent) - case "/popper.min.js" == requestPath: - responseWriter.Header().Set("Content-Type", popperDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(popperDotJSContent) case "/stats" == requestPath: serveHTTPGetOfStats(responseWriter, request) - case "/styles.css" == requestPath: - responseWriter.Header().Set("Content-Type", stylesDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(stylesDotCSSContent)) case "/version" == requestPath: serveHTTPGetOfVersion(responseWriter, request) default: - responseWriter.WriteHeader(http.StatusNotFound) + ok = ihtml.ServeHTTPGet(responseWriter, requestPath) + if !ok { + responseWriter.WriteHeader(http.StatusNotFound) + } } } diff --git a/iclient/iclientpkg/static-content.go b/iclient/iclientpkg/static-content.go deleted file mode 100644 index 133d22e0..00000000 --- a/iclient/iclientpkg/static-content.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package iclientpkg - -//go:generate ../../make-static-content/make-static-content iclientpkg stylesDotCSS text/css s static-content/styles.css styles_dot_css_.go - -//go:generate ../../make-static-content/make-static-content iclientpkg jsontreeDotJS application/javascript s static-content/jsontree.js jsontree_dot_js_.go - -//go:generate ../../make-static-content/make-static-content iclientpkg bootstrapDotCSS text/css s static-content/bootstrap.min.css bootstrap_dot_min_dot_css_.go - -//go:generate ../../make-static-content/make-static-content iclientpkg bootstrapDotJS application/javascript s static-content/bootstrap.min.js bootstrap_dot_min_dot_js_.go -//go:generate ../../make-static-content/make-static-content iclientpkg jqueryDotJS application/javascript s static-content/jquery.min.js jquery_dot_min_dot_js_.go -//go:generate ../../make-static-content/make-static-content iclientpkg popperDotJS application/javascript b static-content/popper.min.js popper_dot_min_dot_js_.go - -//go:generate ../../make-static-content/make-static-content iclientpkg openIconicBootstrapDotCSS text/css s static-content/open-iconic/font/css/open-iconic-bootstrap.min.css open_iconic_bootstrap_dot_css_.go - -//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotEOT application/vnd.ms-fontobject b static-content/open-iconic/font/fonts/open-iconic.eot open_iconic_dot_eot_.go -//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotOTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.otf open_iconic_dot_otf_.go -//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotSVG image/svg+xml s static-content/open-iconic/font/fonts/open-iconic.svg open_iconic_dot_svg_.go -//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotTTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.ttf open_iconic_dot_ttf_.go -//go:generate ../../make-static-content/make-static-content iclientpkg openIconicDotWOFF application/font-woff b static-content/open-iconic/font/fonts/open-iconic.woff open_iconic_dot_woff_.go diff --git a/imgr/imgrpkg/.gitignore b/ihtml/.gitignore similarity index 100% rename from imgr/imgrpkg/.gitignore rename to ihtml/.gitignore diff --git a/ihtml/Makefile b/ihtml/Makefile new file mode 100644 index 00000000..879a2a3e --- /dev/null +++ b/ihtml/Makefile @@ -0,0 +1,22 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/ihtml + +generatedfiles := \ + bootstrap_dot_min_dot_css_.go \ + bootstrap_dot_min_dot_js_.go \ + jquery_dot_min_dot_js_.go \ + jquery_underline_3_dot_2_dot_1_dot_min_dot_js_.go \ + jsontree_dot_js_.go \ + open_iconic_bootstrap_dot_css_.go \ + open_iconic_dot_eot_.go \ + open_iconic_dot_otf_.go \ + open_iconic_dot_svg_.go \ + open_iconic_dot_ttf_.go \ + open_iconic_dot_woff_.go \ + popper_dot_min_dot_js_.go \ + styles_dot_css_.go \ + utils_dot_js_.go + +include ../GoMakefile diff --git a/ihtml/dummy_test.go b/ihtml/dummy_test.go new file mode 100644 index 00000000..80a36ad2 --- /dev/null +++ b/ihtml/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package ihtml + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/ihtml/http-server.go b/ihtml/http-server.go new file mode 100644 index 00000000..31c7cc2b --- /dev/null +++ b/ihtml/http-server.go @@ -0,0 +1,71 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package ihtml + +import ( + "net/http" +) + +func ServeHTTPGet(responseWriter http.ResponseWriter, requestPath string) (ok bool) { + ok = true // Default ok assuming switch statement has a matching case to serve + + switch { + case "/bootstrap.min.css" == requestPath: + responseWriter.Header().Set("Content-Type", bootstrapDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(bootstrapDotCSSContent)) + case "/bootstrap.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", bootstrapDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(bootstrapDotJSContent)) + case "/jquery.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", jqueryDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(jqueryDotJSContent)) + case "/jsontree.js" == requestPath: + responseWriter.Header().Set("Content-Type", jsontreeDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(jsontreeDotJSContent)) + case "/open-iconic/font/css/open-iconic-bootstrap.min.css" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicBootstrapDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(openIconicBootstrapDotCSSContent)) + case "/open-iconic/font/fonts/open-iconic.eot" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotEOTContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotEOTContent) + case "/open-iconic/font/fonts/open-iconic.otf" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotOTFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotOTFContent) + case "/open-iconic/font/fonts/open-iconic.svg" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotSVGContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(openIconicDotSVGContent)) + case "/open-iconic/font/fonts/open-iconic.ttf" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotTTFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotTTFContent) + case "/open-iconic/font/fonts/open-iconic.woff" == requestPath: + responseWriter.Header().Set("Content-Type", openIconicDotWOFFContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(openIconicDotWOFFContent) + case "/popper.min.js" == requestPath: + responseWriter.Header().Set("Content-Type", popperDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(popperDotJSContent) + case "/styles.css" == requestPath: + responseWriter.Header().Set("Content-Type", stylesDotCSSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(stylesDotCSSContent)) + case "/utils.js" == requestPath: + responseWriter.Header().Set("Content-Type", utilsDotJSContentType) + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(utilsDotJSContent)) + default: + ok = false // No switch case match + } + + return +} diff --git a/ihtml/make-static-content/.gitignore b/ihtml/make-static-content/.gitignore new file mode 100644 index 00000000..61c4948c --- /dev/null +++ b/ihtml/make-static-content/.gitignore @@ -0,0 +1 @@ +make-static-content diff --git a/ihtml/make-static-content/Makefile b/ihtml/make-static-content/Makefile new file mode 100644 index 00000000..fe5fcfc2 --- /dev/null +++ b/ihtml/make-static-content/Makefile @@ -0,0 +1,9 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/ihtml/make-static-content + +generatedfiles := \ + make-static-content + +include ../../GoMakefile diff --git a/ihtml/make-static-content/dummy_test.go b/ihtml/make-static-content/dummy_test.go new file mode 100644 index 00000000..d4299172 --- /dev/null +++ b/ihtml/make-static-content/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/ihtml/make-static-content/main.go b/ihtml/make-static-content/main.go new file mode 100644 index 00000000..76d90c65 --- /dev/null +++ b/ihtml/make-static-content/main.go @@ -0,0 +1,128 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" +) + +const bytesPerLine = 16 + +func usage() { + fmt.Println("make-static-content -?") + fmt.Println(" Prints this help text") + fmt.Println("make-static-content ") + fmt.Println(" is the name of the ultimate package for ") + fmt.Println(" is the basename of the desired content resource") + fmt.Println(" is the string to record as the static content's Content-Type") + fmt.Println(" indicates whether the static content is a string (\"s\") or a []byte (\"b\")") + fmt.Println(" is the path to the static content to be embedded") + fmt.Println(" is the name of the generated .go source file containing:") + fmt.Println(" ContentType string holding value of ") + fmt.Println(" Content string or []byte holding contents of ") +} + +var bs = []byte{} + +func main() { + var ( + contentFormat string + contentName string + contentType string + dstFile *os.File + dstFileContents bytes.Buffer + dstFileName string + err error + packageName string + srcFileContentByte byte + srcFileContentIndex int + srcFileContents []byte + srcFileName string + ) + + if (2 == len(os.Args)) && ("-?" == os.Args[1]) { + usage() + os.Exit(0) + } + + if 7 != len(os.Args) { + usage() + os.Exit(1) + } + + packageName = os.Args[1] + contentName = os.Args[2] + contentType = os.Args[3] + contentFormat = os.Args[4] + srcFileName = os.Args[5] + dstFileName = os.Args[6] + + srcFileContents, err = ioutil.ReadFile(srcFileName) + if nil != err { + panic(err.Error()) + } + + dstFile, err = os.Create(dstFileName) + if nil != err { + panic(err.Error()) + } + + _, err = dstFile.Write([]byte(fmt.Sprintf("// Code generated by \"go run make_static_content.go %v %v %v %v %v %v\" - DO NOT EDIT\n\n", packageName, contentName, contentType, contentFormat, srcFileName, dstFileName))) + if nil != err { + panic(err.Error()) + } + _, err = dstFile.Write([]byte(fmt.Sprintf("package %v\n\n", packageName))) + if nil != err { + panic(err.Error()) + } + + _, err = dstFile.Write([]byte(fmt.Sprintf("const %vContentType = \"%v\"\n\n", contentName, contentType))) + if nil != err { + panic(err.Error()) + } + + switch contentFormat { + case "s": + _, err = dstFile.Write([]byte(fmt.Sprintf("const %vContent = `%v`\n", contentName, string(srcFileContents[:])))) + if nil != err { + panic(err.Error()) + } + case "b": + _, err = dstFileContents.Write([]byte(fmt.Sprintf("var %vContent = []byte{", contentName))) + if nil != err { + panic(err.Error()) + } + for srcFileContentIndex, srcFileContentByte = range srcFileContents { + if (srcFileContentIndex % bytesPerLine) == 0 { + _, err = dstFileContents.Write([]byte(fmt.Sprintf("\n\t0x%02X,", srcFileContentByte))) + } else { + _, err = dstFileContents.Write([]byte(fmt.Sprintf(" 0x%02X,", srcFileContentByte))) + } + if nil != err { + panic(err.Error()) + } + } + _, err = dstFileContents.Write([]byte("\n}\n")) + if nil != err { + panic(err.Error()) + } + _, err = dstFile.Write(dstFileContents.Bytes()) + if nil != err { + panic(err.Error()) + } + default: + usage() + os.Exit(1) + } + + err = dstFile.Close() + if nil != err { + panic(err.Error()) + } + + os.Exit(0) +} diff --git a/ihtml/static-content.go b/ihtml/static-content.go new file mode 100644 index 00000000..c7548e7b --- /dev/null +++ b/ihtml/static-content.go @@ -0,0 +1,24 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package ihtml + +//go:generate make-static-content/make-static-content ihtml stylesDotCSS text/css s static-content/styles.css styles_dot_css_.go + +//go:generate make-static-content/make-static-content ihtml utilsDotJS application/javascript s static-content/utils.js utils_dot_js_.go + +//go:generate make-static-content/make-static-content ihtml jsontreeDotJS application/javascript s static-content/jsontree.js jsontree_dot_js_.go + +//go:generate make-static-content/make-static-content ihtml bootstrapDotCSS text/css s static-content/bootstrap.min.css bootstrap_dot_min_dot_css_.go + +//go:generate make-static-content/make-static-content ihtml bootstrapDotJS application/javascript s static-content/bootstrap.min.js bootstrap_dot_min_dot_js_.go +//go:generate make-static-content/make-static-content ihtml jqueryDotJS application/javascript s static-content/jquery.min.js jquery_dot_min_dot_js_.go +//go:generate make-static-content/make-static-content ihtml popperDotJS application/javascript b static-content/popper.min.js popper_dot_min_dot_js_.go + +//go:generate make-static-content/make-static-content ihtml openIconicBootstrapDotCSS text/css s static-content/open-iconic/font/css/open-iconic-bootstrap.min.css open_iconic_bootstrap_dot_css_.go + +//go:generate make-static-content/make-static-content ihtml openIconicDotEOT application/vnd.ms-fontobject b static-content/open-iconic/font/fonts/open-iconic.eot open_iconic_dot_eot_.go +//go:generate make-static-content/make-static-content ihtml openIconicDotOTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.otf open_iconic_dot_otf_.go +//go:generate make-static-content/make-static-content ihtml openIconicDotSVG image/svg+xml s static-content/open-iconic/font/fonts/open-iconic.svg open_iconic_dot_svg_.go +//go:generate make-static-content/make-static-content ihtml openIconicDotTTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.ttf open_iconic_dot_ttf_.go +//go:generate make-static-content/make-static-content ihtml openIconicDotWOFF application/font-woff b static-content/open-iconic/font/fonts/open-iconic.woff open_iconic_dot_woff_.go diff --git a/iclient/iclientpkg/static-content/bootstrap.min.css b/ihtml/static-content/bootstrap.min.css similarity index 100% rename from iclient/iclientpkg/static-content/bootstrap.min.css rename to ihtml/static-content/bootstrap.min.css diff --git a/iclient/iclientpkg/static-content/bootstrap.min.js b/ihtml/static-content/bootstrap.min.js similarity index 100% rename from iclient/iclientpkg/static-content/bootstrap.min.js rename to ihtml/static-content/bootstrap.min.js diff --git a/iclient/iclientpkg/static-content/jquery.min.js b/ihtml/static-content/jquery.min.js similarity index 100% rename from iclient/iclientpkg/static-content/jquery.min.js rename to ihtml/static-content/jquery.min.js diff --git a/iclient/iclientpkg/static-content/jsontree.js b/ihtml/static-content/jsontree.js similarity index 100% rename from iclient/iclientpkg/static-content/jsontree.js rename to ihtml/static-content/jsontree.js diff --git a/iclient/iclientpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css b/ihtml/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css similarity index 100% rename from iclient/iclientpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css rename to ihtml/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css diff --git a/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.eot b/ihtml/static-content/open-iconic/font/fonts/open-iconic.eot similarity index 100% rename from iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.eot rename to ihtml/static-content/open-iconic/font/fonts/open-iconic.eot diff --git a/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.otf b/ihtml/static-content/open-iconic/font/fonts/open-iconic.otf similarity index 100% rename from iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.otf rename to ihtml/static-content/open-iconic/font/fonts/open-iconic.otf diff --git a/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.svg b/ihtml/static-content/open-iconic/font/fonts/open-iconic.svg similarity index 100% rename from iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.svg rename to ihtml/static-content/open-iconic/font/fonts/open-iconic.svg diff --git a/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.ttf b/ihtml/static-content/open-iconic/font/fonts/open-iconic.ttf similarity index 100% rename from iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.ttf rename to ihtml/static-content/open-iconic/font/fonts/open-iconic.ttf diff --git a/iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.woff b/ihtml/static-content/open-iconic/font/fonts/open-iconic.woff similarity index 100% rename from iclient/iclientpkg/static-content/open-iconic/font/fonts/open-iconic.woff rename to ihtml/static-content/open-iconic/font/fonts/open-iconic.woff diff --git a/iclient/iclientpkg/static-content/popper.min.js b/ihtml/static-content/popper.min.js similarity index 100% rename from iclient/iclientpkg/static-content/popper.min.js rename to ihtml/static-content/popper.min.js diff --git a/iclient/iclientpkg/static-content/styles.css b/ihtml/static-content/styles.css similarity index 100% rename from iclient/iclientpkg/static-content/styles.css rename to ihtml/static-content/styles.css diff --git a/imgr/imgrpkg/static-content/utils.js b/ihtml/static-content/utils.js similarity index 100% rename from imgr/imgrpkg/static-content/utils.js rename to ihtml/static-content/utils.js diff --git a/imgr/imgrpkg/Makefile b/imgr/imgrpkg/Makefile index df97cbc8..a2b7d274 100644 --- a/imgr/imgrpkg/Makefile +++ b/imgr/imgrpkg/Makefile @@ -3,20 +3,4 @@ gosubdir := github.com/NVIDIA/proxyfs/imgr/imgrpkg -generatedfiles := \ - bootstrap_dot_min_dot_css_.go \ - bootstrap_dot_min_dot_js_.go \ - jquery_dot_min_dot_js_.go \ - jquery_underline_3_dot_2_dot_1_dot_min_dot_js_.go \ - jsontree_dot_js_.go \ - open_iconic_bootstrap_dot_css_.go \ - open_iconic_dot_eot_.go \ - open_iconic_dot_otf_.go \ - open_iconic_dot_svg_.go \ - open_iconic_dot_ttf_.go \ - open_iconic_dot_woff_.go \ - popper_dot_min_dot_js_.go \ - styles_dot_css_.go \ - utils_dot_js_.go - include ../../GoMakefile diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 2061fa77..be3bf4be 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -15,10 +15,12 @@ import ( "strings" "time" + "github.com/NVIDIA/sortedmap" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/ihtml" "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/version" - "github.com/NVIDIA/sortedmap" ) const ( @@ -155,77 +157,32 @@ func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.R } func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { + var ( + ok bool + ) + switch { case "" == requestPath: responseWriter.Header().Set("Content-Type", "text/html") responseWriter.WriteHeader(http.StatusOK) _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) - case "/bootstrap.min.css" == requestPath: - responseWriter.Header().Set("Content-Type", bootstrapDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(bootstrapDotCSSContent)) - case "/bootstrap.min.js" == requestPath: - responseWriter.Header().Set("Content-Type", bootstrapDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(bootstrapDotJSContent)) case "/config" == requestPath: serveHTTPGetOfConfig(responseWriter, request) case "/index.html" == requestPath: responseWriter.Header().Set("Content-Type", "text/html") responseWriter.WriteHeader(http.StatusOK) _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) - case "/jquery.min.js" == requestPath: - responseWriter.Header().Set("Content-Type", jqueryDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(jqueryDotJSContent)) - case "/jsontree.js" == requestPath: - responseWriter.Header().Set("Content-Type", jsontreeDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(jsontreeDotJSContent)) - case "/open-iconic/font/css/open-iconic-bootstrap.min.css" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicBootstrapDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(openIconicBootstrapDotCSSContent)) - case "/open-iconic/font/fonts/open-iconic.eot" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotEOTContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotEOTContent) - case "/open-iconic/font/fonts/open-iconic.otf" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotOTFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotOTFContent) - case "/open-iconic/font/fonts/open-iconic.svg" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotSVGContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(openIconicDotSVGContent)) - case "/open-iconic/font/fonts/open-iconic.ttf" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotTTFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotTTFContent) - case "/open-iconic/font/fonts/open-iconic.woff" == requestPath: - responseWriter.Header().Set("Content-Type", openIconicDotWOFFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotWOFFContent) - case "/popper.min.js" == requestPath: - responseWriter.Header().Set("Content-Type", popperDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(popperDotJSContent) case "/stats" == requestPath: serveHTTPGetOfStats(responseWriter, request) - case "/styles.css" == requestPath: - responseWriter.Header().Set("Content-Type", stylesDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(stylesDotCSSContent)) - case "/utils.js" == requestPath: - responseWriter.Header().Set("Content-Type", utilsDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(utilsDotJSContent)) case "/version" == requestPath: serveHTTPGetOfVersion(responseWriter, request) case strings.HasPrefix(requestPath, "/volume"): serveHTTPGetOfVolume(responseWriter, request, requestPath) default: - responseWriter.WriteHeader(http.StatusNotFound) + ok = ihtml.ServeHTTPGet(responseWriter, requestPath) + if !ok { + responseWriter.WriteHeader(http.StatusNotFound) + } } } diff --git a/imgr/imgrpkg/static-content.go b/imgr/imgrpkg/static-content.go deleted file mode 100644 index c8e0eabd..00000000 --- a/imgr/imgrpkg/static-content.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package imgrpkg - -//go:generate ../../make-static-content/make-static-content imgrpkg stylesDotCSS text/css s static-content/styles.css styles_dot_css_.go - -//go:generate ../../make-static-content/make-static-content imgrpkg utilsDotJS application/javascript s static-content/utils.js utils_dot_js_.go - -//go:generate ../../make-static-content/make-static-content imgrpkg jsontreeDotJS application/javascript s static-content/jsontree.js jsontree_dot_js_.go - -//go:generate ../../make-static-content/make-static-content imgrpkg bootstrapDotCSS text/css s static-content/bootstrap.min.css bootstrap_dot_min_dot_css_.go - -//go:generate ../../make-static-content/make-static-content imgrpkg bootstrapDotJS application/javascript s static-content/bootstrap.min.js bootstrap_dot_min_dot_js_.go -//go:generate ../../make-static-content/make-static-content imgrpkg jqueryDotJS application/javascript s static-content/jquery.min.js jquery_dot_min_dot_js_.go -//go:generate ../../make-static-content/make-static-content imgrpkg popperDotJS application/javascript b static-content/popper.min.js popper_dot_min_dot_js_.go - -//go:generate ../../make-static-content/make-static-content imgrpkg openIconicBootstrapDotCSS text/css s static-content/open-iconic/font/css/open-iconic-bootstrap.min.css open_iconic_bootstrap_dot_css_.go - -//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotEOT application/vnd.ms-fontobject b static-content/open-iconic/font/fonts/open-iconic.eot open_iconic_dot_eot_.go -//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotOTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.otf open_iconic_dot_otf_.go -//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotSVG image/svg+xml s static-content/open-iconic/font/fonts/open-iconic.svg open_iconic_dot_svg_.go -//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotTTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.ttf open_iconic_dot_ttf_.go -//go:generate ../../make-static-content/make-static-content imgrpkg openIconicDotWOFF application/font-woff b static-content/open-iconic/font/fonts/open-iconic.woff open_iconic_dot_woff_.go diff --git a/imgr/imgrpkg/static-content/bootstrap.min.css b/imgr/imgrpkg/static-content/bootstrap.min.css deleted file mode 100644 index 7d2a868f..00000000 --- a/imgr/imgrpkg/static-content/bootstrap.min.css +++ /dev/null @@ -1,7 +0,0 @@ -/*! - * Bootstrap v4.5.0 (https://getbootstrap.com/) - * Copyright 2011-2020 The Bootstrap Authors - * Copyright 2011-2020 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */:root{--blue:#007bff;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#dc3545;--orange:#fd7e14;--yellow:#ffc107;--green:#28a745;--teal:#20c997;--cyan:#17a2b8;--white:#fff;--gray:#6c757d;--gray-dark:#343a40;--primary:#007bff;--secondary:#6c757d;--success:#28a745;--info:#17a2b8;--warning:#ffc107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus:not(:focus-visible){outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]){color:inherit;text-decoration:none}a:not([href]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-weight:500;line-height:1.2}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#6c757d}.blockquote-footer::before{content:"\2014\00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#6c757d}code{font-size:87.5%;color:#e83e8c;word-wrap:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:87.5%;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;font-size:87.5%;color:#212529}pre code{font-size:inherit;color:inherit;word-break:normal}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container{max-width:540px}}@media (min-width:768px){.container{max-width:720px}}@media (min-width:992px){.container{max-width:960px}}@media (min-width:1200px){.container{max-width:1140px}}.container-fluid,.container-lg,.container-md,.container-sm,.container-xl{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container,.container-sm{max-width:540px}}@media (min-width:768px){.container,.container-md,.container-sm{max-width:720px}}@media (min-width:992px){.container,.container-lg,.container-md,.container-sm{max-width:960px}}@media (min-width:1200px){.container,.container-lg,.container-md,.container-sm,.container-xl{max-width:1140px}}.row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-auto,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-auto,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-auto,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-auto,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9,.col-xl-auto{position:relative;width:100%;padding-right:15px;padding-left:15px}.col{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-first{-ms-flex-order:-1;order:-1}.order-last{-ms-flex-order:13;order:13}.order-0{-ms-flex-order:0;order:0}.order-1{-ms-flex-order:1;order:1}.order-2{-ms-flex-order:2;order:2}.order-3{-ms-flex-order:3;order:3}.order-4{-ms-flex-order:4;order:4}.order-5{-ms-flex-order:5;order:5}.order-6{-ms-flex-order:6;order:6}.order-7{-ms-flex-order:7;order:7}.order-8{-ms-flex-order:8;order:8}.order-9{-ms-flex-order:9;order:9}.order-10{-ms-flex-order:10;order:10}.order-11{-ms-flex-order:11;order:11}.order-12{-ms-flex-order:12;order:12}.offset-1{margin-left:8.333333%}.offset-2{margin-left:16.666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.333333%}.offset-5{margin-left:41.666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.333333%}.offset-8{margin-left:66.666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.333333%}.offset-11{margin-left:91.666667%}@media (min-width:576px){.col-sm{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-sm-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-sm-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-sm-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-sm-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-sm-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-sm-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-sm-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-sm-first{-ms-flex-order:-1;order:-1}.order-sm-last{-ms-flex-order:13;order:13}.order-sm-0{-ms-flex-order:0;order:0}.order-sm-1{-ms-flex-order:1;order:1}.order-sm-2{-ms-flex-order:2;order:2}.order-sm-3{-ms-flex-order:3;order:3}.order-sm-4{-ms-flex-order:4;order:4}.order-sm-5{-ms-flex-order:5;order:5}.order-sm-6{-ms-flex-order:6;order:6}.order-sm-7{-ms-flex-order:7;order:7}.order-sm-8{-ms-flex-order:8;order:8}.order-sm-9{-ms-flex-order:9;order:9}.order-sm-10{-ms-flex-order:10;order:10}.order-sm-11{-ms-flex-order:11;order:11}.order-sm-12{-ms-flex-order:12;order:12}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.333333%}.offset-sm-2{margin-left:16.666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.333333%}.offset-sm-5{margin-left:41.666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.333333%}.offset-sm-8{margin-left:66.666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.333333%}.offset-sm-11{margin-left:91.666667%}}@media (min-width:768px){.col-md{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-md-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-md-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-md-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-md-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-md-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-md-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-md-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-md-first{-ms-flex-order:-1;order:-1}.order-md-last{-ms-flex-order:13;order:13}.order-md-0{-ms-flex-order:0;order:0}.order-md-1{-ms-flex-order:1;order:1}.order-md-2{-ms-flex-order:2;order:2}.order-md-3{-ms-flex-order:3;order:3}.order-md-4{-ms-flex-order:4;order:4}.order-md-5{-ms-flex-order:5;order:5}.order-md-6{-ms-flex-order:6;order:6}.order-md-7{-ms-flex-order:7;order:7}.order-md-8{-ms-flex-order:8;order:8}.order-md-9{-ms-flex-order:9;order:9}.order-md-10{-ms-flex-order:10;order:10}.order-md-11{-ms-flex-order:11;order:11}.order-md-12{-ms-flex-order:12;order:12}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.333333%}.offset-md-2{margin-left:16.666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.333333%}.offset-md-5{margin-left:41.666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.333333%}.offset-md-8{margin-left:66.666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.333333%}.offset-md-11{margin-left:91.666667%}}@media (min-width:992px){.col-lg{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-lg-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-lg-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-lg-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-lg-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-lg-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-lg-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-lg-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-lg-first{-ms-flex-order:-1;order:-1}.order-lg-last{-ms-flex-order:13;order:13}.order-lg-0{-ms-flex-order:0;order:0}.order-lg-1{-ms-flex-order:1;order:1}.order-lg-2{-ms-flex-order:2;order:2}.order-lg-3{-ms-flex-order:3;order:3}.order-lg-4{-ms-flex-order:4;order:4}.order-lg-5{-ms-flex-order:5;order:5}.order-lg-6{-ms-flex-order:6;order:6}.order-lg-7{-ms-flex-order:7;order:7}.order-lg-8{-ms-flex-order:8;order:8}.order-lg-9{-ms-flex-order:9;order:9}.order-lg-10{-ms-flex-order:10;order:10}.order-lg-11{-ms-flex-order:11;order:11}.order-lg-12{-ms-flex-order:12;order:12}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.333333%}.offset-lg-2{margin-left:16.666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.333333%}.offset-lg-5{margin-left:41.666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.333333%}.offset-lg-8{margin-left:66.666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.333333%}.offset-lg-11{margin-left:91.666667%}}@media (min-width:1200px){.col-xl{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-xl-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-xl-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-xl-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-xl-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-xl-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-xl-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-xl-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-xl-first{-ms-flex-order:-1;order:-1}.order-xl-last{-ms-flex-order:13;order:13}.order-xl-0{-ms-flex-order:0;order:0}.order-xl-1{-ms-flex-order:1;order:1}.order-xl-2{-ms-flex-order:2;order:2}.order-xl-3{-ms-flex-order:3;order:3}.order-xl-4{-ms-flex-order:4;order:4}.order-xl-5{-ms-flex-order:5;order:5}.order-xl-6{-ms-flex-order:6;order:6}.order-xl-7{-ms-flex-order:7;order:7}.order-xl-8{-ms-flex-order:8;order:8}.order-xl-9{-ms-flex-order:9;order:9}.order-xl-10{-ms-flex-order:10;order:10}.order-xl-11{-ms-flex-order:11;order:11}.order-xl-12{-ms-flex-order:12;order:12}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.333333%}.offset-xl-2{margin-left:16.666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.333333%}.offset-xl-5{margin-left:41.666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.333333%}.offset-xl-8{margin-left:66.666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.333333%}.offset-xl-11{margin-left:91.666667%}}.table{width:100%;margin-bottom:1rem;color:#212529}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #dee2e6}.table thead th{vertical-align:bottom;border-bottom:2px solid #dee2e6}.table tbody+tbody{border-top:2px solid #dee2e6}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #dee2e6}.table-bordered td,.table-bordered th{border:1px solid #dee2e6}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-borderless tbody+tbody,.table-borderless td,.table-borderless th,.table-borderless thead th{border:0}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{color:#212529;background-color:rgba(0,0,0,.075)}.table-primary,.table-primary>td,.table-primary>th{background-color:#b8daff}.table-primary tbody+tbody,.table-primary td,.table-primary th,.table-primary thead th{border-color:#7abaff}.table-hover .table-primary:hover{background-color:#9fcdff}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#9fcdff}.table-secondary,.table-secondary>td,.table-secondary>th{background-color:#d6d8db}.table-secondary tbody+tbody,.table-secondary td,.table-secondary th,.table-secondary thead th{border-color:#b3b7bb}.table-hover .table-secondary:hover{background-color:#c8cbcf}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#c8cbcf}.table-success,.table-success>td,.table-success>th{background-color:#c3e6cb}.table-success tbody+tbody,.table-success td,.table-success th,.table-success thead th{border-color:#8fd19e}.table-hover .table-success:hover{background-color:#b1dfbb}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#b1dfbb}.table-info,.table-info>td,.table-info>th{background-color:#bee5eb}.table-info tbody+tbody,.table-info td,.table-info th,.table-info thead th{border-color:#86cfda}.table-hover .table-info:hover{background-color:#abdde5}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#abdde5}.table-warning,.table-warning>td,.table-warning>th{background-color:#ffeeba}.table-warning tbody+tbody,.table-warning td,.table-warning th,.table-warning thead th{border-color:#ffdf7e}.table-hover .table-warning:hover{background-color:#ffe8a1}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#ffe8a1}.table-danger,.table-danger>td,.table-danger>th{background-color:#f5c6cb}.table-danger tbody+tbody,.table-danger td,.table-danger th,.table-danger thead th{border-color:#ed969e}.table-hover .table-danger:hover{background-color:#f1b0b7}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#f1b0b7}.table-light,.table-light>td,.table-light>th{background-color:#fdfdfe}.table-light tbody+tbody,.table-light td,.table-light th,.table-light thead th{border-color:#fbfcfc}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>td,.table-dark>th{background-color:#c6c8ca}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#95999c}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.table .thead-dark th{color:#fff;background-color:#343a40;border-color:#454d55}.table .thead-light th{color:#495057;background-color:#e9ecef;border-color:#dee2e6}.table-dark{color:#fff;background-color:#343a40}.table-dark td,.table-dark th,.table-dark thead th{border-color:#454d55}.table-dark.table-bordered{border:0}.table-dark.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,.05)}.table-dark.table-hover tbody tr:hover{color:#fff;background-color:rgba(255,255,255,.075)}@media (max-width:575.98px){.table-responsive-sm{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-sm>.table-bordered{border:0}}@media (max-width:767.98px){.table-responsive-md{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-md>.table-bordered{border:0}}@media (max-width:991.98px){.table-responsive-lg{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-lg>.table-bordered{border:0}}@media (max-width:1199.98px){.table-responsive-xl{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-xl>.table-bordered{border:0}}.table-responsive{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive>.table-bordered{border:0}.form-control{display:block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.form-control:focus{color:#495057;background-color:#fff;border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.form-control::-webkit-input-placeholder{color:#6c757d;opacity:1}.form-control::-moz-placeholder{color:#6c757d;opacity:1}.form-control:-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}input[type=date].form-control,input[type=datetime-local].form-control,input[type=month].form-control,input[type=time].form-control{-webkit-appearance:none;-moz-appearance:none;appearance:none}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block;width:100%}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem;line-height:1.5}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem;line-height:1.5}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;font-size:1rem;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.form-control-lg{height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}select.form-control[multiple],select.form-control[size]{height:auto}textarea.form-control{height:auto}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*=col-]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;padding-left:1.25rem}.form-check-input{position:absolute;margin-top:.3rem;margin-left:-1.25rem}.form-check-input:disabled~.form-check-label,.form-check-input[disabled]~.form-check-label{color:#6c757d}.form-check-label{margin-bottom:0}.form-check-inline{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-align:center;align-items:center;padding-left:0;margin-right:.75rem}.form-check-inline .form-check-input{position:static;margin-top:0;margin-right:.3125rem;margin-left:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#28a745}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(40,167,69,.9);border-radius:.25rem}.is-valid~.valid-feedback,.is-valid~.valid-tooltip,.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip{display:block}.form-control.is-valid,.was-validated .form-control:valid{border-color:#28a745;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-valid,.was-validated .custom-select:valid{border-color:#28a745;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-valid:focus,.was-validated .custom-select:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:#28a745}.form-check-input.is-valid~.valid-feedback,.form-check-input.is-valid~.valid-tooltip,.was-validated .form-check-input:valid~.valid-feedback,.was-validated .form-check-input:valid~.valid-tooltip{display:block}.custom-control-input.is-valid~.custom-control-label,.was-validated .custom-control-input:valid~.custom-control-label{color:#28a745}.custom-control-input.is-valid~.custom-control-label::before,.was-validated .custom-control-input:valid~.custom-control-label::before{border-color:#28a745}.custom-control-input.is-valid:checked~.custom-control-label::before,.was-validated .custom-control-input:valid:checked~.custom-control-label::before{border-color:#34ce57;background-color:#34ce57}.custom-control-input.is-valid:focus~.custom-control-label::before,.was-validated .custom-control-input:valid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-control-input.is-valid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:valid:focus:not(:checked)~.custom-control-label::before{border-color:#28a745}.custom-file-input.is-valid~.custom-file-label,.was-validated .custom-file-input:valid~.custom-file-label{border-color:#28a745}.custom-file-input.is-valid:focus~.custom-file-label,.was-validated .custom-file-input:valid:focus~.custom-file-label{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(220,53,69,.9);border-radius:.25rem}.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip,.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip{display:block}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:#dc3545;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-invalid,.was-validated .custom-select:invalid{border-color:#dc3545;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-invalid:focus,.was-validated .custom-select:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:#dc3545}.form-check-input.is-invalid~.invalid-feedback,.form-check-input.is-invalid~.invalid-tooltip,.was-validated .form-check-input:invalid~.invalid-feedback,.was-validated .form-check-input:invalid~.invalid-tooltip{display:block}.custom-control-input.is-invalid~.custom-control-label,.was-validated .custom-control-input:invalid~.custom-control-label{color:#dc3545}.custom-control-input.is-invalid~.custom-control-label::before,.was-validated .custom-control-input:invalid~.custom-control-label::before{border-color:#dc3545}.custom-control-input.is-invalid:checked~.custom-control-label::before,.was-validated .custom-control-input:invalid:checked~.custom-control-label::before{border-color:#e4606d;background-color:#e4606d}.custom-control-input.is-invalid:focus~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.custom-control-input.is-invalid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus:not(:checked)~.custom-control-label::before{border-color:#dc3545}.custom-file-input.is-invalid~.custom-file-label,.was-validated .custom-file-input:invalid~.custom-file-label{border-color:#dc3545}.custom-file-input.is-invalid:focus~.custom-file-label,.was-validated .custom-file-input:invalid:focus~.custom-file-label{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-inline{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-ms-flexbox;display:flex;-ms-flex:0 0 auto;flex:0 0 auto;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .custom-select,.form-inline .input-group{width:auto}.form-inline .form-check{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:auto;padding-left:0}.form-inline .form-check-input{position:relative;-ms-flex-negative:0;flex-shrink:0;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center}.form-inline .custom-control-label{margin-bottom:0}}.btn{display:inline-block;font-weight:400;color:#212529;text-align:center;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;line-height:1.5;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:#212529;text-decoration:none}.btn.focus,.btn:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.btn.disabled,.btn:disabled{opacity:.65}.btn:not(:disabled):not(.disabled){cursor:pointer}a.btn.disabled,fieldset:disabled a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:hover{color:#fff;background-color:#0069d9;border-color:#0062cc}.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#0069d9;border-color:#0062cc;box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:not(:disabled):not(.disabled).active,.btn-primary:not(:disabled):not(.disabled):active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0062cc;border-color:#005cbf}.btn-primary:not(:disabled):not(.disabled).active:focus,.btn-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-secondary{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:hover{color:#fff;background-color:#5a6268;border-color:#545b62}.btn-secondary.focus,.btn-secondary:focus{color:#fff;background-color:#5a6268;border-color:#545b62;box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:not(:disabled):not(.disabled).active,.btn-secondary:not(:disabled):not(.disabled):active,.show>.btn-secondary.dropdown-toggle{color:#fff;background-color:#545b62;border-color:#4e555b}.btn-secondary:not(:disabled):not(.disabled).active:focus,.btn-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-success{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:hover{color:#fff;background-color:#218838;border-color:#1e7e34}.btn-success.focus,.btn-success:focus{color:#fff;background-color:#218838;border-color:#1e7e34;box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:not(:disabled):not(.disabled).active,.btn-success:not(:disabled):not(.disabled):active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#1e7e34;border-color:#1c7430}.btn-success:not(:disabled):not(.disabled).active:focus,.btn-success:not(:disabled):not(.disabled):active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-info{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:hover{color:#fff;background-color:#138496;border-color:#117a8b}.btn-info.focus,.btn-info:focus{color:#fff;background-color:#138496;border-color:#117a8b;box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-info.disabled,.btn-info:disabled{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:not(:disabled):not(.disabled).active,.btn-info:not(:disabled):not(.disabled):active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#117a8b;border-color:#10707f}.btn-info:not(:disabled):not(.disabled).active:focus,.btn-info:not(:disabled):not(.disabled):active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-warning{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#212529;background-color:#e0a800;border-color:#d39e00}.btn-warning.focus,.btn-warning:focus{color:#212529;background-color:#e0a800;border-color:#d39e00;box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-warning.disabled,.btn-warning:disabled{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:not(:disabled):not(.disabled).active,.btn-warning:not(:disabled):not(.disabled):active,.show>.btn-warning.dropdown-toggle{color:#212529;background-color:#d39e00;border-color:#c69500}.btn-warning:not(:disabled):not(.disabled).active:focus,.btn-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#c82333;border-color:#bd2130}.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c82333;border-color:#bd2130;box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:not(:disabled):not(.disabled).active,.btn-danger:not(:disabled):not(.disabled):active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#bd2130;border-color:#b21f2d}.btn-danger:not(:disabled):not(.disabled).active:focus,.btn-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-light{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#212529;background-color:#e2e6ea;border-color:#dae0e5}.btn-light.focus,.btn-light:focus{color:#212529;background-color:#e2e6ea;border-color:#dae0e5;box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-light.disabled,.btn-light:disabled{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:not(:disabled):not(.disabled).active,.btn-light:not(:disabled):not(.disabled):active,.show>.btn-light.dropdown-toggle{color:#212529;background-color:#dae0e5;border-color:#d3d9df}.btn-light:not(:disabled):not(.disabled).active:focus,.btn-light:not(:disabled):not(.disabled):active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark.focus,.btn-dark:focus{color:#fff;background-color:#23272b;border-color:#1d2124;box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:not(:disabled):not(.disabled).active,.btn-dark:not(:disabled):not(.disabled):active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1d2124;border-color:#171a1d}.btn-dark:not(:disabled):not(.disabled).active:focus,.btn-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-outline-primary{color:#007bff;border-color:#007bff}.btn-outline-primary:hover{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary.focus,.btn-outline-primary:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#007bff;background-color:transparent}.btn-outline-primary:not(:disabled):not(.disabled).active,.btn-outline-primary:not(:disabled):not(.disabled):active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary:not(:disabled):not(.disabled).active:focus,.btn-outline-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-secondary{color:#6c757d;border-color:#6c757d}.btn-outline-secondary:hover{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary.focus,.btn-outline-secondary:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#6c757d;background-color:transparent}.btn-outline-secondary:not(:disabled):not(.disabled).active,.btn-outline-secondary:not(:disabled):not(.disabled):active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary:not(:disabled):not(.disabled).active:focus,.btn-outline-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-success{color:#28a745;border-color:#28a745}.btn-outline-success:hover{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success.focus,.btn-outline-success:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#28a745;background-color:transparent}.btn-outline-success:not(:disabled):not(.disabled).active,.btn-outline-success:not(:disabled):not(.disabled):active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success:not(:disabled):not(.disabled).active:focus,.btn-outline-success:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-info{color:#17a2b8;border-color:#17a2b8}.btn-outline-info:hover{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info.focus,.btn-outline-info:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#17a2b8;background-color:transparent}.btn-outline-info:not(:disabled):not(.disabled).active,.btn-outline-info:not(:disabled):not(.disabled):active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info:not(:disabled):not(.disabled).active:focus,.btn-outline-info:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-warning{color:#ffc107;border-color:#ffc107}.btn-outline-warning:hover{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning.focus,.btn-outline-warning:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-warning:not(:disabled):not(.disabled).active,.btn-outline-warning:not(:disabled):not(.disabled):active,.show>.btn-outline-warning.dropdown-toggle{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning:not(:disabled):not(.disabled).active:focus,.btn-outline-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-danger{color:#dc3545;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger.focus,.btn-outline-danger:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-danger:not(:disabled):not(.disabled).active,.btn-outline-danger:not(:disabled):not(.disabled):active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger:not(:disabled):not(.disabled).active:focus,.btn-outline-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light.focus,.btn-outline-light:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light:not(:disabled):not(.disabled).active,.btn-outline-light:not(:disabled):not(.disabled):active,.show>.btn-outline-light.dropdown-toggle{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:not(:disabled):not(.disabled).active:focus,.btn-outline-light:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-dark{color:#343a40;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark.focus,.btn-outline-dark:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark:not(:disabled):not(.disabled).active,.btn-outline-dark:not(:disabled):not(.disabled):active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:not(:disabled):not(.disabled).active:focus,.btn-outline-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-link{font-weight:400;color:#007bff;text-decoration:none}.btn-link:hover{color:#0056b3;text-decoration:underline}.btn-link.focus,.btn-link:focus{text-decoration:underline}.btn-link.disabled,.btn-link:disabled{color:#6c757d;pointer-events:none}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{position:relative;height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.dropdown,.dropleft,.dropright,.dropup{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-menu-left{right:auto;left:0}.dropdown-menu-right{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-left{right:auto;left:0}.dropdown-menu-sm-right{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-left{right:auto;left:0}.dropdown-menu-md-right{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-left{right:auto;left:0}.dropdown-menu-lg-right{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-left{right:auto;left:0}.dropdown-menu-xl-right{right:0;left:auto}}.dropup .dropdown-menu{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-menu{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropright .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropright .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-toggle::after{vertical-align:0}.dropleft .dropdown-menu{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropleft .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropleft .dropdown-toggle::after{display:none}.dropleft .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropleft .dropdown-toggle:empty::after{margin-left:0}.dropleft .dropdown-toggle::before{vertical-align:0}.dropdown-menu[x-placement^=bottom],.dropdown-menu[x-placement^=left],.dropdown-menu[x-placement^=right],.dropdown-menu[x-placement^=top]{right:auto;bottom:auto}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:.25rem 1.5rem;clear:both;font-weight:400;color:#212529;text-align:inherit;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#16181b;text-decoration:none;background-color:#f8f9fa}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#007bff}.dropdown-item.disabled,.dropdown-item:disabled{color:#6c757d;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1.5rem;color:#212529}.btn-group,.btn-group-vertical{position:relative;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-ms-flex:1 1 auto;flex:1 1 auto}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:1}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:1}.btn-toolbar{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn-group:not(:first-child),.btn-group>.btn:not(:first-child){margin-left:-1px}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropright .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropleft .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{-ms-flex-direction:column;flex-direction:column;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:center;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn:not(:first-child){border-top-left-radius:0;border-top-right-radius:0}.btn-group-toggle>.btn,.btn-group-toggle>.btn-group>.btn{margin-bottom:0}.btn-group-toggle>.btn input[type=checkbox],.btn-group-toggle>.btn input[type=radio],.btn-group-toggle>.btn-group>.btn input[type=checkbox],.btn-group-toggle>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:stretch;align-items:stretch;width:100%}.input-group>.custom-file,.input-group>.custom-select,.input-group>.form-control,.input-group>.form-control-plaintext{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;min-width:0;margin-bottom:0}.input-group>.custom-file+.custom-file,.input-group>.custom-file+.custom-select,.input-group>.custom-file+.form-control,.input-group>.custom-select+.custom-file,.input-group>.custom-select+.custom-select,.input-group>.custom-select+.form-control,.input-group>.form-control+.custom-file,.input-group>.form-control+.custom-select,.input-group>.form-control+.form-control,.input-group>.form-control-plaintext+.custom-file,.input-group>.form-control-plaintext+.custom-select,.input-group>.form-control-plaintext+.form-control{margin-left:-1px}.input-group>.custom-file .custom-file-input:focus~.custom-file-label,.input-group>.custom-select:focus,.input-group>.form-control:focus{z-index:3}.input-group>.custom-file .custom-file-input:focus{z-index:4}.input-group>.custom-select:not(:last-child),.input-group>.form-control:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-select:not(:first-child),.input-group>.form-control:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.custom-file{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center}.input-group>.custom-file:not(:last-child) .custom-file-label,.input-group>.custom-file:not(:last-child) .custom-file-label::after{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-file:not(:first-child) .custom-file-label{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-append,.input-group-prepend{display:-ms-flexbox;display:flex}.input-group-append .btn,.input-group-prepend .btn{position:relative;z-index:2}.input-group-append .btn:focus,.input-group-prepend .btn:focus{z-index:3}.input-group-append .btn+.btn,.input-group-append .btn+.input-group-text,.input-group-append .input-group-text+.btn,.input-group-append .input-group-text+.input-group-text,.input-group-prepend .btn+.btn,.input-group-prepend .btn+.input-group-text,.input-group-prepend .input-group-text+.btn,.input-group-prepend .input-group-text+.input-group-text{margin-left:-1px}.input-group-prepend{margin-right:-1px}.input-group-append{margin-left:-1px}.input-group-text{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.375rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-text input[type=checkbox],.input-group-text input[type=radio]{margin-top:0}.input-group-lg>.custom-select,.input-group-lg>.form-control:not(textarea){height:calc(1.5em + 1rem + 2px)}.input-group-lg>.custom-select,.input-group-lg>.form-control,.input-group-lg>.input-group-append>.btn,.input-group-lg>.input-group-append>.input-group-text,.input-group-lg>.input-group-prepend>.btn,.input-group-lg>.input-group-prepend>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.input-group-sm>.custom-select,.input-group-sm>.form-control:not(textarea){height:calc(1.5em + .5rem + 2px)}.input-group-sm>.custom-select,.input-group-sm>.form-control,.input-group-sm>.input-group-append>.btn,.input-group-sm>.input-group-append>.input-group-text,.input-group-sm>.input-group-prepend>.btn,.input-group-sm>.input-group-prepend>.input-group-text{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.input-group-lg>.custom-select,.input-group-sm>.custom-select{padding-right:1.75rem}.input-group>.input-group-append:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group>.input-group-append:last-child>.input-group-text:not(:last-child),.input-group>.input-group-append:not(:last-child)>.btn,.input-group>.input-group-append:not(:last-child)>.input-group-text,.input-group>.input-group-prepend>.btn,.input-group>.input-group-prepend>.input-group-text{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.input-group-append>.btn,.input-group>.input-group-append>.input-group-text,.input-group>.input-group-prepend:first-child>.btn:not(:first-child),.input-group>.input-group-prepend:first-child>.input-group-text:not(:first-child),.input-group>.input-group-prepend:not(:first-child)>.btn,.input-group>.input-group-prepend:not(:first-child)>.input-group-text{border-top-left-radius:0;border-bottom-left-radius:0}.custom-control{position:relative;display:block;min-height:1.5rem;padding-left:1.5rem}.custom-control-inline{display:-ms-inline-flexbox;display:inline-flex;margin-right:1rem}.custom-control-input{position:absolute;left:0;z-index:-1;width:1rem;height:1.25rem;opacity:0}.custom-control-input:checked~.custom-control-label::before{color:#fff;border-color:#007bff;background-color:#007bff}.custom-control-input:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-control-input:focus:not(:checked)~.custom-control-label::before{border-color:#80bdff}.custom-control-input:not(:disabled):active~.custom-control-label::before{color:#fff;background-color:#b3d7ff;border-color:#b3d7ff}.custom-control-input:disabled~.custom-control-label,.custom-control-input[disabled]~.custom-control-label{color:#6c757d}.custom-control-input:disabled~.custom-control-label::before,.custom-control-input[disabled]~.custom-control-label::before{background-color:#e9ecef}.custom-control-label{position:relative;margin-bottom:0;vertical-align:top}.custom-control-label::before{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;pointer-events:none;content:"";background-color:#fff;border:#adb5bd solid 1px}.custom-control-label::after{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;content:"";background:no-repeat 50%/50% 50%}.custom-checkbox .custom-control-label::before{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26l2.974 2.99L8 2.193z'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::before{border-color:#007bff;background-color:#007bff}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='4' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-checkbox .custom-control-input:disabled:indeterminate~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-radio .custom-control-label::before{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.custom-radio .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-switch{padding-left:2.25rem}.custom-switch .custom-control-label::before{left:-2.25rem;width:1.75rem;pointer-events:all;border-radius:.5rem}.custom-switch .custom-control-label::after{top:calc(.25rem + 2px);left:calc(-2.25rem + 2px);width:calc(1rem - 4px);height:calc(1rem - 4px);background-color:#adb5bd;border-radius:.5rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-switch .custom-control-label::after{transition:none}}.custom-switch .custom-control-input:checked~.custom-control-label::after{background-color:#fff;-webkit-transform:translateX(.75rem);transform:translateX(.75rem)}.custom-switch .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-select{display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem 1.75rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;vertical-align:middle;background:#fff url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px;border:1px solid #ced4da;border-radius:.25rem;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-select:focus{border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select[multiple],.custom-select[size]:not([size="1"]){height:auto;padding-right:.75rem;background-image:none}.custom-select:disabled{color:#6c757d;background-color:#e9ecef}.custom-select::-ms-expand{display:none}.custom-select:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.custom-select-sm{height:calc(1.5em + .5rem + 2px);padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem}.custom-select-lg{height:calc(1.5em + 1rem + 2px);padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}.custom-file{position:relative;display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);margin-bottom:0}.custom-file-input{position:relative;z-index:2;width:100%;height:calc(1.5em + .75rem + 2px);margin:0;opacity:0}.custom-file-input:focus~.custom-file-label{border-color:#80bdff;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-file-input:disabled~.custom-file-label,.custom-file-input[disabled]~.custom-file-label{background-color:#e9ecef}.custom-file-input:lang(en)~.custom-file-label::after{content:"Browse"}.custom-file-input~.custom-file-label[data-browse]::after{content:attr(data-browse)}.custom-file-label{position:absolute;top:0;right:0;left:0;z-index:1;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem}.custom-file-label::after{position:absolute;top:0;right:0;bottom:0;z-index:3;display:block;height:calc(1.5em + .75rem);padding:.375rem .75rem;line-height:1.5;color:#495057;content:"Browse";background-color:#e9ecef;border-left:inherit;border-radius:0 .25rem .25rem 0}.custom-range{width:100%;height:1.4rem;padding:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-range:focus{outline:0}.custom-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-ms-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range::-moz-focus-outer{border:0}.custom-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#007bff;border:0;border-radius:1rem;-webkit-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.custom-range::-webkit-slider-thumb:active{background-color:#b3d7ff}.custom-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#007bff;border:0;border-radius:1rem;-moz-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-moz-range-thumb{-moz-transition:none;transition:none}}.custom-range::-moz-range-thumb:active{background-color:#b3d7ff}.custom-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-ms-thumb{width:1rem;height:1rem;margin-top:0;margin-right:.2rem;margin-left:.2rem;background-color:#007bff;border:0;border-radius:1rem;-ms-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-ms-thumb{-ms-transition:none;transition:none}}.custom-range::-ms-thumb:active{background-color:#b3d7ff}.custom-range::-ms-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:transparent;border-color:transparent;border-width:.5rem}.custom-range::-ms-fill-lower{background-color:#dee2e6;border-radius:1rem}.custom-range::-ms-fill-upper{margin-right:15px;background-color:#dee2e6;border-radius:1rem}.custom-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.custom-range:disabled::-webkit-slider-runnable-track{cursor:default}.custom-range:disabled::-moz-range-thumb{background-color:#adb5bd}.custom-range:disabled::-moz-range-track{cursor:default}.custom-range:disabled::-ms-thumb{background-color:#adb5bd}.custom-control-label::before,.custom-file-label,.custom-select{transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-control-label::before,.custom-file-label,.custom-select{transition:none}}.nav{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #dee2e6}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#007bff}.nav-fill .nav-item{-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between;padding:.5rem 1rem}.navbar .container,.navbar .container-fluid,.navbar .container-lg,.navbar .container-md,.navbar .container-sm,.navbar .container-xl{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{-ms-flex-preferred-size:100%;flex-basis:100%;-ms-flex-positive:1;flex-grow:1;-ms-flex-align:center;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width:575.98px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-expand-sm{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-sm .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width:767.98px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-expand-md{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-md .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width:991.98px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-expand-lg{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-lg .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width:1199.98px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-expand-xl{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-xl .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.5);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%280, 0, 0, 0.5%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-light .navbar-text a{color:rgba(0,0,0,.9)}.navbar-light .navbar-text a:focus,.navbar-light .navbar-text a:hover{color:rgba(0,0,0,.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.5);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.5%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,.5)}.navbar-dark .navbar-text a{color:#fff}.navbar-dark .navbar-text a:focus,.navbar-dark .navbar-text a:hover{color:#fff}.card{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card>.list-group:last-child{border-bottom-width:0;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-body{-ms-flex:1 1 auto;flex:1 1 auto;min-height:1px;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-header+.list-group .list-group-item:first-child{border-top:0}.card-footer{padding:.75rem 1.25rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img,.card-img-bottom,.card-img-top{-ms-flex-negative:0;flex-shrink:0;width:100%}.card-img,.card-img-top{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-deck .card{margin-bottom:15px}@media (min-width:576px){.card-deck{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{-ms-flex:1 0 0%;flex:1 0 0%;margin-right:15px;margin-bottom:0;margin-left:15px}}.card-group>.card{margin-bottom:15px}@media (min-width:576px){.card-group{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group>.card{-ms-flex:1 0 0%;flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.card-columns .card{margin-bottom:.75rem}@media (min-width:576px){.card-columns{-webkit-column-count:3;-moz-column-count:3;column-count:3;-webkit-column-gap:1.25rem;-moz-column-gap:1.25rem;column-gap:1.25rem;orphans:1;widows:1}.card-columns .card{display:inline-block;width:100%}}.accordion>.card{overflow:hidden}.accordion>.card:not(:last-of-type){border-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.accordion>.card:not(:first-of-type){border-top-left-radius:0;border-top-right-radius:0}.accordion>.card>.card-header{border-radius:0;margin-bottom:-1px}.breadcrumb{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:.25rem}.breadcrumb-item{display:-ms-flexbox;display:flex}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;color:#6c757d;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#6c757d}.pagination{display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#007bff;background-color:#fff;border:1px solid #dee2e6}.page-link:hover{z-index:2;color:#0056b3;text-decoration:none;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:3;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.page-item.active .page-link{z-index:3;color:#fff;background-color:#007bff;border-color:#007bff}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;cursor:auto;background-color:#fff;border-color:#dee2e6}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.badge{transition:none}}a.badge:focus,a.badge:hover{text-decoration:none}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#007bff}a.badge-primary:focus,a.badge-primary:hover{color:#fff;background-color:#0062cc}a.badge-primary.focus,a.badge-primary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.badge-secondary{color:#fff;background-color:#6c757d}a.badge-secondary:focus,a.badge-secondary:hover{color:#fff;background-color:#545b62}a.badge-secondary.focus,a.badge-secondary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.badge-success{color:#fff;background-color:#28a745}a.badge-success:focus,a.badge-success:hover{color:#fff;background-color:#1e7e34}a.badge-success.focus,a.badge-success:focus{outline:0;box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.badge-info{color:#fff;background-color:#17a2b8}a.badge-info:focus,a.badge-info:hover{color:#fff;background-color:#117a8b}a.badge-info.focus,a.badge-info:focus{outline:0;box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.badge-warning{color:#212529;background-color:#ffc107}a.badge-warning:focus,a.badge-warning:hover{color:#212529;background-color:#d39e00}a.badge-warning.focus,a.badge-warning:focus{outline:0;box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.badge-danger{color:#fff;background-color:#dc3545}a.badge-danger:focus,a.badge-danger:hover{color:#fff;background-color:#bd2130}a.badge-danger.focus,a.badge-danger:focus{outline:0;box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.badge-light{color:#212529;background-color:#f8f9fa}a.badge-light:focus,a.badge-light:hover{color:#212529;background-color:#dae0e5}a.badge-light.focus,a.badge-light:focus{outline:0;box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.badge-dark{color:#fff;background-color:#343a40}a.badge-dark:focus,a.badge-dark:hover{color:#fff;background-color:#1d2124}a.badge-dark.focus,a.badge-dark:focus{outline:0;box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{position:relative;padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:4rem}.alert-dismissible .close{position:absolute;top:0;right:0;padding:.75rem 1.25rem;color:inherit}.alert-primary{color:#004085;background-color:#cce5ff;border-color:#b8daff}.alert-primary hr{border-top-color:#9fcdff}.alert-primary .alert-link{color:#002752}.alert-secondary{color:#383d41;background-color:#e2e3e5;border-color:#d6d8db}.alert-secondary hr{border-top-color:#c8cbcf}.alert-secondary .alert-link{color:#202326}.alert-success{color:#155724;background-color:#d4edda;border-color:#c3e6cb}.alert-success hr{border-top-color:#b1dfbb}.alert-success .alert-link{color:#0b2e13}.alert-info{color:#0c5460;background-color:#d1ecf1;border-color:#bee5eb}.alert-info hr{border-top-color:#abdde5}.alert-info .alert-link{color:#062c33}.alert-warning{color:#856404;background-color:#fff3cd;border-color:#ffeeba}.alert-warning hr{border-top-color:#ffe8a1}.alert-warning .alert-link{color:#533f03}.alert-danger{color:#721c24;background-color:#f8d7da;border-color:#f5c6cb}.alert-danger hr{border-top-color:#f1b0b7}.alert-danger .alert-link{color:#491217}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-ms-flexbox;display:flex;height:1rem;overflow:hidden;line-height:0;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;overflow:hidden;color:#fff;text-align:center;white-space:nowrap;background-color:#007bff;transition:width .6s ease}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}@media (prefers-reduced-motion:reduce){.progress-bar-animated{-webkit-animation:none;animation:none}}.media{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start}.media-body{-ms-flex:1;flex:1}.list-group{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;border-radius:.25rem}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.75rem 1.25rem;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.list-group-item:last-child{border-bottom-right-radius:inherit;border-bottom-left-radius:inherit}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#007bff;border-color:#007bff}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:-1px;border-top-width:1px}.list-group-horizontal{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}@media (min-width:576px){.list-group-horizontal-sm{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-sm>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:768px){.list-group-horizontal-md{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-md>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:992px){.list-group-horizontal-lg{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-lg>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1200px){.list-group-horizontal-xl{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-xl>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}.list-group-flush{border-radius:0}.list-group-flush>.list-group-item{border-width:0 0 1px}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{color:#004085;background-color:#b8daff}.list-group-item-primary.list-group-item-action:focus,.list-group-item-primary.list-group-item-action:hover{color:#004085;background-color:#9fcdff}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#004085;border-color:#004085}.list-group-item-secondary{color:#383d41;background-color:#d6d8db}.list-group-item-secondary.list-group-item-action:focus,.list-group-item-secondary.list-group-item-action:hover{color:#383d41;background-color:#c8cbcf}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#383d41;border-color:#383d41}.list-group-item-success{color:#155724;background-color:#c3e6cb}.list-group-item-success.list-group-item-action:focus,.list-group-item-success.list-group-item-action:hover{color:#155724;background-color:#b1dfbb}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#155724;border-color:#155724}.list-group-item-info{color:#0c5460;background-color:#bee5eb}.list-group-item-info.list-group-item-action:focus,.list-group-item-info.list-group-item-action:hover{color:#0c5460;background-color:#abdde5}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#0c5460;border-color:#0c5460}.list-group-item-warning{color:#856404;background-color:#ffeeba}.list-group-item-warning.list-group-item-action:focus,.list-group-item-warning.list-group-item-action:hover{color:#856404;background-color:#ffe8a1}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#856404;border-color:#856404}.list-group-item-danger{color:#721c24;background-color:#f5c6cb}.list-group-item-danger.list-group-item-action:focus,.list-group-item-danger.list-group-item-action:hover{color:#721c24;background-color:#f1b0b7}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#721c24;border-color:#721c24}.list-group-item-light{color:#818182;background-color:#fdfdfe}.list-group-item-light.list-group-item-action:focus,.list-group-item-light.list-group-item-action:hover{color:#818182;background-color:#ececf6}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}.list-group-item-dark.list-group-item-action:focus,.list-group-item-dark.list-group-item-action:hover{color:#1b1e21;background-color:#b9bbbe}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:hover{color:#000;text-decoration:none}.close:not(:disabled):not(.disabled):focus,.close:not(:disabled):not(.disabled):hover{opacity:.75}button.close{padding:0;background-color:transparent;border:0}a.close.disabled{pointer-events:none}.toast{max-width:350px;overflow:hidden;font-size:.875rem;background-color:rgba(255,255,255,.85);background-clip:padding-box;border:1px solid rgba(0,0,0,.1);box-shadow:0 .25rem .75rem rgba(0,0,0,.1);-webkit-backdrop-filter:blur(10px);backdrop-filter:blur(10px);opacity:0;border-radius:.25rem}.toast:not(:last-child){margin-bottom:.75rem}.toast.showing{opacity:1}.toast.show{display:block;opacity:1}.toast.hide{display:none}.toast-header{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.25rem .75rem;color:#6c757d;background-color:rgba(255,255,255,.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,.05)}.toast-body{padding:.75rem}.modal-open{overflow:hidden}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal{position:fixed;top:0;left:0;z-index:1050;display:none;width:100%;height:100%;overflow:hidden;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:-webkit-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out;-webkit-transform:translate(0,-50px);transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{-webkit-transform:none;transform:none}.modal.modal-static .modal-dialog{-webkit-transform:scale(1.02);transform:scale(1.02)}.modal-dialog-scrollable{display:-ms-flexbox;display:flex;max-height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 1rem);overflow:hidden}.modal-dialog-scrollable .modal-footer,.modal-dialog-scrollable .modal-header{-ms-flex-negative:0;flex-shrink:0}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;min-height:calc(100% - 1rem)}.modal-dialog-centered::before{display:block;height:calc(100vh - 1rem);height:-webkit-min-content;height:-moz-min-content;height:min-content;content:""}.modal-dialog-centered.modal-dialog-scrollable{-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;height:100%}.modal-dialog-centered.modal-dialog-scrollable .modal-content{max-height:none}.modal-dialog-centered.modal-dialog-scrollable::before{content:none}.modal-content{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:justify;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.modal-header .close{padding:1rem 1rem;margin:-1rem -1rem -1rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;padding:1rem}.modal-footer{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:end;justify-content:flex-end;padding:.75rem;border-top:1px solid #dee2e6;border-bottom-right-radius:calc(.3rem - 1px);border-bottom-left-radius:calc(.3rem - 1px)}.modal-footer>*{margin:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{max-height:calc(100% - 3.5rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-dialog-centered::before{height:calc(100vh - 3.5rem);height:-webkit-min-content;height:-moz-min-content;height:min-content}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width:1200px){.modal-xl{max-width:1140px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[x-placement^=top],.bs-tooltip-top{padding:.4rem 0}.bs-tooltip-auto[x-placement^=top] .arrow,.bs-tooltip-top .arrow{bottom:0}.bs-tooltip-auto[x-placement^=top] .arrow::before,.bs-tooltip-top .arrow::before{top:0;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-auto[x-placement^=right],.bs-tooltip-right{padding:0 .4rem}.bs-tooltip-auto[x-placement^=right] .arrow,.bs-tooltip-right .arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=right] .arrow::before,.bs-tooltip-right .arrow::before{right:0;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-auto[x-placement^=bottom],.bs-tooltip-bottom{padding:.4rem 0}.bs-tooltip-auto[x-placement^=bottom] .arrow,.bs-tooltip-bottom .arrow{top:0}.bs-tooltip-auto[x-placement^=bottom] .arrow::before,.bs-tooltip-bottom .arrow::before{bottom:0;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-auto[x-placement^=left],.bs-tooltip-left{padding:0 .4rem}.bs-tooltip-auto[x-placement^=left] .arrow,.bs-tooltip-left .arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=left] .arrow::before,.bs-tooltip-left .arrow::before{left:0;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .arrow{position:absolute;display:block;width:1rem;height:.5rem;margin:0 .3rem}.popover .arrow::after,.popover .arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-auto[x-placement^=top],.bs-popover-top{margin-bottom:.5rem}.bs-popover-auto[x-placement^=top]>.arrow,.bs-popover-top>.arrow{bottom:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=top]>.arrow::before,.bs-popover-top>.arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=top]>.arrow::after,.bs-popover-top>.arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-auto[x-placement^=right],.bs-popover-right{margin-left:.5rem}.bs-popover-auto[x-placement^=right]>.arrow,.bs-popover-right>.arrow{left:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=right]>.arrow::before,.bs-popover-right>.arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=right]>.arrow::after,.bs-popover-right>.arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-auto[x-placement^=bottom],.bs-popover-bottom{margin-top:.5rem}.bs-popover-auto[x-placement^=bottom]>.arrow,.bs-popover-bottom>.arrow{top:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=bottom]>.arrow::before,.bs-popover-bottom>.arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=bottom]>.arrow::after,.bs-popover-bottom>.arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-auto[x-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f7f7f7}.bs-popover-auto[x-placement^=left],.bs-popover-left{margin-right:.5rem}.bs-popover-auto[x-placement^=left]>.arrow,.bs-popover-left>.arrow{right:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=left]>.arrow::before,.bs-popover-left>.arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=left]>.arrow::after,.bs-popover-left>.arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:.5rem .75rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{-ms-touch-action:pan-y;touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:-webkit-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-right,.carousel-item-next:not(.carousel-item-left){-webkit-transform:translateX(100%);transform:translateX(100%)}.active.carousel-item-left,.carousel-item-prev:not(.carousel-item-right){-webkit-transform:translateX(-100%);transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;-webkit-transform:none;transform:none}.carousel-fade .carousel-item-next.carousel-item-left,.carousel-fade .carousel-item-prev.carousel-item-right,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:no-repeat 50%/100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5L4.25 4l2.5-2.5L5.25 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5L3.75 4l-2.5 2.5L2.75 8l4-4-4-4z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:15;display:-ms-flexbox;display:flex;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{box-sizing:content-box;-ms-flex:0 1 auto;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators li{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}@-webkit-keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;-webkit-animation:spinner-border .75s linear infinite;animation:spinner-border .75s linear infinite}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@-webkit-keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1;-webkit-transform:none;transform:none}}@keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1;-webkit-transform:none;transform:none}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;background-color:currentColor;border-radius:50%;opacity:0;-webkit-animation:spinner-grow .75s linear infinite;animation:spinner-grow .75s linear infinite}.spinner-grow-sm{width:1rem;height:1rem}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-primary{background-color:#007bff!important}a.bg-primary:focus,a.bg-primary:hover,button.bg-primary:focus,button.bg-primary:hover{background-color:#0062cc!important}.bg-secondary{background-color:#6c757d!important}a.bg-secondary:focus,a.bg-secondary:hover,button.bg-secondary:focus,button.bg-secondary:hover{background-color:#545b62!important}.bg-success{background-color:#28a745!important}a.bg-success:focus,a.bg-success:hover,button.bg-success:focus,button.bg-success:hover{background-color:#1e7e34!important}.bg-info{background-color:#17a2b8!important}a.bg-info:focus,a.bg-info:hover,button.bg-info:focus,button.bg-info:hover{background-color:#117a8b!important}.bg-warning{background-color:#ffc107!important}a.bg-warning:focus,a.bg-warning:hover,button.bg-warning:focus,button.bg-warning:hover{background-color:#d39e00!important}.bg-danger{background-color:#dc3545!important}a.bg-danger:focus,a.bg-danger:hover,button.bg-danger:focus,button.bg-danger:hover{background-color:#bd2130!important}.bg-light{background-color:#f8f9fa!important}a.bg-light:focus,a.bg-light:hover,button.bg-light:focus,button.bg-light:hover{background-color:#dae0e5!important}.bg-dark{background-color:#343a40!important}a.bg-dark:focus,a.bg-dark:hover,button.bg-dark:focus,button.bg-dark:hover{background-color:#1d2124!important}.bg-white{background-color:#fff!important}.bg-transparent{background-color:transparent!important}.border{border:1px solid #dee2e6!important}.border-top{border-top:1px solid #dee2e6!important}.border-right{border-right:1px solid #dee2e6!important}.border-bottom{border-bottom:1px solid #dee2e6!important}.border-left{border-left:1px solid #dee2e6!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.border-primary{border-color:#007bff!important}.border-secondary{border-color:#6c757d!important}.border-success{border-color:#28a745!important}.border-info{border-color:#17a2b8!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#343a40!important}.border-white{border-color:#fff!important}.rounded-sm{border-radius:.2rem!important}.rounded{border-radius:.25rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-right{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-left{border-top-left-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-lg{border-radius:.3rem!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:50rem!important}.rounded-0{border-radius:0!important}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media print{.d-print-none{display:none!important}.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:-ms-flexbox!important;display:flex!important}.d-print-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-sm-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-sm-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-sm-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-sm-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-sm-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-sm-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-sm-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-sm-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-md-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-md-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-md-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-md-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-md-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-md-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-md-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-md-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-lg-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-lg-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-lg-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-lg-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-lg-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-lg-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-lg-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-lg-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-xl-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-xl-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-xl-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-xl-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-xl-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-xl-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-xl-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-xl-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.user-select-all{-webkit-user-select:all!important;-moz-user-select:all!important;-ms-user-select:all!important;user-select:all!important}.user-select-auto{-webkit-user-select:auto!important;-moz-user-select:auto!important;-ms-user-select:auto!important;user-select:auto!important}.user-select-none{-webkit-user-select:none!important;-moz-user-select:none!important;-ms-user-select:none!important;user-select:none!important}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports ((position:-webkit-sticky) or (position:sticky)){.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075)!important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15)!important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175)!important}.shadow-none{box-shadow:none!important}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.min-vw-100{min-width:100vw!important}.min-vh-100{min-height:100vh!important}.vw-100{width:100vw!important}.vh-100{height:100vh!important}.m-0{margin:0!important}.mt-0,.my-0{margin-top:0!important}.mr-0,.mx-0{margin-right:0!important}.mb-0,.my-0{margin-bottom:0!important}.ml-0,.mx-0{margin-left:0!important}.m-1{margin:.25rem!important}.mt-1,.my-1{margin-top:.25rem!important}.mr-1,.mx-1{margin-right:.25rem!important}.mb-1,.my-1{margin-bottom:.25rem!important}.ml-1,.mx-1{margin-left:.25rem!important}.m-2{margin:.5rem!important}.mt-2,.my-2{margin-top:.5rem!important}.mr-2,.mx-2{margin-right:.5rem!important}.mb-2,.my-2{margin-bottom:.5rem!important}.ml-2,.mx-2{margin-left:.5rem!important}.m-3{margin:1rem!important}.mt-3,.my-3{margin-top:1rem!important}.mr-3,.mx-3{margin-right:1rem!important}.mb-3,.my-3{margin-bottom:1rem!important}.ml-3,.mx-3{margin-left:1rem!important}.m-4{margin:1.5rem!important}.mt-4,.my-4{margin-top:1.5rem!important}.mr-4,.mx-4{margin-right:1.5rem!important}.mb-4,.my-4{margin-bottom:1.5rem!important}.ml-4,.mx-4{margin-left:1.5rem!important}.m-5{margin:3rem!important}.mt-5,.my-5{margin-top:3rem!important}.mr-5,.mx-5{margin-right:3rem!important}.mb-5,.my-5{margin-bottom:3rem!important}.ml-5,.mx-5{margin-left:3rem!important}.p-0{padding:0!important}.pt-0,.py-0{padding-top:0!important}.pr-0,.px-0{padding-right:0!important}.pb-0,.py-0{padding-bottom:0!important}.pl-0,.px-0{padding-left:0!important}.p-1{padding:.25rem!important}.pt-1,.py-1{padding-top:.25rem!important}.pr-1,.px-1{padding-right:.25rem!important}.pb-1,.py-1{padding-bottom:.25rem!important}.pl-1,.px-1{padding-left:.25rem!important}.p-2{padding:.5rem!important}.pt-2,.py-2{padding-top:.5rem!important}.pr-2,.px-2{padding-right:.5rem!important}.pb-2,.py-2{padding-bottom:.5rem!important}.pl-2,.px-2{padding-left:.5rem!important}.p-3{padding:1rem!important}.pt-3,.py-3{padding-top:1rem!important}.pr-3,.px-3{padding-right:1rem!important}.pb-3,.py-3{padding-bottom:1rem!important}.pl-3,.px-3{padding-left:1rem!important}.p-4{padding:1.5rem!important}.pt-4,.py-4{padding-top:1.5rem!important}.pr-4,.px-4{padding-right:1.5rem!important}.pb-4,.py-4{padding-bottom:1.5rem!important}.pl-4,.px-4{padding-left:1.5rem!important}.p-5{padding:3rem!important}.pt-5,.py-5{padding-top:3rem!important}.pr-5,.px-5{padding-right:3rem!important}.pb-5,.py-5{padding-bottom:3rem!important}.pl-5,.px-5{padding-left:3rem!important}.m-n1{margin:-.25rem!important}.mt-n1,.my-n1{margin-top:-.25rem!important}.mr-n1,.mx-n1{margin-right:-.25rem!important}.mb-n1,.my-n1{margin-bottom:-.25rem!important}.ml-n1,.mx-n1{margin-left:-.25rem!important}.m-n2{margin:-.5rem!important}.mt-n2,.my-n2{margin-top:-.5rem!important}.mr-n2,.mx-n2{margin-right:-.5rem!important}.mb-n2,.my-n2{margin-bottom:-.5rem!important}.ml-n2,.mx-n2{margin-left:-.5rem!important}.m-n3{margin:-1rem!important}.mt-n3,.my-n3{margin-top:-1rem!important}.mr-n3,.mx-n3{margin-right:-1rem!important}.mb-n3,.my-n3{margin-bottom:-1rem!important}.ml-n3,.mx-n3{margin-left:-1rem!important}.m-n4{margin:-1.5rem!important}.mt-n4,.my-n4{margin-top:-1.5rem!important}.mr-n4,.mx-n4{margin-right:-1.5rem!important}.mb-n4,.my-n4{margin-bottom:-1.5rem!important}.ml-n4,.mx-n4{margin-left:-1.5rem!important}.m-n5{margin:-3rem!important}.mt-n5,.my-n5{margin-top:-3rem!important}.mr-n5,.mx-n5{margin-right:-3rem!important}.mb-n5,.my-n5{margin-bottom:-3rem!important}.ml-n5,.mx-n5{margin-left:-3rem!important}.m-auto{margin:auto!important}.mt-auto,.my-auto{margin-top:auto!important}.mr-auto,.mx-auto{margin-right:auto!important}.mb-auto,.my-auto{margin-bottom:auto!important}.ml-auto,.mx-auto{margin-left:auto!important}@media (min-width:576px){.m-sm-0{margin:0!important}.mt-sm-0,.my-sm-0{margin-top:0!important}.mr-sm-0,.mx-sm-0{margin-right:0!important}.mb-sm-0,.my-sm-0{margin-bottom:0!important}.ml-sm-0,.mx-sm-0{margin-left:0!important}.m-sm-1{margin:.25rem!important}.mt-sm-1,.my-sm-1{margin-top:.25rem!important}.mr-sm-1,.mx-sm-1{margin-right:.25rem!important}.mb-sm-1,.my-sm-1{margin-bottom:.25rem!important}.ml-sm-1,.mx-sm-1{margin-left:.25rem!important}.m-sm-2{margin:.5rem!important}.mt-sm-2,.my-sm-2{margin-top:.5rem!important}.mr-sm-2,.mx-sm-2{margin-right:.5rem!important}.mb-sm-2,.my-sm-2{margin-bottom:.5rem!important}.ml-sm-2,.mx-sm-2{margin-left:.5rem!important}.m-sm-3{margin:1rem!important}.mt-sm-3,.my-sm-3{margin-top:1rem!important}.mr-sm-3,.mx-sm-3{margin-right:1rem!important}.mb-sm-3,.my-sm-3{margin-bottom:1rem!important}.ml-sm-3,.mx-sm-3{margin-left:1rem!important}.m-sm-4{margin:1.5rem!important}.mt-sm-4,.my-sm-4{margin-top:1.5rem!important}.mr-sm-4,.mx-sm-4{margin-right:1.5rem!important}.mb-sm-4,.my-sm-4{margin-bottom:1.5rem!important}.ml-sm-4,.mx-sm-4{margin-left:1.5rem!important}.m-sm-5{margin:3rem!important}.mt-sm-5,.my-sm-5{margin-top:3rem!important}.mr-sm-5,.mx-sm-5{margin-right:3rem!important}.mb-sm-5,.my-sm-5{margin-bottom:3rem!important}.ml-sm-5,.mx-sm-5{margin-left:3rem!important}.p-sm-0{padding:0!important}.pt-sm-0,.py-sm-0{padding-top:0!important}.pr-sm-0,.px-sm-0{padding-right:0!important}.pb-sm-0,.py-sm-0{padding-bottom:0!important}.pl-sm-0,.px-sm-0{padding-left:0!important}.p-sm-1{padding:.25rem!important}.pt-sm-1,.py-sm-1{padding-top:.25rem!important}.pr-sm-1,.px-sm-1{padding-right:.25rem!important}.pb-sm-1,.py-sm-1{padding-bottom:.25rem!important}.pl-sm-1,.px-sm-1{padding-left:.25rem!important}.p-sm-2{padding:.5rem!important}.pt-sm-2,.py-sm-2{padding-top:.5rem!important}.pr-sm-2,.px-sm-2{padding-right:.5rem!important}.pb-sm-2,.py-sm-2{padding-bottom:.5rem!important}.pl-sm-2,.px-sm-2{padding-left:.5rem!important}.p-sm-3{padding:1rem!important}.pt-sm-3,.py-sm-3{padding-top:1rem!important}.pr-sm-3,.px-sm-3{padding-right:1rem!important}.pb-sm-3,.py-sm-3{padding-bottom:1rem!important}.pl-sm-3,.px-sm-3{padding-left:1rem!important}.p-sm-4{padding:1.5rem!important}.pt-sm-4,.py-sm-4{padding-top:1.5rem!important}.pr-sm-4,.px-sm-4{padding-right:1.5rem!important}.pb-sm-4,.py-sm-4{padding-bottom:1.5rem!important}.pl-sm-4,.px-sm-4{padding-left:1.5rem!important}.p-sm-5{padding:3rem!important}.pt-sm-5,.py-sm-5{padding-top:3rem!important}.pr-sm-5,.px-sm-5{padding-right:3rem!important}.pb-sm-5,.py-sm-5{padding-bottom:3rem!important}.pl-sm-5,.px-sm-5{padding-left:3rem!important}.m-sm-n1{margin:-.25rem!important}.mt-sm-n1,.my-sm-n1{margin-top:-.25rem!important}.mr-sm-n1,.mx-sm-n1{margin-right:-.25rem!important}.mb-sm-n1,.my-sm-n1{margin-bottom:-.25rem!important}.ml-sm-n1,.mx-sm-n1{margin-left:-.25rem!important}.m-sm-n2{margin:-.5rem!important}.mt-sm-n2,.my-sm-n2{margin-top:-.5rem!important}.mr-sm-n2,.mx-sm-n2{margin-right:-.5rem!important}.mb-sm-n2,.my-sm-n2{margin-bottom:-.5rem!important}.ml-sm-n2,.mx-sm-n2{margin-left:-.5rem!important}.m-sm-n3{margin:-1rem!important}.mt-sm-n3,.my-sm-n3{margin-top:-1rem!important}.mr-sm-n3,.mx-sm-n3{margin-right:-1rem!important}.mb-sm-n3,.my-sm-n3{margin-bottom:-1rem!important}.ml-sm-n3,.mx-sm-n3{margin-left:-1rem!important}.m-sm-n4{margin:-1.5rem!important}.mt-sm-n4,.my-sm-n4{margin-top:-1.5rem!important}.mr-sm-n4,.mx-sm-n4{margin-right:-1.5rem!important}.mb-sm-n4,.my-sm-n4{margin-bottom:-1.5rem!important}.ml-sm-n4,.mx-sm-n4{margin-left:-1.5rem!important}.m-sm-n5{margin:-3rem!important}.mt-sm-n5,.my-sm-n5{margin-top:-3rem!important}.mr-sm-n5,.mx-sm-n5{margin-right:-3rem!important}.mb-sm-n5,.my-sm-n5{margin-bottom:-3rem!important}.ml-sm-n5,.mx-sm-n5{margin-left:-3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto,.my-sm-auto{margin-top:auto!important}.mr-sm-auto,.mx-sm-auto{margin-right:auto!important}.mb-sm-auto,.my-sm-auto{margin-bottom:auto!important}.ml-sm-auto,.mx-sm-auto{margin-left:auto!important}}@media (min-width:768px){.m-md-0{margin:0!important}.mt-md-0,.my-md-0{margin-top:0!important}.mr-md-0,.mx-md-0{margin-right:0!important}.mb-md-0,.my-md-0{margin-bottom:0!important}.ml-md-0,.mx-md-0{margin-left:0!important}.m-md-1{margin:.25rem!important}.mt-md-1,.my-md-1{margin-top:.25rem!important}.mr-md-1,.mx-md-1{margin-right:.25rem!important}.mb-md-1,.my-md-1{margin-bottom:.25rem!important}.ml-md-1,.mx-md-1{margin-left:.25rem!important}.m-md-2{margin:.5rem!important}.mt-md-2,.my-md-2{margin-top:.5rem!important}.mr-md-2,.mx-md-2{margin-right:.5rem!important}.mb-md-2,.my-md-2{margin-bottom:.5rem!important}.ml-md-2,.mx-md-2{margin-left:.5rem!important}.m-md-3{margin:1rem!important}.mt-md-3,.my-md-3{margin-top:1rem!important}.mr-md-3,.mx-md-3{margin-right:1rem!important}.mb-md-3,.my-md-3{margin-bottom:1rem!important}.ml-md-3,.mx-md-3{margin-left:1rem!important}.m-md-4{margin:1.5rem!important}.mt-md-4,.my-md-4{margin-top:1.5rem!important}.mr-md-4,.mx-md-4{margin-right:1.5rem!important}.mb-md-4,.my-md-4{margin-bottom:1.5rem!important}.ml-md-4,.mx-md-4{margin-left:1.5rem!important}.m-md-5{margin:3rem!important}.mt-md-5,.my-md-5{margin-top:3rem!important}.mr-md-5,.mx-md-5{margin-right:3rem!important}.mb-md-5,.my-md-5{margin-bottom:3rem!important}.ml-md-5,.mx-md-5{margin-left:3rem!important}.p-md-0{padding:0!important}.pt-md-0,.py-md-0{padding-top:0!important}.pr-md-0,.px-md-0{padding-right:0!important}.pb-md-0,.py-md-0{padding-bottom:0!important}.pl-md-0,.px-md-0{padding-left:0!important}.p-md-1{padding:.25rem!important}.pt-md-1,.py-md-1{padding-top:.25rem!important}.pr-md-1,.px-md-1{padding-right:.25rem!important}.pb-md-1,.py-md-1{padding-bottom:.25rem!important}.pl-md-1,.px-md-1{padding-left:.25rem!important}.p-md-2{padding:.5rem!important}.pt-md-2,.py-md-2{padding-top:.5rem!important}.pr-md-2,.px-md-2{padding-right:.5rem!important}.pb-md-2,.py-md-2{padding-bottom:.5rem!important}.pl-md-2,.px-md-2{padding-left:.5rem!important}.p-md-3{padding:1rem!important}.pt-md-3,.py-md-3{padding-top:1rem!important}.pr-md-3,.px-md-3{padding-right:1rem!important}.pb-md-3,.py-md-3{padding-bottom:1rem!important}.pl-md-3,.px-md-3{padding-left:1rem!important}.p-md-4{padding:1.5rem!important}.pt-md-4,.py-md-4{padding-top:1.5rem!important}.pr-md-4,.px-md-4{padding-right:1.5rem!important}.pb-md-4,.py-md-4{padding-bottom:1.5rem!important}.pl-md-4,.px-md-4{padding-left:1.5rem!important}.p-md-5{padding:3rem!important}.pt-md-5,.py-md-5{padding-top:3rem!important}.pr-md-5,.px-md-5{padding-right:3rem!important}.pb-md-5,.py-md-5{padding-bottom:3rem!important}.pl-md-5,.px-md-5{padding-left:3rem!important}.m-md-n1{margin:-.25rem!important}.mt-md-n1,.my-md-n1{margin-top:-.25rem!important}.mr-md-n1,.mx-md-n1{margin-right:-.25rem!important}.mb-md-n1,.my-md-n1{margin-bottom:-.25rem!important}.ml-md-n1,.mx-md-n1{margin-left:-.25rem!important}.m-md-n2{margin:-.5rem!important}.mt-md-n2,.my-md-n2{margin-top:-.5rem!important}.mr-md-n2,.mx-md-n2{margin-right:-.5rem!important}.mb-md-n2,.my-md-n2{margin-bottom:-.5rem!important}.ml-md-n2,.mx-md-n2{margin-left:-.5rem!important}.m-md-n3{margin:-1rem!important}.mt-md-n3,.my-md-n3{margin-top:-1rem!important}.mr-md-n3,.mx-md-n3{margin-right:-1rem!important}.mb-md-n3,.my-md-n3{margin-bottom:-1rem!important}.ml-md-n3,.mx-md-n3{margin-left:-1rem!important}.m-md-n4{margin:-1.5rem!important}.mt-md-n4,.my-md-n4{margin-top:-1.5rem!important}.mr-md-n4,.mx-md-n4{margin-right:-1.5rem!important}.mb-md-n4,.my-md-n4{margin-bottom:-1.5rem!important}.ml-md-n4,.mx-md-n4{margin-left:-1.5rem!important}.m-md-n5{margin:-3rem!important}.mt-md-n5,.my-md-n5{margin-top:-3rem!important}.mr-md-n5,.mx-md-n5{margin-right:-3rem!important}.mb-md-n5,.my-md-n5{margin-bottom:-3rem!important}.ml-md-n5,.mx-md-n5{margin-left:-3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto,.my-md-auto{margin-top:auto!important}.mr-md-auto,.mx-md-auto{margin-right:auto!important}.mb-md-auto,.my-md-auto{margin-bottom:auto!important}.ml-md-auto,.mx-md-auto{margin-left:auto!important}}@media (min-width:992px){.m-lg-0{margin:0!important}.mt-lg-0,.my-lg-0{margin-top:0!important}.mr-lg-0,.mx-lg-0{margin-right:0!important}.mb-lg-0,.my-lg-0{margin-bottom:0!important}.ml-lg-0,.mx-lg-0{margin-left:0!important}.m-lg-1{margin:.25rem!important}.mt-lg-1,.my-lg-1{margin-top:.25rem!important}.mr-lg-1,.mx-lg-1{margin-right:.25rem!important}.mb-lg-1,.my-lg-1{margin-bottom:.25rem!important}.ml-lg-1,.mx-lg-1{margin-left:.25rem!important}.m-lg-2{margin:.5rem!important}.mt-lg-2,.my-lg-2{margin-top:.5rem!important}.mr-lg-2,.mx-lg-2{margin-right:.5rem!important}.mb-lg-2,.my-lg-2{margin-bottom:.5rem!important}.ml-lg-2,.mx-lg-2{margin-left:.5rem!important}.m-lg-3{margin:1rem!important}.mt-lg-3,.my-lg-3{margin-top:1rem!important}.mr-lg-3,.mx-lg-3{margin-right:1rem!important}.mb-lg-3,.my-lg-3{margin-bottom:1rem!important}.ml-lg-3,.mx-lg-3{margin-left:1rem!important}.m-lg-4{margin:1.5rem!important}.mt-lg-4,.my-lg-4{margin-top:1.5rem!important}.mr-lg-4,.mx-lg-4{margin-right:1.5rem!important}.mb-lg-4,.my-lg-4{margin-bottom:1.5rem!important}.ml-lg-4,.mx-lg-4{margin-left:1.5rem!important}.m-lg-5{margin:3rem!important}.mt-lg-5,.my-lg-5{margin-top:3rem!important}.mr-lg-5,.mx-lg-5{margin-right:3rem!important}.mb-lg-5,.my-lg-5{margin-bottom:3rem!important}.ml-lg-5,.mx-lg-5{margin-left:3rem!important}.p-lg-0{padding:0!important}.pt-lg-0,.py-lg-0{padding-top:0!important}.pr-lg-0,.px-lg-0{padding-right:0!important}.pb-lg-0,.py-lg-0{padding-bottom:0!important}.pl-lg-0,.px-lg-0{padding-left:0!important}.p-lg-1{padding:.25rem!important}.pt-lg-1,.py-lg-1{padding-top:.25rem!important}.pr-lg-1,.px-lg-1{padding-right:.25rem!important}.pb-lg-1,.py-lg-1{padding-bottom:.25rem!important}.pl-lg-1,.px-lg-1{padding-left:.25rem!important}.p-lg-2{padding:.5rem!important}.pt-lg-2,.py-lg-2{padding-top:.5rem!important}.pr-lg-2,.px-lg-2{padding-right:.5rem!important}.pb-lg-2,.py-lg-2{padding-bottom:.5rem!important}.pl-lg-2,.px-lg-2{padding-left:.5rem!important}.p-lg-3{padding:1rem!important}.pt-lg-3,.py-lg-3{padding-top:1rem!important}.pr-lg-3,.px-lg-3{padding-right:1rem!important}.pb-lg-3,.py-lg-3{padding-bottom:1rem!important}.pl-lg-3,.px-lg-3{padding-left:1rem!important}.p-lg-4{padding:1.5rem!important}.pt-lg-4,.py-lg-4{padding-top:1.5rem!important}.pr-lg-4,.px-lg-4{padding-right:1.5rem!important}.pb-lg-4,.py-lg-4{padding-bottom:1.5rem!important}.pl-lg-4,.px-lg-4{padding-left:1.5rem!important}.p-lg-5{padding:3rem!important}.pt-lg-5,.py-lg-5{padding-top:3rem!important}.pr-lg-5,.px-lg-5{padding-right:3rem!important}.pb-lg-5,.py-lg-5{padding-bottom:3rem!important}.pl-lg-5,.px-lg-5{padding-left:3rem!important}.m-lg-n1{margin:-.25rem!important}.mt-lg-n1,.my-lg-n1{margin-top:-.25rem!important}.mr-lg-n1,.mx-lg-n1{margin-right:-.25rem!important}.mb-lg-n1,.my-lg-n1{margin-bottom:-.25rem!important}.ml-lg-n1,.mx-lg-n1{margin-left:-.25rem!important}.m-lg-n2{margin:-.5rem!important}.mt-lg-n2,.my-lg-n2{margin-top:-.5rem!important}.mr-lg-n2,.mx-lg-n2{margin-right:-.5rem!important}.mb-lg-n2,.my-lg-n2{margin-bottom:-.5rem!important}.ml-lg-n2,.mx-lg-n2{margin-left:-.5rem!important}.m-lg-n3{margin:-1rem!important}.mt-lg-n3,.my-lg-n3{margin-top:-1rem!important}.mr-lg-n3,.mx-lg-n3{margin-right:-1rem!important}.mb-lg-n3,.my-lg-n3{margin-bottom:-1rem!important}.ml-lg-n3,.mx-lg-n3{margin-left:-1rem!important}.m-lg-n4{margin:-1.5rem!important}.mt-lg-n4,.my-lg-n4{margin-top:-1.5rem!important}.mr-lg-n4,.mx-lg-n4{margin-right:-1.5rem!important}.mb-lg-n4,.my-lg-n4{margin-bottom:-1.5rem!important}.ml-lg-n4,.mx-lg-n4{margin-left:-1.5rem!important}.m-lg-n5{margin:-3rem!important}.mt-lg-n5,.my-lg-n5{margin-top:-3rem!important}.mr-lg-n5,.mx-lg-n5{margin-right:-3rem!important}.mb-lg-n5,.my-lg-n5{margin-bottom:-3rem!important}.ml-lg-n5,.mx-lg-n5{margin-left:-3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto,.my-lg-auto{margin-top:auto!important}.mr-lg-auto,.mx-lg-auto{margin-right:auto!important}.mb-lg-auto,.my-lg-auto{margin-bottom:auto!important}.ml-lg-auto,.mx-lg-auto{margin-left:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0!important}.mt-xl-0,.my-xl-0{margin-top:0!important}.mr-xl-0,.mx-xl-0{margin-right:0!important}.mb-xl-0,.my-xl-0{margin-bottom:0!important}.ml-xl-0,.mx-xl-0{margin-left:0!important}.m-xl-1{margin:.25rem!important}.mt-xl-1,.my-xl-1{margin-top:.25rem!important}.mr-xl-1,.mx-xl-1{margin-right:.25rem!important}.mb-xl-1,.my-xl-1{margin-bottom:.25rem!important}.ml-xl-1,.mx-xl-1{margin-left:.25rem!important}.m-xl-2{margin:.5rem!important}.mt-xl-2,.my-xl-2{margin-top:.5rem!important}.mr-xl-2,.mx-xl-2{margin-right:.5rem!important}.mb-xl-2,.my-xl-2{margin-bottom:.5rem!important}.ml-xl-2,.mx-xl-2{margin-left:.5rem!important}.m-xl-3{margin:1rem!important}.mt-xl-3,.my-xl-3{margin-top:1rem!important}.mr-xl-3,.mx-xl-3{margin-right:1rem!important}.mb-xl-3,.my-xl-3{margin-bottom:1rem!important}.ml-xl-3,.mx-xl-3{margin-left:1rem!important}.m-xl-4{margin:1.5rem!important}.mt-xl-4,.my-xl-4{margin-top:1.5rem!important}.mr-xl-4,.mx-xl-4{margin-right:1.5rem!important}.mb-xl-4,.my-xl-4{margin-bottom:1.5rem!important}.ml-xl-4,.mx-xl-4{margin-left:1.5rem!important}.m-xl-5{margin:3rem!important}.mt-xl-5,.my-xl-5{margin-top:3rem!important}.mr-xl-5,.mx-xl-5{margin-right:3rem!important}.mb-xl-5,.my-xl-5{margin-bottom:3rem!important}.ml-xl-5,.mx-xl-5{margin-left:3rem!important}.p-xl-0{padding:0!important}.pt-xl-0,.py-xl-0{padding-top:0!important}.pr-xl-0,.px-xl-0{padding-right:0!important}.pb-xl-0,.py-xl-0{padding-bottom:0!important}.pl-xl-0,.px-xl-0{padding-left:0!important}.p-xl-1{padding:.25rem!important}.pt-xl-1,.py-xl-1{padding-top:.25rem!important}.pr-xl-1,.px-xl-1{padding-right:.25rem!important}.pb-xl-1,.py-xl-1{padding-bottom:.25rem!important}.pl-xl-1,.px-xl-1{padding-left:.25rem!important}.p-xl-2{padding:.5rem!important}.pt-xl-2,.py-xl-2{padding-top:.5rem!important}.pr-xl-2,.px-xl-2{padding-right:.5rem!important}.pb-xl-2,.py-xl-2{padding-bottom:.5rem!important}.pl-xl-2,.px-xl-2{padding-left:.5rem!important}.p-xl-3{padding:1rem!important}.pt-xl-3,.py-xl-3{padding-top:1rem!important}.pr-xl-3,.px-xl-3{padding-right:1rem!important}.pb-xl-3,.py-xl-3{padding-bottom:1rem!important}.pl-xl-3,.px-xl-3{padding-left:1rem!important}.p-xl-4{padding:1.5rem!important}.pt-xl-4,.py-xl-4{padding-top:1.5rem!important}.pr-xl-4,.px-xl-4{padding-right:1.5rem!important}.pb-xl-4,.py-xl-4{padding-bottom:1.5rem!important}.pl-xl-4,.px-xl-4{padding-left:1.5rem!important}.p-xl-5{padding:3rem!important}.pt-xl-5,.py-xl-5{padding-top:3rem!important}.pr-xl-5,.px-xl-5{padding-right:3rem!important}.pb-xl-5,.py-xl-5{padding-bottom:3rem!important}.pl-xl-5,.px-xl-5{padding-left:3rem!important}.m-xl-n1{margin:-.25rem!important}.mt-xl-n1,.my-xl-n1{margin-top:-.25rem!important}.mr-xl-n1,.mx-xl-n1{margin-right:-.25rem!important}.mb-xl-n1,.my-xl-n1{margin-bottom:-.25rem!important}.ml-xl-n1,.mx-xl-n1{margin-left:-.25rem!important}.m-xl-n2{margin:-.5rem!important}.mt-xl-n2,.my-xl-n2{margin-top:-.5rem!important}.mr-xl-n2,.mx-xl-n2{margin-right:-.5rem!important}.mb-xl-n2,.my-xl-n2{margin-bottom:-.5rem!important}.ml-xl-n2,.mx-xl-n2{margin-left:-.5rem!important}.m-xl-n3{margin:-1rem!important}.mt-xl-n3,.my-xl-n3{margin-top:-1rem!important}.mr-xl-n3,.mx-xl-n3{margin-right:-1rem!important}.mb-xl-n3,.my-xl-n3{margin-bottom:-1rem!important}.ml-xl-n3,.mx-xl-n3{margin-left:-1rem!important}.m-xl-n4{margin:-1.5rem!important}.mt-xl-n4,.my-xl-n4{margin-top:-1.5rem!important}.mr-xl-n4,.mx-xl-n4{margin-right:-1.5rem!important}.mb-xl-n4,.my-xl-n4{margin-bottom:-1.5rem!important}.ml-xl-n4,.mx-xl-n4{margin-left:-1.5rem!important}.m-xl-n5{margin:-3rem!important}.mt-xl-n5,.my-xl-n5{margin-top:-3rem!important}.mr-xl-n5,.mx-xl-n5{margin-right:-3rem!important}.mb-xl-n5,.my-xl-n5{margin-bottom:-3rem!important}.ml-xl-n5,.mx-xl-n5{margin-left:-3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto,.my-xl-auto{margin-top:auto!important}.mr-xl-auto,.mx-xl-auto{margin-right:auto!important}.mb-xl-auto,.my-xl-auto{margin-bottom:auto!important}.ml-xl-auto,.mx-xl-auto{margin-left:auto!important}}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;pointer-events:auto;content:"";background-color:rgba(0,0,0,0)}.text-monospace{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace!important}.text-justify{text-align:justify!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-light{font-weight:300!important}.font-weight-lighter{font-weight:lighter!important}.font-weight-normal{font-weight:400!important}.font-weight-bold{font-weight:700!important}.font-weight-bolder{font-weight:bolder!important}.font-italic{font-style:italic!important}.text-white{color:#fff!important}.text-primary{color:#007bff!important}a.text-primary:focus,a.text-primary:hover{color:#0056b3!important}.text-secondary{color:#6c757d!important}a.text-secondary:focus,a.text-secondary:hover{color:#494f54!important}.text-success{color:#28a745!important}a.text-success:focus,a.text-success:hover{color:#19692c!important}.text-info{color:#17a2b8!important}a.text-info:focus,a.text-info:hover{color:#0f6674!important}.text-warning{color:#ffc107!important}a.text-warning:focus,a.text-warning:hover{color:#ba8b00!important}.text-danger{color:#dc3545!important}a.text-danger:focus,a.text-danger:hover{color:#a71d2a!important}.text-light{color:#f8f9fa!important}a.text-light:focus,a.text-light:hover{color:#cbd3da!important}.text-dark{color:#343a40!important}a.text-dark:focus,a.text-dark:hover{color:#121416!important}.text-body{color:#212529!important}.text-muted{color:#6c757d!important}.text-black-50{color:rgba(0,0,0,.5)!important}.text-white-50{color:rgba(255,255,255,.5)!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.text-decoration-none{text-decoration:none!important}.text-break{word-wrap:break-word!important}.text-reset{color:inherit!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}@media print{*,::after,::before{text-shadow:none!important;box-shadow:none!important}a:not(.btn){text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap!important}blockquote,pre{border:1px solid #adb5bd;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}@page{size:a3}body{min-width:992px!important}.container{min-width:992px!important}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #dee2e6!important}.table-dark{color:inherit}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#dee2e6}.table .thead-dark th{color:inherit;border-color:#dee2e6}} -/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/imgr/imgrpkg/static-content/bootstrap.min.js b/imgr/imgrpkg/static-content/bootstrap.min.js deleted file mode 100644 index 3ecf55f2..00000000 --- a/imgr/imgrpkg/static-content/bootstrap.min.js +++ /dev/null @@ -1,7 +0,0 @@ -/*! - * Bootstrap v4.5.0 (https://getbootstrap.com/) - * Copyright 2011-2020 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */ -!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports,require("jquery"),require("popper.js")):"function"==typeof define&&define.amd?define(["exports","jquery","popper.js"],e):e((t=t||self).bootstrap={},t.jQuery,t.Popper)}(this,(function(t,e,n){"use strict";function i(t,e){for(var n=0;n=4)throw new Error("Bootstrap's JavaScript requires at least jQuery v1.9.1 but less than v4.0.0")}};c.jQueryDetection(),e.fn.emulateTransitionEnd=l,e.event.special[c.TRANSITION_END]={bindType:"transitionend",delegateType:"transitionend",handle:function(t){if(e(t.target).is(this))return t.handleObj.handler.apply(this,arguments)}};var h="alert",u=e.fn[h],d=function(){function t(t){this._element=t}var n=t.prototype;return n.close=function(t){var e=this._element;t&&(e=this._getRootElement(t)),this._triggerCloseEvent(e).isDefaultPrevented()||this._removeElement(e)},n.dispose=function(){e.removeData(this._element,"bs.alert"),this._element=null},n._getRootElement=function(t){var n=c.getSelectorFromElement(t),i=!1;return n&&(i=document.querySelector(n)),i||(i=e(t).closest(".alert")[0]),i},n._triggerCloseEvent=function(t){var n=e.Event("close.bs.alert");return e(t).trigger(n),n},n._removeElement=function(t){var n=this;if(e(t).removeClass("show"),e(t).hasClass("fade")){var i=c.getTransitionDurationFromElement(t);e(t).one(c.TRANSITION_END,(function(e){return n._destroyElement(t,e)})).emulateTransitionEnd(i)}else this._destroyElement(t)},n._destroyElement=function(t){e(t).detach().trigger("closed.bs.alert").remove()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.alert");o||(o=new t(this),i.data("bs.alert",o)),"close"===n&&o[n](this)}))},t._handleDismiss=function(t){return function(e){e&&e.preventDefault(),t.close(this)}},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.alert.data-api",'[data-dismiss="alert"]',d._handleDismiss(new d)),e.fn[h]=d._jQueryInterface,e.fn[h].Constructor=d,e.fn[h].noConflict=function(){return e.fn[h]=u,d._jQueryInterface};var f=e.fn.button,g=function(){function t(t){this._element=t}var n=t.prototype;return n.toggle=function(){var t=!0,n=!0,i=e(this._element).closest('[data-toggle="buttons"]')[0];if(i){var o=this._element.querySelector('input:not([type="hidden"])');if(o){if("radio"===o.type)if(o.checked&&this._element.classList.contains("active"))t=!1;else{var s=i.querySelector(".active");s&&e(s).removeClass("active")}t&&("checkbox"!==o.type&&"radio"!==o.type||(o.checked=!this._element.classList.contains("active")),e(o).trigger("change")),o.focus(),n=!1}}this._element.hasAttribute("disabled")||this._element.classList.contains("disabled")||(n&&this._element.setAttribute("aria-pressed",!this._element.classList.contains("active")),t&&e(this._element).toggleClass("active"))},n.dispose=function(){e.removeData(this._element,"bs.button"),this._element=null},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.button");i||(i=new t(this),e(this).data("bs.button",i)),"toggle"===n&&i[n]()}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.button.data-api",'[data-toggle^="button"]',(function(t){var n=t.target,i=n;if(e(n).hasClass("btn")||(n=e(n).closest(".btn")[0]),!n||n.hasAttribute("disabled")||n.classList.contains("disabled"))t.preventDefault();else{var o=n.querySelector('input:not([type="hidden"])');if(o&&(o.hasAttribute("disabled")||o.classList.contains("disabled")))return void t.preventDefault();"LABEL"===i.tagName&&o&&"checkbox"===o.type&&t.preventDefault(),g._jQueryInterface.call(e(n),"toggle")}})).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',(function(t){var n=e(t.target).closest(".btn")[0];e(n).toggleClass("focus",/^focus(in)?$/.test(t.type))})),e(window).on("load.bs.button.data-api",(function(){for(var t=[].slice.call(document.querySelectorAll('[data-toggle="buttons"] .btn')),e=0,n=t.length;e0,this._pointerEvent=Boolean(window.PointerEvent||window.MSPointerEvent),this._addEventListeners()}var n=t.prototype;return n.next=function(){this._isSliding||this._slide("next")},n.nextWhenVisible=function(){!document.hidden&&e(this._element).is(":visible")&&"hidden"!==e(this._element).css("visibility")&&this.next()},n.prev=function(){this._isSliding||this._slide("prev")},n.pause=function(t){t||(this._isPaused=!0),this._element.querySelector(".carousel-item-next, .carousel-item-prev")&&(c.triggerTransitionEnd(this._element),this.cycle(!0)),clearInterval(this._interval),this._interval=null},n.cycle=function(t){t||(this._isPaused=!1),this._interval&&(clearInterval(this._interval),this._interval=null),this._config.interval&&!this._isPaused&&(this._interval=setInterval((document.visibilityState?this.nextWhenVisible:this.next).bind(this),this._config.interval))},n.to=function(t){var n=this;this._activeElement=this._element.querySelector(".active.carousel-item");var i=this._getItemIndex(this._activeElement);if(!(t>this._items.length-1||t<0))if(this._isSliding)e(this._element).one("slid.bs.carousel",(function(){return n.to(t)}));else{if(i===t)return this.pause(),void this.cycle();var o=t>i?"next":"prev";this._slide(o,this._items[t])}},n.dispose=function(){e(this._element).off(p),e.removeData(this._element,"bs.carousel"),this._items=null,this._config=null,this._element=null,this._interval=null,this._isPaused=null,this._isSliding=null,this._activeElement=null,this._indicatorsElement=null},n._getConfig=function(t){return t=a(a({},v),t),c.typeCheckConfig(m,t,b),t},n._handleSwipe=function(){var t=Math.abs(this.touchDeltaX);if(!(t<=40)){var e=t/this.touchDeltaX;this.touchDeltaX=0,e>0&&this.prev(),e<0&&this.next()}},n._addEventListeners=function(){var t=this;this._config.keyboard&&e(this._element).on("keydown.bs.carousel",(function(e){return t._keydown(e)})),"hover"===this._config.pause&&e(this._element).on("mouseenter.bs.carousel",(function(e){return t.pause(e)})).on("mouseleave.bs.carousel",(function(e){return t.cycle(e)})),this._config.touch&&this._addTouchEventListeners()},n._addTouchEventListeners=function(){var t=this;if(this._touchSupported){var n=function(e){t._pointerEvent&&y[e.originalEvent.pointerType.toUpperCase()]?t.touchStartX=e.originalEvent.clientX:t._pointerEvent||(t.touchStartX=e.originalEvent.touches[0].clientX)},i=function(e){t._pointerEvent&&y[e.originalEvent.pointerType.toUpperCase()]&&(t.touchDeltaX=e.originalEvent.clientX-t.touchStartX),t._handleSwipe(),"hover"===t._config.pause&&(t.pause(),t.touchTimeout&&clearTimeout(t.touchTimeout),t.touchTimeout=setTimeout((function(e){return t.cycle(e)}),500+t._config.interval))};e(this._element.querySelectorAll(".carousel-item img")).on("dragstart.bs.carousel",(function(t){return t.preventDefault()})),this._pointerEvent?(e(this._element).on("pointerdown.bs.carousel",(function(t){return n(t)})),e(this._element).on("pointerup.bs.carousel",(function(t){return i(t)})),this._element.classList.add("pointer-event")):(e(this._element).on("touchstart.bs.carousel",(function(t){return n(t)})),e(this._element).on("touchmove.bs.carousel",(function(e){return function(e){e.originalEvent.touches&&e.originalEvent.touches.length>1?t.touchDeltaX=0:t.touchDeltaX=e.originalEvent.touches[0].clientX-t.touchStartX}(e)})),e(this._element).on("touchend.bs.carousel",(function(t){return i(t)})))}},n._keydown=function(t){if(!/input|textarea/i.test(t.target.tagName))switch(t.which){case 37:t.preventDefault(),this.prev();break;case 39:t.preventDefault(),this.next()}},n._getItemIndex=function(t){return this._items=t&&t.parentNode?[].slice.call(t.parentNode.querySelectorAll(".carousel-item")):[],this._items.indexOf(t)},n._getItemByDirection=function(t,e){var n="next"===t,i="prev"===t,o=this._getItemIndex(e),s=this._items.length-1;if((i&&0===o||n&&o===s)&&!this._config.wrap)return e;var r=(o+("prev"===t?-1:1))%this._items.length;return-1===r?this._items[this._items.length-1]:this._items[r]},n._triggerSlideEvent=function(t,n){var i=this._getItemIndex(t),o=this._getItemIndex(this._element.querySelector(".active.carousel-item")),s=e.Event("slide.bs.carousel",{relatedTarget:t,direction:n,from:o,to:i});return e(this._element).trigger(s),s},n._setActiveIndicatorElement=function(t){if(this._indicatorsElement){var n=[].slice.call(this._indicatorsElement.querySelectorAll(".active"));e(n).removeClass("active");var i=this._indicatorsElement.children[this._getItemIndex(t)];i&&e(i).addClass("active")}},n._slide=function(t,n){var i,o,s,r=this,a=this._element.querySelector(".active.carousel-item"),l=this._getItemIndex(a),h=n||a&&this._getItemByDirection(t,a),u=this._getItemIndex(h),d=Boolean(this._interval);if("next"===t?(i="carousel-item-left",o="carousel-item-next",s="left"):(i="carousel-item-right",o="carousel-item-prev",s="right"),h&&e(h).hasClass("active"))this._isSliding=!1;else if(!this._triggerSlideEvent(h,s).isDefaultPrevented()&&a&&h){this._isSliding=!0,d&&this.pause(),this._setActiveIndicatorElement(h);var f=e.Event("slid.bs.carousel",{relatedTarget:h,direction:s,from:l,to:u});if(e(this._element).hasClass("slide")){e(h).addClass(o),c.reflow(h),e(a).addClass(i),e(h).addClass(i);var g=parseInt(h.getAttribute("data-interval"),10);g?(this._config.defaultInterval=this._config.defaultInterval||this._config.interval,this._config.interval=g):this._config.interval=this._config.defaultInterval||this._config.interval;var m=c.getTransitionDurationFromElement(a);e(a).one(c.TRANSITION_END,(function(){e(h).removeClass(i+" "+o).addClass("active"),e(a).removeClass("active "+o+" "+i),r._isSliding=!1,setTimeout((function(){return e(r._element).trigger(f)}),0)})).emulateTransitionEnd(m)}else e(a).removeClass("active"),e(h).addClass("active"),this._isSliding=!1,e(this._element).trigger(f);d&&this.cycle()}},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.carousel"),o=a(a({},v),e(this).data());"object"==typeof n&&(o=a(a({},o),n));var s="string"==typeof n?n:o.slide;if(i||(i=new t(this,o),e(this).data("bs.carousel",i)),"number"==typeof n)i.to(n);else if("string"==typeof s){if("undefined"==typeof i[s])throw new TypeError('No method named "'+s+'"');i[s]()}else o.interval&&o.ride&&(i.pause(),i.cycle())}))},t._dataApiClickHandler=function(n){var i=c.getSelectorFromElement(this);if(i){var o=e(i)[0];if(o&&e(o).hasClass("carousel")){var s=a(a({},e(o).data()),e(this).data()),r=this.getAttribute("data-slide-to");r&&(s.interval=!1),t._jQueryInterface.call(e(o),s),r&&e(o).data("bs.carousel").to(r),n.preventDefault()}}},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return v}}]),t}();e(document).on("click.bs.carousel.data-api","[data-slide], [data-slide-to]",E._dataApiClickHandler),e(window).on("load.bs.carousel.data-api",(function(){for(var t=[].slice.call(document.querySelectorAll('[data-ride="carousel"]')),n=0,i=t.length;n0&&(this._selector=r,this._triggerArray.push(s))}this._parent=this._config.parent?this._getParent():null,this._config.parent||this._addAriaAndCollapsedClass(this._element,this._triggerArray),this._config.toggle&&this.toggle()}var n=t.prototype;return n.toggle=function(){e(this._element).hasClass("show")?this.hide():this.show()},n.show=function(){var n,i,o=this;if(!this._isTransitioning&&!e(this._element).hasClass("show")&&(this._parent&&0===(n=[].slice.call(this._parent.querySelectorAll(".show, .collapsing")).filter((function(t){return"string"==typeof o._config.parent?t.getAttribute("data-parent")===o._config.parent:t.classList.contains("collapse")}))).length&&(n=null),!(n&&(i=e(n).not(this._selector).data("bs.collapse"))&&i._isTransitioning))){var s=e.Event("show.bs.collapse");if(e(this._element).trigger(s),!s.isDefaultPrevented()){n&&(t._jQueryInterface.call(e(n).not(this._selector),"hide"),i||e(n).data("bs.collapse",null));var r=this._getDimension();e(this._element).removeClass("collapse").addClass("collapsing"),this._element.style[r]=0,this._triggerArray.length&&e(this._triggerArray).removeClass("collapsed").attr("aria-expanded",!0),this.setTransitioning(!0);var a="scroll"+(r[0].toUpperCase()+r.slice(1)),l=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,(function(){e(o._element).removeClass("collapsing").addClass("collapse show"),o._element.style[r]="",o.setTransitioning(!1),e(o._element).trigger("shown.bs.collapse")})).emulateTransitionEnd(l),this._element.style[r]=this._element[a]+"px"}}},n.hide=function(){var t=this;if(!this._isTransitioning&&e(this._element).hasClass("show")){var n=e.Event("hide.bs.collapse");if(e(this._element).trigger(n),!n.isDefaultPrevented()){var i=this._getDimension();this._element.style[i]=this._element.getBoundingClientRect()[i]+"px",c.reflow(this._element),e(this._element).addClass("collapsing").removeClass("collapse show");var o=this._triggerArray.length;if(o>0)for(var s=0;s0},i._getOffset=function(){var t=this,e={};return"function"==typeof this._config.offset?e.fn=function(e){return e.offsets=a(a({},e.offsets),t._config.offset(e.offsets,t._element)||{}),e}:e.offset=this._config.offset,e},i._getPopperConfig=function(){var t={placement:this._getPlacement(),modifiers:{offset:this._getOffset(),flip:{enabled:this._config.flip},preventOverflow:{boundariesElement:this._config.boundary}}};return"static"===this._config.display&&(t.modifiers.applyStyle={enabled:!1}),a(a({},t),this._config.popperConfig)},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.dropdown");if(i||(i=new t(this,"object"==typeof n?n:null),e(this).data("bs.dropdown",i)),"string"==typeof n){if("undefined"==typeof i[n])throw new TypeError('No method named "'+n+'"');i[n]()}}))},t._clearMenus=function(n){if(!n||3!==n.which&&("keyup"!==n.type||9===n.which))for(var i=[].slice.call(document.querySelectorAll('[data-toggle="dropdown"]')),o=0,s=i.length;o0&&r--,40===n.which&&rdocument.documentElement.clientHeight;!this._isBodyOverflowing&&t&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!t&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},n._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},n._checkScrollbar=function(){var t=document.body.getBoundingClientRect();this._isBodyOverflowing=Math.round(t.left+t.right)

',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip",boundary:"scrollParent",sanitize:!0,sanitizeFn:null,whiteList:F,popperConfig:null},Y={HIDE:"hide.bs.tooltip",HIDDEN:"hidden.bs.tooltip",SHOW:"show.bs.tooltip",SHOWN:"shown.bs.tooltip",INSERTED:"inserted.bs.tooltip",CLICK:"click.bs.tooltip",FOCUSIN:"focusin.bs.tooltip",FOCUSOUT:"focusout.bs.tooltip",MOUSEENTER:"mouseenter.bs.tooltip",MOUSELEAVE:"mouseleave.bs.tooltip"},$=function(){function t(t,e){if("undefined"==typeof n)throw new TypeError("Bootstrap's tooltips require Popper.js (https://popper.js.org/)");this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=t,this.config=this._getConfig(e),this.tip=null,this._setListeners()}var i=t.prototype;return i.enable=function(){this._isEnabled=!0},i.disable=function(){this._isEnabled=!1},i.toggleEnabled=function(){this._isEnabled=!this._isEnabled},i.toggle=function(t){if(this._isEnabled)if(t){var n=this.constructor.DATA_KEY,i=e(t.currentTarget).data(n);i||(i=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(n,i)),i._activeTrigger.click=!i._activeTrigger.click,i._isWithActiveTrigger()?i._enter(null,i):i._leave(null,i)}else{if(e(this.getTipElement()).hasClass("show"))return void this._leave(null,this);this._enter(null,this)}},i.dispose=function(){clearTimeout(this._timeout),e.removeData(this.element,this.constructor.DATA_KEY),e(this.element).off(this.constructor.EVENT_KEY),e(this.element).closest(".modal").off("hide.bs.modal",this._hideModalHandler),this.tip&&e(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,this._activeTrigger=null,this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},i.show=function(){var t=this;if("none"===e(this.element).css("display"))throw new Error("Please use show on visible elements");var i=e.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){e(this.element).trigger(i);var o=c.findShadowRoot(this.element),s=e.contains(null!==o?o:this.element.ownerDocument.documentElement,this.element);if(i.isDefaultPrevented()||!s)return;var r=this.getTipElement(),a=c.getUID(this.constructor.NAME);r.setAttribute("id",a),this.element.setAttribute("aria-describedby",a),this.setContent(),this.config.animation&&e(r).addClass("fade");var l="function"==typeof this.config.placement?this.config.placement.call(this,r,this.element):this.config.placement,h=this._getAttachment(l);this.addAttachmentClass(h);var u=this._getContainer();e(r).data(this.constructor.DATA_KEY,this),e.contains(this.element.ownerDocument.documentElement,this.tip)||e(r).appendTo(u),e(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new n(this.element,r,this._getPopperConfig(h)),e(r).addClass("show"),"ontouchstart"in document.documentElement&&e(document.body).children().on("mouseover",null,e.noop);var d=function(){t.config.animation&&t._fixTransition();var n=t._hoverState;t._hoverState=null,e(t.element).trigger(t.constructor.Event.SHOWN),"out"===n&&t._leave(null,t)};if(e(this.tip).hasClass("fade")){var f=c.getTransitionDurationFromElement(this.tip);e(this.tip).one(c.TRANSITION_END,d).emulateTransitionEnd(f)}else d()}},i.hide=function(t){var n=this,i=this.getTipElement(),o=e.Event(this.constructor.Event.HIDE),s=function(){"show"!==n._hoverState&&i.parentNode&&i.parentNode.removeChild(i),n._cleanTipClass(),n.element.removeAttribute("aria-describedby"),e(n.element).trigger(n.constructor.Event.HIDDEN),null!==n._popper&&n._popper.destroy(),t&&t()};if(e(this.element).trigger(o),!o.isDefaultPrevented()){if(e(i).removeClass("show"),"ontouchstart"in document.documentElement&&e(document.body).children().off("mouseover",null,e.noop),this._activeTrigger.click=!1,this._activeTrigger.focus=!1,this._activeTrigger.hover=!1,e(this.tip).hasClass("fade")){var r=c.getTransitionDurationFromElement(i);e(i).one(c.TRANSITION_END,s).emulateTransitionEnd(r)}else s();this._hoverState=""}},i.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},i.isWithContent=function(){return Boolean(this.getTitle())},i.addAttachmentClass=function(t){e(this.getTipElement()).addClass("bs-tooltip-"+t)},i.getTipElement=function(){return this.tip=this.tip||e(this.config.template)[0],this.tip},i.setContent=function(){var t=this.getTipElement();this.setElementContent(e(t.querySelectorAll(".tooltip-inner")),this.getTitle()),e(t).removeClass("fade show")},i.setElementContent=function(t,n){"object"!=typeof n||!n.nodeType&&!n.jquery?this.config.html?(this.config.sanitize&&(n=H(n,this.config.whiteList,this.config.sanitizeFn)),t.html(n)):t.text(n):this.config.html?e(n).parent().is(t)||t.empty().append(n):t.text(e(n).text())},i.getTitle=function(){var t=this.element.getAttribute("data-original-title");return t||(t="function"==typeof this.config.title?this.config.title.call(this.element):this.config.title),t},i._getPopperConfig=function(t){var e=this;return a(a({},{placement:t,modifiers:{offset:this._getOffset(),flip:{behavior:this.config.fallbackPlacement},arrow:{element:".arrow"},preventOverflow:{boundariesElement:this.config.boundary}},onCreate:function(t){t.originalPlacement!==t.placement&&e._handlePopperPlacementChange(t)},onUpdate:function(t){return e._handlePopperPlacementChange(t)}}),this.config.popperConfig)},i._getOffset=function(){var t=this,e={};return"function"==typeof this.config.offset?e.fn=function(e){return e.offsets=a(a({},e.offsets),t.config.offset(e.offsets,t.element)||{}),e}:e.offset=this.config.offset,e},i._getContainer=function(){return!1===this.config.container?document.body:c.isElement(this.config.container)?e(this.config.container):e(document).find(this.config.container)},i._getAttachment=function(t){return K[t.toUpperCase()]},i._setListeners=function(){var t=this;this.config.trigger.split(" ").forEach((function(n){if("click"===n)e(t.element).on(t.constructor.Event.CLICK,t.config.selector,(function(e){return t.toggle(e)}));else if("manual"!==n){var i="hover"===n?t.constructor.Event.MOUSEENTER:t.constructor.Event.FOCUSIN,o="hover"===n?t.constructor.Event.MOUSELEAVE:t.constructor.Event.FOCUSOUT;e(t.element).on(i,t.config.selector,(function(e){return t._enter(e)})).on(o,t.config.selector,(function(e){return t._leave(e)}))}})),this._hideModalHandler=function(){t.element&&t.hide()},e(this.element).closest(".modal").on("hide.bs.modal",this._hideModalHandler),this.config.selector?this.config=a(a({},this.config),{},{trigger:"manual",selector:""}):this._fixTitle()},i._fixTitle=function(){var t=typeof this.element.getAttribute("data-original-title");(this.element.getAttribute("title")||"string"!==t)&&(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},i._enter=function(t,n){var i=this.constructor.DATA_KEY;(n=n||e(t.currentTarget).data(i))||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(i,n)),t&&(n._activeTrigger["focusin"===t.type?"focus":"hover"]=!0),e(n.getTipElement()).hasClass("show")||"show"===n._hoverState?n._hoverState="show":(clearTimeout(n._timeout),n._hoverState="show",n.config.delay&&n.config.delay.show?n._timeout=setTimeout((function(){"show"===n._hoverState&&n.show()}),n.config.delay.show):n.show())},i._leave=function(t,n){var i=this.constructor.DATA_KEY;(n=n||e(t.currentTarget).data(i))||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(i,n)),t&&(n._activeTrigger["focusout"===t.type?"focus":"hover"]=!1),n._isWithActiveTrigger()||(clearTimeout(n._timeout),n._hoverState="out",n.config.delay&&n.config.delay.hide?n._timeout=setTimeout((function(){"out"===n._hoverState&&n.hide()}),n.config.delay.hide):n.hide())},i._isWithActiveTrigger=function(){for(var t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1},i._getConfig=function(t){var n=e(this.element).data();return Object.keys(n).forEach((function(t){-1!==V.indexOf(t)&&delete n[t]})),"number"==typeof(t=a(a(a({},this.constructor.Default),n),"object"==typeof t&&t?t:{})).delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),c.typeCheckConfig(U,t,this.constructor.DefaultType),t.sanitize&&(t.template=H(t.template,t.whiteList,t.sanitizeFn)),t},i._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]!==this.config[e]&&(t[e]=this.config[e]);return t},i._cleanTipClass=function(){var t=e(this.getTipElement()),n=t.attr("class").match(W);null!==n&&n.length&&t.removeClass(n.join(""))},i._handlePopperPlacementChange=function(t){this.tip=t.instance.popper,this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(t.placement))},i._fixTransition=function(){var t=this.getTipElement(),n=this.config.animation;null===t.getAttribute("x-placement")&&(e(t).removeClass("fade"),this.config.animation=!1,this.hide(),this.show(),this.config.animation=n)},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.tooltip"),o="object"==typeof n&&n;if((i||!/dispose|hide/.test(n))&&(i||(i=new t(this,o),e(this).data("bs.tooltip",i)),"string"==typeof n)){if("undefined"==typeof i[n])throw new TypeError('No method named "'+n+'"');i[n]()}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return X}},{key:"NAME",get:function(){return U}},{key:"DATA_KEY",get:function(){return"bs.tooltip"}},{key:"Event",get:function(){return Y}},{key:"EVENT_KEY",get:function(){return".bs.tooltip"}},{key:"DefaultType",get:function(){return z}}]),t}();e.fn[U]=$._jQueryInterface,e.fn[U].Constructor=$,e.fn[U].noConflict=function(){return e.fn[U]=M,$._jQueryInterface};var J="popover",G=e.fn[J],Z=new RegExp("(^|\\s)bs-popover\\S+","g"),tt=a(a({},$.Default),{},{placement:"right",trigger:"click",content:"",template:''}),et=a(a({},$.DefaultType),{},{content:"(string|element|function)"}),nt={HIDE:"hide.bs.popover",HIDDEN:"hidden.bs.popover",SHOW:"show.bs.popover",SHOWN:"shown.bs.popover",INSERTED:"inserted.bs.popover",CLICK:"click.bs.popover",FOCUSIN:"focusin.bs.popover",FOCUSOUT:"focusout.bs.popover",MOUSEENTER:"mouseenter.bs.popover",MOUSELEAVE:"mouseleave.bs.popover"},it=function(t){var n,i;function s(){return t.apply(this,arguments)||this}i=t,(n=s).prototype=Object.create(i.prototype),n.prototype.constructor=n,n.__proto__=i;var r=s.prototype;return r.isWithContent=function(){return this.getTitle()||this._getContent()},r.addAttachmentClass=function(t){e(this.getTipElement()).addClass("bs-popover-"+t)},r.getTipElement=function(){return this.tip=this.tip||e(this.config.template)[0],this.tip},r.setContent=function(){var t=e(this.getTipElement());this.setElementContent(t.find(".popover-header"),this.getTitle());var n=this._getContent();"function"==typeof n&&(n=n.call(this.element)),this.setElementContent(t.find(".popover-body"),n),t.removeClass("fade show")},r._getContent=function(){return this.element.getAttribute("data-content")||this.config.content},r._cleanTipClass=function(){var t=e(this.getTipElement()),n=t.attr("class").match(Z);null!==n&&n.length>0&&t.removeClass(n.join(""))},s._jQueryInterface=function(t){return this.each((function(){var n=e(this).data("bs.popover"),i="object"==typeof t?t:null;if((n||!/dispose|hide/.test(t))&&(n||(n=new s(this,i),e(this).data("bs.popover",n)),"string"==typeof t)){if("undefined"==typeof n[t])throw new TypeError('No method named "'+t+'"');n[t]()}}))},o(s,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return tt}},{key:"NAME",get:function(){return J}},{key:"DATA_KEY",get:function(){return"bs.popover"}},{key:"Event",get:function(){return nt}},{key:"EVENT_KEY",get:function(){return".bs.popover"}},{key:"DefaultType",get:function(){return et}}]),s}($);e.fn[J]=it._jQueryInterface,e.fn[J].Constructor=it,e.fn[J].noConflict=function(){return e.fn[J]=G,it._jQueryInterface};var ot="scrollspy",st=e.fn[ot],rt={offset:10,method:"auto",target:""},at={offset:"number",method:"string",target:"(string|element)"},lt=function(){function t(t,n){var i=this;this._element=t,this._scrollElement="BODY"===t.tagName?window:t,this._config=this._getConfig(n),this._selector=this._config.target+" .nav-link,"+this._config.target+" .list-group-item,"+this._config.target+" .dropdown-item",this._offsets=[],this._targets=[],this._activeTarget=null,this._scrollHeight=0,e(this._scrollElement).on("scroll.bs.scrollspy",(function(t){return i._process(t)})),this.refresh(),this._process()}var n=t.prototype;return n.refresh=function(){var t=this,n=this._scrollElement===this._scrollElement.window?"offset":"position",i="auto"===this._config.method?n:this._config.method,o="position"===i?this._getScrollTop():0;this._offsets=[],this._targets=[],this._scrollHeight=this._getScrollHeight(),[].slice.call(document.querySelectorAll(this._selector)).map((function(t){var n,s=c.getSelectorFromElement(t);if(s&&(n=document.querySelector(s)),n){var r=n.getBoundingClientRect();if(r.width||r.height)return[e(n)[i]().top+o,s]}return null})).filter((function(t){return t})).sort((function(t,e){return t[0]-e[0]})).forEach((function(e){t._offsets.push(e[0]),t._targets.push(e[1])}))},n.dispose=function(){e.removeData(this._element,"bs.scrollspy"),e(this._scrollElement).off(".bs.scrollspy"),this._element=null,this._scrollElement=null,this._config=null,this._selector=null,this._offsets=null,this._targets=null,this._activeTarget=null,this._scrollHeight=null},n._getConfig=function(t){if("string"!=typeof(t=a(a({},rt),"object"==typeof t&&t?t:{})).target&&c.isElement(t.target)){var n=e(t.target).attr("id");n||(n=c.getUID(ot),e(t.target).attr("id",n)),t.target="#"+n}return c.typeCheckConfig(ot,t,at),t},n._getScrollTop=function(){return this._scrollElement===window?this._scrollElement.pageYOffset:this._scrollElement.scrollTop},n._getScrollHeight=function(){return this._scrollElement.scrollHeight||Math.max(document.body.scrollHeight,document.documentElement.scrollHeight)},n._getOffsetHeight=function(){return this._scrollElement===window?window.innerHeight:this._scrollElement.getBoundingClientRect().height},n._process=function(){var t=this._getScrollTop()+this._config.offset,e=this._getScrollHeight(),n=this._config.offset+e-this._getOffsetHeight();if(this._scrollHeight!==e&&this.refresh(),t>=n){var i=this._targets[this._targets.length-1];this._activeTarget!==i&&this._activate(i)}else{if(this._activeTarget&&t0)return this._activeTarget=null,void this._clear();for(var o=this._offsets.length;o--;){this._activeTarget!==this._targets[o]&&t>=this._offsets[o]&&("undefined"==typeof this._offsets[o+1]||t li > .active":".active";i=(i=e.makeArray(e(o).find(r)))[i.length-1]}var a=e.Event("hide.bs.tab",{relatedTarget:this._element}),l=e.Event("show.bs.tab",{relatedTarget:i});if(i&&e(i).trigger(a),e(this._element).trigger(l),!l.isDefaultPrevented()&&!a.isDefaultPrevented()){s&&(n=document.querySelector(s)),this._activate(this._element,o);var h=function(){var n=e.Event("hidden.bs.tab",{relatedTarget:t._element}),o=e.Event("shown.bs.tab",{relatedTarget:i});e(i).trigger(n),e(t._element).trigger(o)};n?this._activate(n,n.parentNode,h):h()}}},n.dispose=function(){e.removeData(this._element,"bs.tab"),this._element=null},n._activate=function(t,n,i){var o=this,s=(!n||"UL"!==n.nodeName&&"OL"!==n.nodeName?e(n).children(".active"):e(n).find("> li > .active"))[0],r=i&&s&&e(s).hasClass("fade"),a=function(){return o._transitionComplete(t,s,i)};if(s&&r){var l=c.getTransitionDurationFromElement(s);e(s).removeClass("show").one(c.TRANSITION_END,a).emulateTransitionEnd(l)}else a()},n._transitionComplete=function(t,n,i){if(n){e(n).removeClass("active");var o=e(n.parentNode).find("> .dropdown-menu .active")[0];o&&e(o).removeClass("active"),"tab"===n.getAttribute("role")&&n.setAttribute("aria-selected",!1)}if(e(t).addClass("active"),"tab"===t.getAttribute("role")&&t.setAttribute("aria-selected",!0),c.reflow(t),t.classList.contains("fade")&&t.classList.add("show"),t.parentNode&&e(t.parentNode).hasClass("dropdown-menu")){var s=e(t).closest(".dropdown")[0];if(s){var r=[].slice.call(s.querySelectorAll(".dropdown-toggle"));e(r).addClass("active")}t.setAttribute("aria-expanded",!0)}i&&i()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.tab");if(o||(o=new t(this),i.data("bs.tab",o)),"string"==typeof n){if("undefined"==typeof o[n])throw new TypeError('No method named "'+n+'"');o[n]()}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.tab.data-api",'[data-toggle="tab"], [data-toggle="pill"], [data-toggle="list"]',(function(t){t.preventDefault(),ht._jQueryInterface.call(e(this),"show")})),e.fn.tab=ht._jQueryInterface,e.fn.tab.Constructor=ht,e.fn.tab.noConflict=function(){return e.fn.tab=ct,ht._jQueryInterface};var ut=e.fn.toast,dt={animation:"boolean",autohide:"boolean",delay:"number"},ft={animation:!0,autohide:!0,delay:500},gt=function(){function t(t,e){this._element=t,this._config=this._getConfig(e),this._timeout=null,this._setListeners()}var n=t.prototype;return n.show=function(){var t=this,n=e.Event("show.bs.toast");if(e(this._element).trigger(n),!n.isDefaultPrevented()){this._config.animation&&this._element.classList.add("fade");var i=function(){t._element.classList.remove("showing"),t._element.classList.add("show"),e(t._element).trigger("shown.bs.toast"),t._config.autohide&&(t._timeout=setTimeout((function(){t.hide()}),t._config.delay))};if(this._element.classList.remove("hide"),c.reflow(this._element),this._element.classList.add("showing"),this._config.animation){var o=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,i).emulateTransitionEnd(o)}else i()}},n.hide=function(){if(this._element.classList.contains("show")){var t=e.Event("hide.bs.toast");e(this._element).trigger(t),t.isDefaultPrevented()||this._close()}},n.dispose=function(){clearTimeout(this._timeout),this._timeout=null,this._element.classList.contains("show")&&this._element.classList.remove("show"),e(this._element).off("click.dismiss.bs.toast"),e.removeData(this._element,"bs.toast"),this._element=null,this._config=null},n._getConfig=function(t){return t=a(a(a({},ft),e(this._element).data()),"object"==typeof t&&t?t:{}),c.typeCheckConfig("toast",t,this.constructor.DefaultType),t},n._setListeners=function(){var t=this;e(this._element).on("click.dismiss.bs.toast",'[data-dismiss="toast"]',(function(){return t.hide()}))},n._close=function(){var t=this,n=function(){t._element.classList.add("hide"),e(t._element).trigger("hidden.bs.toast")};if(this._element.classList.remove("show"),this._config.animation){var i=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,n).emulateTransitionEnd(i)}else n()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.toast");if(o||(o=new t(this,"object"==typeof n&&n),i.data("bs.toast",o)),"string"==typeof n){if("undefined"==typeof o[n])throw new TypeError('No method named "'+n+'"');o[n](this)}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"DefaultType",get:function(){return dt}},{key:"Default",get:function(){return ft}}]),t}();e.fn.toast=gt._jQueryInterface,e.fn.toast.Constructor=gt,e.fn.toast.noConflict=function(){return e.fn.toast=ut,gt._jQueryInterface},t.Alert=d,t.Button=g,t.Carousel=E,t.Collapse=D,t.Dropdown=j,t.Modal=R,t.Popover=it,t.Scrollspy=lt,t.Tab=ht,t.Toast=gt,t.Tooltip=$,t.Util=c,Object.defineProperty(t,"__esModule",{value:!0})})); -//# sourceMappingURL=bootstrap.min.js.map \ No newline at end of file diff --git a/imgr/imgrpkg/static-content/jquery.min.js b/imgr/imgrpkg/static-content/jquery.min.js deleted file mode 100644 index 36b4e1a1..00000000 --- a/imgr/imgrpkg/static-content/jquery.min.js +++ /dev/null @@ -1,2 +0,0 @@ -/*! jQuery v3.5.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-deprecated/ajax-event-alias,-effects,-effects/Tween,-effects/animatedSelector | (c) JS Foundation and other contributors | jquery.org/license */ -!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(g,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,v=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,y=n.hasOwnProperty,a=y.toString,l=a.call(Object),m={},b=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},w=g.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function C(e,t,n){var r,i,o=(n=n||w).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function T(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-deprecated/ajax-event-alias,-effects,-effects/Tween,-effects/animatedSelector",E=function(e,t){return new E.fn.init(e,t)};function d(e){var t=!!e&&"length"in e&&e.length,n=T(e);return!b(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+R+")"+R+"*"),U=new RegExp(R+"|>"),V=new RegExp(W),X=new RegExp("^"+B+"$"),Q={ID:new RegExp("^#("+B+")"),CLASS:new RegExp("^\\.("+B+")"),TAG:new RegExp("^("+B+"|[*])"),ATTR:new RegExp("^"+M),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+R+"*(even|odd|(([+-]|)(\\d*)n|)"+R+"*(?:([+-]|)"+R+"*(\\d+)|))"+R+"*\\)|)","i"),bool:new RegExp("^(?:"+I+")$","i"),needsContext:new RegExp("^"+R+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+R+"*((?:-\\d)?\\d*)"+R+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,G=/^(?:input|select|textarea|button)$/i,K=/^h\d$/i,J=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+R+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){C()},ae=xe(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{O.apply(t=P.call(d.childNodes),d.childNodes),t[d.childNodes.length].nodeType}catch(e){O={apply:t.length?function(e,t){q.apply(e,P.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,d=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==d&&9!==d&&11!==d)return n;if(!r&&(C(e),e=e||T,E)){if(11!==d&&(u=Z.exec(t)))if(i=u[1]){if(9===d){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return O.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&p.getElementsByClassName&&e.getElementsByClassName)return O.apply(n,e.getElementsByClassName(i)),n}if(p.qsa&&!k[t+" "]&&(!v||!v.test(t))&&(1!==d||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===d&&(U.test(t)||_.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&p.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=A)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+be(l[o]);c=l.join(",")}try{return O.apply(n,f.querySelectorAll(c)),n}catch(e){k(t,!0)}finally{s===A&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>x.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[A]=!0,e}function ce(e){var t=T.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)x.attrHandle[n[r]]=t}function de(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function pe(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in p=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},C=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:d;return r!=T&&9===r.nodeType&&r.documentElement&&(a=(T=r).documentElement,E=!i(T),d!=T&&(n=T.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),p.scope=ce(function(e){return a.appendChild(e).appendChild(T.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),p.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),p.getElementsByTagName=ce(function(e){return e.appendChild(T.createComment("")),!e.getElementsByTagName("*").length}),p.getElementsByClassName=J.test(T.getElementsByClassName),p.getById=ce(function(e){return a.appendChild(e).id=A,!T.getElementsByName||!T.getElementsByName(A).length}),p.getById?(x.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},x.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(x.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},x.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),x.find.TAG=p.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):p.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},x.find.CLASS=p.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(p.qsa=J.test(T.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+R+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+R+"*(?:value|"+I+")"),e.querySelectorAll("[id~="+A+"-]").length||v.push("~="),(t=T.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+R+"*name"+R+"*="+R+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+A+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=T.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+R+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(p.matchesSelector=J.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){p.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",W)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=J.test(a.compareDocumentPosition),y=t||J.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!p.sortDetached&&t.compareDocumentPosition(e)===n?e==T||e.ownerDocument==d&&y(d,e)?-1:t==T||t.ownerDocument==d&&y(d,t)?1:u?H(u,e)-H(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==T?-1:t==T?1:i?-1:o?1:u?H(u,e)-H(u,t):0;if(i===o)return de(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?de(a[r],s[r]):a[r]==d?-1:s[r]==d?1:0}),T},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(C(e),p.matchesSelector&&E&&!k[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||p.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){k(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return Q.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&V.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+R+")"+e+"("+R+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return b(n)?E.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?E.grep(e,function(e){return e===n!==r}):"string"!=typeof n?E.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(E.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||L,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:j.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof E?t[0]:t,E.merge(this,E.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:w,!0)),k.test(r[1])&&E.isPlainObject(t))for(r in t)b(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=w.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):b(e)?void 0!==n.ready?n.ready(e):e(E):E.makeArray(e,this)}).prototype=E.fn,L=E(w);var q=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}E.fn.extend({has:function(e){var t=E(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,pe=/^$|^module$|\/(?:java|ecma)script/i;le=w.createDocumentFragment().appendChild(w.createElement("div")),(ce=w.createElement("input")).setAttribute("type","radio"),ce.setAttribute("checked","checked"),ce.setAttribute("name","t"),le.appendChild(ce),m.checkClone=le.cloneNode(!0).cloneNode(!0).lastChild.checked,le.innerHTML="",m.noCloneChecked=!!le.cloneNode(!0).lastChild.defaultValue,le.innerHTML="",m.option=!!le.lastChild;var he={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ge(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&S(e,t)?E.merge([e],n):n}function ve(e,t){for(var n=0,r=e.length;n",""]);var ye=/<|&#?\w+;/;function me(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),d=[],p=0,h=e.length;p\s*$/g;function Le(e,t){return S(e,"table")&&S(11!==t.nodeType?t:t.firstChild,"tr")&&E(e).children("tbody")[0]||e}function je(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n
",2===ft.childNodes.length),E.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(m.createHTMLDocument?((r=(t=w.implementation.createHTMLDocument("")).createElement("base")).href=w.location.href,t.head.appendChild(r)):t=w),o=!n&&[],(i=k.exec(e))?[t.createElement(i[1])]:(i=me([e],t,o),o&&o.length&&E(o).remove(),E.merge([],i.childNodes)));var r,i,o},E.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=E.css(e,"position"),c=E(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=E.css(e,"top"),u=E.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),b(t)&&(t=t.call(e,n,E.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},E.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){E.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===E.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===E.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=E(e).offset()).top+=E.css(e,"borderTopWidth",!0),i.left+=E.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-E.css(r,"marginTop",!0),left:t.left-i.left-E.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===E.css(e,"position"))e=e.offsetParent;return e||re})}}),E.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;E.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),E.each(["top","left"],function(e,n){E.cssHooks[n]=Fe(m.pixelPosition,function(e,t){if(t)return t=We(e,n),Ie.test(t)?E(e).position()[n]+"px":t})}),E.each({Height:"height",Width:"width"},function(a,s){E.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){E.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?E.css(e,t,i):E.style(e,t,n,i)},s,n?e:void 0,n)}})}),E.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),E.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){E.fn[n]=function(e,t){return 0': '>', - '"': '"', - '\'': ''', - '/': '/' - }; - - var defaultSettings = { - indent: 2 - }; - - var id = 0; - var instances = 0; - var current_collapse_level = null; - var ids_to_collapse = []; - - this.create = function(data, settings, collapse_depth) { - current_collapse_level = typeof collapse_depth !== 'undefined' ? collapse_depth : -1; - instances += 1; - return _span(_jsVal(data, 0, false), {class: 'jstValue'}); - }; - - this.collapse = function() { - var arrayLength = ids_to_collapse.length; - for (var i = 0; i < arrayLength; i++) { - JSONTree.toggle(ids_to_collapse[i]); - } - }; - - var _escape = function(text) { - return text.replace(/[&<>'"]/g, function(c) { - return escapeMap[c]; - }); - }; - - var _id = function() { - return instances + '_' + id++; - }; - - var _lastId = function() { - return instances + '_' + (id - 1); - }; - - var _jsVal = function(value, depth, indent) { - if (value !== null) { - var type = typeof value; - switch (type) { - case 'boolean': - return _jsBool(value, indent ? depth : 0); - case 'number': - return _jsNum(value, indent ? depth : 0); - case 'string': - return _jsStr(value, indent ? depth : 0); - default: - if (value instanceof Array) { - return _jsArr(value, depth, indent); - } else { - return _jsObj(value, depth, indent); - } - } - } else { - return _jsNull(indent ? depth : 0); - } - }; - - var _jsObj = function(object, depth, indent) { - var id = _id(); - _decrementCollapseLevel("_jsObj"); - var content = Object.keys(object).map(function(property) { - return _property(property, object[property], depth + 1, true); - }).join(_comma()); - var body = [ - _openBracket('{', indent ? depth : 0, id), - _span(content, {id: id}), - _closeBracket('}', depth) - ].join('\n'); - _incrementCollapseLevel("_jsObj"); - return _span(body, {}); - }; - - var _jsArr = function(array, depth, indent) { - var id = _id(); - _decrementCollapseLevel("_jsArr"); - var body = array.map(function(element) { - return _jsVal(element, depth + 1, true); - }).join(_comma()); - var arr = [ - _openBracket('[', indent ? depth : 0, id), - _span(body, {id: id}), - _closeBracket(']', depth) - ].join('\n'); - _incrementCollapseLevel("_jsArr"); - return arr; - }; - - var _jsStr = function(value, depth) { - var jsonString = _escape(JSON.stringify(value)); - return _span(_indent(jsonString, depth), {class: 'jstStr'}); - }; - - var _jsNum = function(value, depth) { - return _span(_indent(value, depth), {class: 'jstNum'}); - }; - - var _jsBool = function(value, depth) { - return _span(_indent(value, depth), {class: 'jstBool'}); - }; - - var _jsNull = function(depth) { - return _span(_indent('null', depth), {class: 'jstNull'}); - }; - - var _property = function(name, value, depth) { - var property = _indent(_escape(JSON.stringify(name)) + ': ', depth); - var propertyValue = _span(_jsVal(value, depth, false), {}); - return _span(property + propertyValue, {class: 'jstProperty'}); - }; - - var _comma = function() { - return _span(',\n', {class: 'jstComma'}); - }; - - var _span = function(value, attrs) { - return _tag('span', attrs, value); - }; - - var _tag = function(tag, attrs, content) { - return '<' + tag + Object.keys(attrs).map(function(attr) { - return ' ' + attr + '="' + attrs[attr] + '"'; - }).join('') + '>' + - content + - ''; - }; - - var _openBracket = function(symbol, depth, id) { - return ( - _span(_indent(symbol, depth), {class: 'jstBracket'}) + - _span('', {class: 'jstFold', onclick: 'JSONTree.toggle(\'' + id + '\')'}) - ); - }; - - this.toggle = function(id) { - var element = document.getElementById(id); - var parent = element.parentNode; - var toggleButton = element.previousElementSibling; - if (element.className === '') { - element.className = 'jstHiddenBlock'; - parent.className = 'jstFolded'; - toggleButton.className = 'jstExpand'; - } else { - element.className = ''; - parent.className = ''; - toggleButton.className = 'jstFold'; - } - }; - - var _closeBracket = function(symbol, depth) { - return _span(_indent(symbol, depth), {}); - }; - - var _indent = function(value, depth) { - return Array((depth * 2) + 1).join(' ') + value; - }; - - var _decrementCollapseLevel = function(caller) { - if (current_collapse_level <= 0) { - ids_to_collapse.push(_lastId()); - } else { - } - current_collapse_level--; - }; - - var _incrementCollapseLevel = function(caller) { - current_collapse_level++; - }; - - return this; -})(); diff --git a/imgr/imgrpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css b/imgr/imgrpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css deleted file mode 100755 index 4664f2e8..00000000 --- a/imgr/imgrpkg/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css +++ /dev/null @@ -1 +0,0 @@ -@font-face{font-family:Icons;src:url(../fonts/open-iconic.eot);src:url(../fonts/open-iconic.eot?#iconic-sm) format('embedded-opentype'),url(../fonts/open-iconic.woff) format('woff'),url(../fonts/open-iconic.ttf) format('truetype'),url(../fonts/open-iconic.otf) format('opentype'),url(../fonts/open-iconic.svg#iconic-sm) format('svg');font-weight:400;font-style:normal}.oi{position:relative;top:1px;display:inline-block;speak:none;font-family:Icons;font-style:normal;font-weight:400;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.oi:empty:before{width:1em;text-align:center;box-sizing:content-box}.oi.oi-align-center:before{text-align:center}.oi.oi-align-left:before{text-align:left}.oi.oi-align-right:before{text-align:right}.oi.oi-flip-horizontal:before{-webkit-transform:scale(-1,1);-ms-transform:scale(-1,1);transform:scale(-1,1)}.oi.oi-flip-vertical:before{-webkit-transform:scale(1,-1);-ms-transform:scale(-1,1);transform:scale(1,-1)}.oi.oi-flip-horizontal-vertical:before{-webkit-transform:scale(-1,-1);-ms-transform:scale(-1,1);transform:scale(-1,-1)}.oi-account-login:before{content:'\e000'}.oi-account-logout:before{content:'\e001'}.oi-action-redo:before{content:'\e002'}.oi-action-undo:before{content:'\e003'}.oi-align-center:before{content:'\e004'}.oi-align-left:before{content:'\e005'}.oi-align-right:before{content:'\e006'}.oi-aperture:before{content:'\e007'}.oi-arrow-bottom:before{content:'\e008'}.oi-arrow-circle-bottom:before{content:'\e009'}.oi-arrow-circle-left:before{content:'\e00a'}.oi-arrow-circle-right:before{content:'\e00b'}.oi-arrow-circle-top:before{content:'\e00c'}.oi-arrow-left:before{content:'\e00d'}.oi-arrow-right:before{content:'\e00e'}.oi-arrow-thick-bottom:before{content:'\e00f'}.oi-arrow-thick-left:before{content:'\e010'}.oi-arrow-thick-right:before{content:'\e011'}.oi-arrow-thick-top:before{content:'\e012'}.oi-arrow-top:before{content:'\e013'}.oi-audio-spectrum:before{content:'\e014'}.oi-audio:before{content:'\e015'}.oi-badge:before{content:'\e016'}.oi-ban:before{content:'\e017'}.oi-bar-chart:before{content:'\e018'}.oi-basket:before{content:'\e019'}.oi-battery-empty:before{content:'\e01a'}.oi-battery-full:before{content:'\e01b'}.oi-beaker:before{content:'\e01c'}.oi-bell:before{content:'\e01d'}.oi-bluetooth:before{content:'\e01e'}.oi-bold:before{content:'\e01f'}.oi-bolt:before{content:'\e020'}.oi-book:before{content:'\e021'}.oi-bookmark:before{content:'\e022'}.oi-box:before{content:'\e023'}.oi-briefcase:before{content:'\e024'}.oi-british-pound:before{content:'\e025'}.oi-browser:before{content:'\e026'}.oi-brush:before{content:'\e027'}.oi-bug:before{content:'\e028'}.oi-bullhorn:before{content:'\e029'}.oi-calculator:before{content:'\e02a'}.oi-calendar:before{content:'\e02b'}.oi-camera-slr:before{content:'\e02c'}.oi-caret-bottom:before{content:'\e02d'}.oi-caret-left:before{content:'\e02e'}.oi-caret-right:before{content:'\e02f'}.oi-caret-top:before{content:'\e030'}.oi-cart:before{content:'\e031'}.oi-chat:before{content:'\e032'}.oi-check:before{content:'\e033'}.oi-chevron-bottom:before{content:'\e034'}.oi-chevron-left:before{content:'\e035'}.oi-chevron-right:before{content:'\e036'}.oi-chevron-top:before{content:'\e037'}.oi-circle-check:before{content:'\e038'}.oi-circle-x:before{content:'\e039'}.oi-clipboard:before{content:'\e03a'}.oi-clock:before{content:'\e03b'}.oi-cloud-download:before{content:'\e03c'}.oi-cloud-upload:before{content:'\e03d'}.oi-cloud:before{content:'\e03e'}.oi-cloudy:before{content:'\e03f'}.oi-code:before{content:'\e040'}.oi-cog:before{content:'\e041'}.oi-collapse-down:before{content:'\e042'}.oi-collapse-left:before{content:'\e043'}.oi-collapse-right:before{content:'\e044'}.oi-collapse-up:before{content:'\e045'}.oi-command:before{content:'\e046'}.oi-comment-square:before{content:'\e047'}.oi-compass:before{content:'\e048'}.oi-contrast:before{content:'\e049'}.oi-copywriting:before{content:'\e04a'}.oi-credit-card:before{content:'\e04b'}.oi-crop:before{content:'\e04c'}.oi-dashboard:before{content:'\e04d'}.oi-data-transfer-download:before{content:'\e04e'}.oi-data-transfer-upload:before{content:'\e04f'}.oi-delete:before{content:'\e050'}.oi-dial:before{content:'\e051'}.oi-document:before{content:'\e052'}.oi-dollar:before{content:'\e053'}.oi-double-quote-sans-left:before{content:'\e054'}.oi-double-quote-sans-right:before{content:'\e055'}.oi-double-quote-serif-left:before{content:'\e056'}.oi-double-quote-serif-right:before{content:'\e057'}.oi-droplet:before{content:'\e058'}.oi-eject:before{content:'\e059'}.oi-elevator:before{content:'\e05a'}.oi-ellipses:before{content:'\e05b'}.oi-envelope-closed:before{content:'\e05c'}.oi-envelope-open:before{content:'\e05d'}.oi-euro:before{content:'\e05e'}.oi-excerpt:before{content:'\e05f'}.oi-expand-down:before{content:'\e060'}.oi-expand-left:before{content:'\e061'}.oi-expand-right:before{content:'\e062'}.oi-expand-up:before{content:'\e063'}.oi-external-link:before{content:'\e064'}.oi-eye:before{content:'\e065'}.oi-eyedropper:before{content:'\e066'}.oi-file:before{content:'\e067'}.oi-fire:before{content:'\e068'}.oi-flag:before{content:'\e069'}.oi-flash:before{content:'\e06a'}.oi-folder:before{content:'\e06b'}.oi-fork:before{content:'\e06c'}.oi-fullscreen-enter:before{content:'\e06d'}.oi-fullscreen-exit:before{content:'\e06e'}.oi-globe:before{content:'\e06f'}.oi-graph:before{content:'\e070'}.oi-grid-four-up:before{content:'\e071'}.oi-grid-three-up:before{content:'\e072'}.oi-grid-two-up:before{content:'\e073'}.oi-hard-drive:before{content:'\e074'}.oi-header:before{content:'\e075'}.oi-headphones:before{content:'\e076'}.oi-heart:before{content:'\e077'}.oi-home:before{content:'\e078'}.oi-image:before{content:'\e079'}.oi-inbox:before{content:'\e07a'}.oi-infinity:before{content:'\e07b'}.oi-info:before{content:'\e07c'}.oi-italic:before{content:'\e07d'}.oi-justify-center:before{content:'\e07e'}.oi-justify-left:before{content:'\e07f'}.oi-justify-right:before{content:'\e080'}.oi-key:before{content:'\e081'}.oi-laptop:before{content:'\e082'}.oi-layers:before{content:'\e083'}.oi-lightbulb:before{content:'\e084'}.oi-link-broken:before{content:'\e085'}.oi-link-intact:before{content:'\e086'}.oi-list-rich:before{content:'\e087'}.oi-list:before{content:'\e088'}.oi-location:before{content:'\e089'}.oi-lock-locked:before{content:'\e08a'}.oi-lock-unlocked:before{content:'\e08b'}.oi-loop-circular:before{content:'\e08c'}.oi-loop-square:before{content:'\e08d'}.oi-loop:before{content:'\e08e'}.oi-magnifying-glass:before{content:'\e08f'}.oi-map-marker:before{content:'\e090'}.oi-map:before{content:'\e091'}.oi-media-pause:before{content:'\e092'}.oi-media-play:before{content:'\e093'}.oi-media-record:before{content:'\e094'}.oi-media-skip-backward:before{content:'\e095'}.oi-media-skip-forward:before{content:'\e096'}.oi-media-step-backward:before{content:'\e097'}.oi-media-step-forward:before{content:'\e098'}.oi-media-stop:before{content:'\e099'}.oi-medical-cross:before{content:'\e09a'}.oi-menu:before{content:'\e09b'}.oi-microphone:before{content:'\e09c'}.oi-minus:before{content:'\e09d'}.oi-monitor:before{content:'\e09e'}.oi-moon:before{content:'\e09f'}.oi-move:before{content:'\e0a0'}.oi-musical-note:before{content:'\e0a1'}.oi-paperclip:before{content:'\e0a2'}.oi-pencil:before{content:'\e0a3'}.oi-people:before{content:'\e0a4'}.oi-person:before{content:'\e0a5'}.oi-phone:before{content:'\e0a6'}.oi-pie-chart:before{content:'\e0a7'}.oi-pin:before{content:'\e0a8'}.oi-play-circle:before{content:'\e0a9'}.oi-plus:before{content:'\e0aa'}.oi-power-standby:before{content:'\e0ab'}.oi-print:before{content:'\e0ac'}.oi-project:before{content:'\e0ad'}.oi-pulse:before{content:'\e0ae'}.oi-puzzle-piece:before{content:'\e0af'}.oi-question-mark:before{content:'\e0b0'}.oi-rain:before{content:'\e0b1'}.oi-random:before{content:'\e0b2'}.oi-reload:before{content:'\e0b3'}.oi-resize-both:before{content:'\e0b4'}.oi-resize-height:before{content:'\e0b5'}.oi-resize-width:before{content:'\e0b6'}.oi-rss-alt:before{content:'\e0b7'}.oi-rss:before{content:'\e0b8'}.oi-script:before{content:'\e0b9'}.oi-share-boxed:before{content:'\e0ba'}.oi-share:before{content:'\e0bb'}.oi-shield:before{content:'\e0bc'}.oi-signal:before{content:'\e0bd'}.oi-signpost:before{content:'\e0be'}.oi-sort-ascending:before{content:'\e0bf'}.oi-sort-descending:before{content:'\e0c0'}.oi-spreadsheet:before{content:'\e0c1'}.oi-star:before{content:'\e0c2'}.oi-sun:before{content:'\e0c3'}.oi-tablet:before{content:'\e0c4'}.oi-tag:before{content:'\e0c5'}.oi-tags:before{content:'\e0c6'}.oi-target:before{content:'\e0c7'}.oi-task:before{content:'\e0c8'}.oi-terminal:before{content:'\e0c9'}.oi-text:before{content:'\e0ca'}.oi-thumb-down:before{content:'\e0cb'}.oi-thumb-up:before{content:'\e0cc'}.oi-timer:before{content:'\e0cd'}.oi-transfer:before{content:'\e0ce'}.oi-trash:before{content:'\e0cf'}.oi-underline:before{content:'\e0d0'}.oi-vertical-align-bottom:before{content:'\e0d1'}.oi-vertical-align-center:before{content:'\e0d2'}.oi-vertical-align-top:before{content:'\e0d3'}.oi-video:before{content:'\e0d4'}.oi-volume-high:before{content:'\e0d5'}.oi-volume-low:before{content:'\e0d6'}.oi-volume-off:before{content:'\e0d7'}.oi-warning:before{content:'\e0d8'}.oi-wifi:before{content:'\e0d9'}.oi-wrench:before{content:'\e0da'}.oi-x:before{content:'\e0db'}.oi-yen:before{content:'\e0dc'}.oi-zoom-in:before{content:'\e0dd'}.oi-zoom-out:before{content:'\e0de'} \ No newline at end of file diff --git a/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.eot b/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.eot deleted file mode 100755 index f98177dbf711863eff7c90f84d5d419d02d99ba8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28196 zcmdsfdwg8gedj&r&QluAL-W#Wq&pgEMvsv!&0Cf&+mau`20w)Dj4&8Iu59zN6=RG; z451+<)Ej~^SrrmCp$=hb!Zu?PlZ0v^rFqOYfzqruY1s`+ve{(Uv}w|M+teR4-tX_6 zJJQHDgm(Majx=-5J@?%6_?_SRz0Ykss3^zpP!y(cg+5#{t0IGvlZlxgLVa!|Pwg%0HwaAkJPsR_7CkF z{hz=5BS2$bQO4>H%uMR+@Bes%qU=0}`qqrY1!(P0t>lnf>u?>hCHF7DiD%jIRLs_gA0(b1L}rzgltYVrt?gc2Y5;9UDjQ z%B)P;{Yp$h?WOgkCosju&-Q&Abmg0GDQ~^0YA77V?+nuN;!-_LToFFdx5>D-3RhIC zNim@Y28=&kzxC#&OZZhTUDD)z++voc1{on3eJelI&j0@(PPn1`HTMH@R>gMK0^H#} z-APZ<6H9s`4L|t$XFtpR3vV~DpGXL)8ZghQI8nFC#;Gm~d%|gaTbMPC42!c1B?miM zn$?TN(kwg4=NH!N?1DZwr|Va=QM0@at3QmtSVbGuP_f*EuIqDh*>o`umty&fMPWVN zwOSy=lGa!#OKqKlS=4KL6^YiDEHv;MA!Dj|%KqdbXOLRkVPgo+>xM z`tdLxr03~jdXO4;l(4}>Kca7fS2gy1&DtubqsnG6amCcr?ZNni_*#ur)!una=lO+a z(W#N+^Oy#G-fw#XCIlD!Q7hD3IjwB$Uoy5LHCCk7M6R+q+PRlLC+2F#Og&0KX;fTm z9gRV6t=nO-P_Az=CG4l*~#0dwv=AFvG8)~&n&z! z>wcqjdUo&ccd;$(NdM=j`265c&L?J1yxG?F>}_{_wry>?^aan|yPK}R#cpg(b^$xz zf;Gl2?&aw=%jBtFht&{S}(z)fW6^mCJSIuQ@i4|p+ zx3$z#v51krkNGj$t;x!E@Z?f6a(ZZoC>r5@Ucl5$FlAy4?Q*}B&hb1!m&U%lE*Euc z#N62h7Dtl~c7f-y5Wr$VDS7_#wX$QaKmmSK`iqLyDz`g-`54&Z80Kl-ofTt{b;TI$ zT#%ThARiNAa&`dV8`oF>zV?w_b1QPe8_mRA%fyml9N}zE z_-m(6zyG|m?j+Mnf7=xbb%mHqB&x=o>~}ut(o3hDKA)2v)LFgfzUPV|zwQq${}Jm! zdvqS0#f$auxa~yCyx|1clRx73VPI)bD(DG&?EH&%UAHgnwu8I!`Kp(SFWc>Wqg^Ma zTe*j+Ez4Kzf`(q!&Qco{4bZc|i%U<6aYU6B7)Lx7;53d@W>5_ia)5Ny1_i;Fuu5e! z-gKnZ5^0T^BYvyJ8eYL}Z1AdPGrK^uOnkDgwNvdLC@Di@t#zMFFbngC*yBaZnjCxO zZVNwAs{vvUm;SyZn;h!w92-hzJ6O%btT}YL>chAEtV)iFcrVtkM#9EvCDS2-twqu&y5y= zw;q?%OgQCDn!(c|X=^MS%LcRltks{LOR&8^`AO+?V#}7fxh-2D&&;XX#mAnwc+n^T z?I3bku^;?ONNGpAEzQ9|wZK)t4otF{`3c3+*b1IhG!ph>Qy^76GG!OWj>gw*J9S{; z4GguD#dS*bxuJZ1h^DeJ+j4C4fm1qeo$MT>2@;LZAJ13vO*7V9&^G2tG7zXZ?FfUm z#SMB%w5<{KY9(%XvO$a>;P-@EExte!yNWhJc8Fzlj6qNMLkn-vTJq?^8$)^3(jB7q zK=I-s|H2zsK0QCgqux+AWHJJLC*aI54Qv=}8o8CR zZwEnEGeI;95)@8khtt_i7IdVSr-7d=zV}u=kyugRRIfhw zeDDVL_QJF74|wmnm%D6ymv^z?^V}7hzydG+3&|d1l55zYhOj3av4&o`Cs_*%Sec7K6kNmX1R1PD zYix+tfd4N`+-xrWgR9=NE#s(Rcb7VHTc13*dDZG`u2Vy5+-xoVUX3HO%~S7URi&d_ za|fSnjU2xwx0TQZaKH4&{58k8C}uC~%bS*!t{HKh8i(U_G87Y4V6Mbq6(WCwXB8|!8EMz7QHK&Z*mcFpc< z+RRN&4^&tAL+^tIcvp=oXtiyp&{<>WDx_onB*c$TJG+1&G7a-fJb(lhUsyZ?n4aYuiGF!~%5BNht zkLp&(Oy-jvTIYsHHM$C!I<(f1-`DJlUJRPI*qqTW+kTY1z~}7?FWT8-kChzvs)6UdU2dnB zx$Q4tyPa>#r3G#wn2l*V56=aR2F{ncODvttVSQ>#9gal)dghYmi{bh)=H+FHv=R)hRtN(5RM_@E0? z5kM8i9$Uerye_+vY3w_3_P#}l!_lo1O@m<2iy=ee^_*n$LO%GqY8Q0?Zgjgfu%~GcgW`lM%ck$vJ0hs4ShNL&iUr07ttjmJdpcTs@YpWWi zLeN`YSMXY|ok4QJ?b0l&5gLe$Y$tuGLVQ^KYqd>=*0HTNl+kS35%>Tm0`e`E!ED_IcN2j(%)=h7jWUMUO0+h zRRdK=F-j8tO~s;7T+L5ZJE`9#xx)%NSO@&}!yd9s-zo3*_M|@$v_@C3vckh1zbO=c zQz)I*Tce|GeeMd4hi+VZwk!ITF`O4lyst z4Y9otCo>pme1^Sp;8gd3{bk67rC&829rHZ0Sv4^W_lM?+#W|mfdf9!dfV9s|K;O|StI2k1ficm_+HH-M&Az?i*JgaZ@5^* zE(GBy_gO3&{S94&SP6KeFT!J~`_y882z_O7zCy_m6O~Qphe|_ZM`==gUbZ=u2Swa{ zc-fe%m1d0D?+|)|HxUHK2lEHO%w;$(wR`cy*WG%iYh_pcDb`1TTj~Ka=bd}qEvd|b zQ^m{sB3zJTR-u==fD1KM#C|~QSdzg!U=2oM?a81uk|lZ~xEUA=&kOD%%>%Gb(5GU} zTOiHa&bDc8$;Tnw1g$O1?*a*kxmaWcc5HS9ORvEu4`$0U9^0!Yn(iJ=IPSjNkr=(Z zDY5+W^zl3}LDjB$vt0K9RLLL5oR)B01*NRQyg(`CyrhZKYKCkpBzcJRl8dOC)PO3V zwaRCOc~t7^!d#+yVgv-}OF|o3m8R8-X8{D#>>(A*N?k%eEp2Xp{Og1~APhL#`%a==_CxDO?0Cstm3 z30%#eV0U(fut|VC7qL}fR)`ZvgHV2zC*{}rc8UrQR$o+3OBx1mZ zBw=TjS?FXCbR;9PLY)=VCY?28(R%*NYUev|5yJtCsjYSrP2lsA^AtqzGR9J<&#=SZlzmY*a6=bs1jPR3mA)Spy%lFF5 zROWpz3sBDaoT_RIIQP`UxG^?pxxq~=8DPB}F$ARVc7;st8!RO5cGmB4ZoCptXt$F* zCv5*@5{La6dkp?4(js8{AS3-dZwU(s)Cst!XwFM`ri$l@b{jSbv$P3IT0yOVSP=dS zw*x&V*WCoyCHggs=e+QPsqGa4jr6auy%nO1Ao}q)D@u%U$o8tSy3nH?Dvbl+CYu7R zr;${9Fe_A8p_~#-b)dOUM&F@rV13*8{M%o^J~;k`hJ4<8%LsADky~hvVqJxtWL9i& zd%G1Mt!u5vSyM$+o%}ek3E&T+d^?dS@rBYBXD1idLoy_TzhGTt(IHuqpa=xQPQX9) z0h)5@Nist!gP>qOtZ~ zMv}`QE9zVNwYYBcTms~PKGwK=(ESy}0lC<7k|w5-tgTAbC1>SlGFV{0;z+^k=% zP^`6tvGjFXO#;T4IOYvy2(y&V4OomZUoa&6Vs1-oEuS+>A1T9w;)~}99&%k-92Wn0 z#WQ5b|rc;Pr&qX~%&%}F#z(-avRX_b{G<+PY*7c;v8*q~hfsmb>XW+&kft>v*aLckMzT1J z?H52T$v0c|wF=q6AAu|`zT{OizHk$e;I$04CdhHNvo^$$PQGVNwOorbI=H7r;%%PvE>$cds9X%hLl`MJ6ID0UQ$ zMeHT$iSw|nEZP>KML>Fm^x}gE6TyOH{baI=g|o?MIs%(H=}Lgtd<{kFSU|8gs^G;wS0(6~;HoUQld?%1QRZPOq4L+V$^Kce3< zza;Al%6f$Xs zJ(ifhc0+%g-EIkP+x_5%O&`B;lgFbvI(tX2(;pCqr(#uYQ^?=!6x^22htq48xpO$v_M&$&HhkRZI$5SG*{TDTls&4?T2*ow$^%;=-wcMati4n z1CHQ>9wQCHD;N>p7-?idNGxoNs;bt2YwvLPeckc+x|?c4{(9F?>4DPUv%A;0{U0rT z_kOmD&oj?W>$p&VVcQqtdrO##R}$gZvxB^K55{&58Yt zJxOe?lC{aLO=P4@bLhDSp?60bYv?&Ikwm8{*lPk&G^LoJkdZLui?+rM>F(~;>w2o| zMK;_&(66yNkzdnZIw!7G&E(FlJ&^0YY17!o8++wN$M&_u>xQ?M7Ubo=DWd@UWC>?f zaBRpICMlP|)$9eavi2=$}kiDm__jweO@3rN;(HfCW16c9Drzu=v&AdeV|?K z)Hl>6;GWe_22rqia&JR(5=A5kv`TN7kZQ7Nx(gj9+tU~<`a?Zgk%=6%J-S;Vf)l z0Lt7Py8yV%l2=b$%8RSCQEe5x!D~D$o5J(-tk}HN7&Sr#rE{V&8p{&>vO=@mh5fr@ zQ*622sGaQeFjBNykn}REr5UPzt2F@U1^%tXhqD=YE_!)(NR36wpAto)W}`tTHWeJ$ z>Kc}gmd$AFZ|-gi@CbSTFbq6RJAy4%%b{gEY$%uTDdmFttp;N%I-l% z_DCo&{xE-elH$n7{aCg!AftazXDcW*!Ul!TUdgkhUm~V-!*`ujvXDvFDD7)ohgPl3 zWm1X0-gs9>w5?TZZfdBjTAsney4@_8{!`-jJF=) z!Ih4dvLfo`b6!xSXZ<1gZ}Sax-i2Gee9%xRy`{56px72K`EN^adc9{21=65bkhPMa zR}Dn3Al|?mA(VFLEopIu&Y`6UD>6tJS#HW#Rgp`MU*q7S=7Roe3s? zbg=ZL(wEq2hzDcPE1w=LJ;!!djFtF|h&6!Q0rm&jArNo?F@_L_;&0BWr8|IO@M|p5 zV^z@OMSa^7_Ik3gs==b^kpd(=UXG#yyApH&grKsGYS>(CXI*eP5|0)*5;5XqlEGv) z>GAT5Uhjg%i|r)ZqCAxW=_qVL;vCo@d{ur$1HGvFS~T1cs1i7rfLDhc3FNwt#^9_X z`3W{;p$@^_j3^24E}?yX_{*-JGFZvcEqWTGQ3FhTSQW5DIvH?aGyF zk3DtFNc2_PSEc&;QuIYu!pDfmBKavGX=2$iW)X~27!K12bis%qj}Q|O76PUUm*Ff- zh(K=yW32f=f-Gtf8ik+mT7n?g`{Fb;KX*699YJse1^RPncoAwWVN!L?8DcsO|&<8t7Kdq z`Q9J`nkB+!vSBC#S1)l1?-teTmXcyN2z!u8TG~Z)8QW1+P4O3{b27q$os{tyrP<}z zx7OA-`w?YU^oCs3PI!_{W{^hEMU?qN`~?|#F(>0GzkJ~2VzhR7p{k1)r2?m6sBWH{_0ElUbM_IgNLK-IGf3H)siHZ*NlW8BqDLfvrrdWs4Q)9dtse@ zdgUjCVS;eqtTrRor(4+x+}wGcodNd|HfhW?)@zo&Kqz^^fH7$!vL>6cBDm6s!HHpl z#=MPK9r)$MtSMq*b3{&d=aeH*<1sr~L&)!RxEiuaV}1e(iF*QComGb3c$)@#%l813 zpfU5g?P{nz=baV?-BPtdTWz*ha}(MUGZoWM{SRhCnFzkYoX}SJUdUO7!Q6JDaqr(o zLb8vfcTx_Lc_9mdGtxeS>Lq@OQ_38%N{X~2GqXscyW%7GGs(zgkD-Vgl572IYkT7z zkYbx4!@3a-Yf@}N*%Eqw7JY+R{MNh>gF=GJk+TUtTB4p;&mta7RDt|*^%O%D@{~bW zj5rfJQ`?DTU`|A(F)!2;bd*BO#H?&*-40?SRIJPwWee=&%AG603XhI~c)|FF{nSOFGh!?# z$5_gC)e2iJoat~E2P2Di)sxrX1@%rZu%q~ai52n-sVc2aS;J)k-@p zd;{Wy3fO83T!q5&L-ERaY7XE@%u(n#W=fLr#fwEffiJ}Ja(e<+LE<| zAKks(g4^Amu2r=T-DK~?6Q#RO-ipICub*04fAsAZ{tmxK*q(*0z{wFf2t!Mmg~HS< z>`uZ0#bj`lsuhmsPTqG=(;VIR-t}1S__ab%HRvO3wh`Qv~V zG&_H|9c+aQBq1r93w9*CE!)muNoGLTzeVug92sfn5XkrE$Maj-qZVJPLz8<%)fWDT zYO|`pyy$C&v*cMl#O}-w#qaIxfR$|J=B6QX#Ts!(SZYHyqH|Va4G|3|{NW@V%W!qt zet-|{BU!&P7E4MthFhYdjup5s;)wu1vE>0W{6qMs6irp&xM52#`!HY%^9b?-BDCbe zxT3yEmE)D3l9RN7s6GvaZ1A$ap@)-g-y;2CG(Ru%Kn)<@5P3$(YF{3Ys4sm1mF*`z zWJN{{f4O};u>=p;jThsI!xA9IeMQin>M|XGoeaHWV?;bj0bXenCTp2cMTEYoihVET z)k=SXLAtLHE$8)bgCWbk^CZ^uo50^ynC}X|!3)9CL!8!NHBV)%i$OWY;Q<)FNR5Mo z4G0$|PZum+RFegqHeo^SJ!b+lN01IFab2NDZcAX#&JK1aZhOSX=S_p1CPXYFPML>S z{t1QZBuJ+dieKX3Gqtx4c6JWlTKmkwgbd#yxGnlb7U3qvWdPWihk${mv|%2t;aZ_f zErt@qWwkU`(l?~sxh#bEA_&UDvxt>Oe1dPg3>+>wAcoRtAd+J3N%#cL(0DFAuU26n zES^bVhJ{)vSfFOi9XS8Yx-}iIfApF2kMsF8>z+9uIQIDYXFmEm@P_a}#%Khw&JNO3 z7{ZQ{X%IssbOJEqkCBHx!uFCK4rEXK<44fI@&%>k_5|L9(4Jeg2hEx^JvcAZChO9L zXUGK8BgJV18%zJ^ca5CMmp}G1PyqzQqs0E2t*dmW%(5p;&en#281ton$6v&pbEmcw=4n?au4S-Sy0OJ!_)R437?}-km!s`%H9AALC89lE}Q4u=a{lsF?svCed+$tOaa z7j01y!_E-)lp}n->@^&SN_b&c_#Gi1sao0GfB+13L7b4F;FcvjFxlAyXuB3Cz*OnS zLFh&Xup&LLHOAWIaWJ;Gp|13!8P;+CbFV)7;c4bB?f;u|8Jq=COLwx){kM8wdEn7k zcQE%~oIlrf&ql+pbLmMzUxg2m>^jTN?ub3@vBo@-2+8o<8-?zdFfJ=@giXjUz22DTppvsdH%LW6F|Deg9C$UdSM+ zp7x>W(CDkBH(v!RK|E#3)|M^z&|%-f{gIZfE&V6Q9)0!IN5@WzQ~pb9rV1&%>T3ZX z`D6q>&~aZGYfl21IG+XS6HKNw`!b@b?0XiT-D4M*6e4FY{oGzG+F64gv%yqkd`1Ny zq8KZR&sg-iQhbIXD9|A=I$A3-(&ZcZ!(Y^Fjs_FH{2%G9mVVYK`jKbF20-6h3|u3L3WtCZ?%+>khd2<9P#On9qR?tn zD3Q`R#3ncc!J<>KUS1s7Jz#gM>M!5}2?cAq2L`%pf+4FV@C#LS+sik_1<$|B-OC^4 zc~K&91~DqX1|25-$#%9k?h?EXv{($)X`)ya*weB@HV~>Po#eq8OdMbMCb%Whq zt->d?0gkZ?msD9O$U4ug~o53-O@Y zXY)D(L1$-uYkOUfV_X05!g^AJDrjj7EYO>jJw!`)Ub{9IZ>u7C6|__a{914>6a(r- zAdQtqM)(Y;zq%x0Tq$!HCGA(#kukJu`aN5E8$&hQ_ie8UH4b#7DV(;!5I-P$_+G5Y zv(FmA!*rt@$D7<<)0J}cuUXUYXkB@&h#z*4P$JCDMPmANCCx6lGA+BR*!x7Igsq!& zng~K&B|pbm9V?97=_G<(fuzEJJcu|49L9g*%a%Z~Sl_EX^8~_w^k+V=>UyvC#KSEs z5Zw;m{_<-o@%`vaFGcm&URL$!^UuTMWXKPK-uM^!eL^_$094|_*&whq>dvr}r|-VI zbncGvV~A$?O@8#qvtM}oZA8yf*&c}1D4`gv zO6G7O=P!87;&V8M?59KS=?E0SB7G~Uo{)jDpY!ktmHUC9gJandKaOyhDJ8*2JWXR; zqFYsXfeG=kfY(_q&NzA!ra&#WB5#Wz{F=hdkYX#IW}QF$Nb#xCUqAgCix$6p@7Pfc z;v+vS{pj@5%=eUDdgHZwzpNjH=DZ{aRDohqOagFMYYO@(FbTNpO_-?tUXFIb(H1*E zM`hE5{t_FW*KdC6zu)uF&mYv!KO+?APQyexUwY}Kd;a@VH|r1n{Gn&gOJ%!kC>3&` zSjRA6;Sq9MnD&ZP`jJv3l(dveW`K|@a{7}r4HRZ4Ni8Pn6tPJ#k9QV@o%CYqoRF@? z1&?-$bD~@TlI#PuIM0a~cyE=U8=wl{QDu`X+%lOkp)WQl+y+~I0)nr{TS`MM@i?dG z!Hu`OJ#Re$k`3kjUKFk-)zFzjPXGpqjQ0<5BRHvT`n68n1WDt$)8LXx794u=Jl9inhOTl zy4*tU3>eu#sT3Fv|_Nmk$>MddiLLcl?ftEQR)K?w&D2nwZuD7ZAh`NI%oX?s8k zMEAs_A-z8f?rCt%O1ysWHp@C9+BVuO+wo}IE^kwuTNAvv^5k5M&d#;BEuEgT8fWL0 z9aW)2tK^1}=hl|eE&K$b(ZW&u=HSjE^TXmVpU0gy%4kL=MS`L6Q%MJjmI&Jc^M!YV0ahT)5@ za9#<`svH+wRt?I;;PUeFb@@K~un?<%EPlC1B&DB=kR@r1F@m%gzFk>ER!6uB6>bv0 zWamU)Sd3)3EctQeU6GgcQ{XzSTRrG!5QiMChEIC=GQpYzT>vrtt^61r^j~-gzuVb` zAFm8Gt!h#=l(bPf|8ICxfYb;QiA3f8HDUKtEU^)LXy>qjibDbva|2t8qkJY%y!_+> zo&3h>Kcexv;0qLkSc@^b5Q8Z62^{^lvUdE$vSn);tt0S$=Tk_x-d*aFu!0Ro-Y9Op zM;sS`p0Y&W%WI9jRbE%@t+Ie$Zn?Z(pg^bE9+ zJX1I?X2i=u$_Bkf#13LZ;3nn>0eJ#+fP`L91YozIt)D|_xuBB&(Hm_1fDOI8MxOB( zGCOz#C^sFg!x=PeGCKZ1Co<gp2|!4jrbaSO6X!>?9ULbX+xTXvAmyQl}9%v~VI= z3!M8u(_J*DN5n14CUSX+?wpH_?oUJJiCINd(OXJh+ks_BR}#7t1V)I&!e15kkn~O@ot<>Ic)hij70o`d z$5cbTGh8|yZ?ffvN{0daPq(P5rQP=gIt%$7Pi?-Yg`I4&9r$qRpXgL5=4R-lEwC5Z z&PKGL;Guw-I3Xv6FR~bjNJXixr6V{?EQ}zK$$_4FBGB5oLYR=u#~x_PWUkePBgr`}zS=;U4%-t?Dj4?Q=CpUG}+675F7%!W>pkV-far zsGNdN2rIgXFUF}%kaB517sm6;&K|lz0Wlx9i0PzofhBucDgzcs`!|g>Tuce$Fc-)k zK!Nqpt_MFS-1Q(hI@u3M8X?0O+3IDm2HU%sVg<_U2YyKyZ9D6$#d$%&>K6MTM2V(V za47Nq3y5op{f}XPEUYJ0mqZ+5Rbxjf%)C+$0ZvpyN{nDm*z3`@P@M;xMetFn;L>IZ z8wblNZ?4Fbzl#nlzhLK+A}Re?Cc^K7lh&nXoMQed0&rwnBu$v~U^qVr|Ce~Aq&Fl{ zc0(%yk6aOtwY4-g7(9i}m(#l)psZmmBE>jlN=z9d8Rnlx%+s>8>a4xUr|?sHlYYdg ziWn^jq5W)?{KY6=#%omY)$MzrwCg%u(OG$<7^6WG0VjHA1-*3wa0)m1-DC^^oXB*6 zcMc$4h(@p+R+VrgF-XFSr3H|T1Q-khK^aaGJmqVG5z!q<>q&nRbO&)SkbB{)kHpAo z1eq88W)k$;6=L{^0e~qsM8N=XGo90gXe+{vmUIJpZ$KMpV;hdp3Y!M)_ZXCNyrKj& z0S4;`oiNA_(IJf}y-Idn{9nm!^>p9}5`n8g}>V zUrayz^{+gV{$l?8bb55puFaX}3@zx6u|0dn?kJrb+O=ZEu3wh*9|1d+{9F_%XFJ>6 zAZ!`*IyQe&kWexolH3mqGT90gLz3Vz%{5t^R3F>l)mM6}Dc=;rzVSX*dQr#$(5P?| z5hVt(sSYrJlWqR{?Xxg96*D6-wK{Y7L#b~VfIer zzOlAP7Mk|$iayeI{Y>M+!^!Xd6GQO!KQ+xrrT&F?_WiQxm?Z??tp^etdbtAaLlWc)xcYL#)OVvH1n*7eUFBOS(lA7c~Y z2IQT6?~!HXyAD|W6W!IHsK42@>i;O!z%+c8z28&0^cmqjR^UAl_=pNvLsh%<8D&)c z7}Zx><*HKN`22)XY&|}#it4`i7q*Ufty6iA@|D*VYWQAlm+O|(%KGK9_j;b{S3Xl& zm!5w=ZB#zQ&Z#x4Blyo$o9;7x(e%Ge z@0jD}A@g4Ilja{g{GwTJL#a3tQvK_O{*O0kr>aOb1>I2meR$p|~I<9pbbUfuaS7WJ}sJXx9$(nD~{GGGS zdDMBz`JD5I&XOzR+UnZp`k3n}*Ppp9?wotK`>6XQP) z-Rt!o^{eV9>OWfl#rhxAml{?z9BBAz!}lBBY`D7XE3jegVp>?=*qV+`US6knS)J0B4UWxp)&DplOZMN;nw(qoEY)`e{)Ba@p8&Okq zWAyRpUq(x@q1aUHSnS!@f9t60*w``K@k%EJ-V)#Zsd5032=w9NmwcF+>f1$LfnDs6 z7U}S?@}QAt@I3t&BTrEn|J%r`N*h~g=j5;%tTT#VU)}> zSRnqBk>{{x{8uBdDx=D;jJ!#yWj7mnv(m)wHS!iEz`m%A;1%36$|PR0O|RJ2lquyy z_}z|3p3V4bcq79>yq^0oUc;>^cZ-*CA3$!ScxCqyksijo!DdjFK>a?X9e~Xd{LLyW zVXIo9>@(_8D(m**rQiEd`yie>f_D}vBZp@ukId-W)Q7a~y_zD2wHmLmtW zjfV~%*?8#i{uwRN+oyFLIC5lm<%$*iP`Zywd+*%WdvN9m+NgNf_%+jq4q`=?y>I*$ zl-)9|yywVQV)R$ObX>zcG`v@-2X?m}%(4&p6dGDKu$9`bgGX*Ta{G+ludUSjd$K)= zzJAoYvN>h3qVnEvK;J!c_|97n9n|`J@uw+(-YnpC5Mx+2u|u;n2Ybr1lh~+SdI00R z+UKVz#3^9LnaWIfqmu>pDjVJySH-H8^~wf7XA>~z8s=a%piM63Mzm5b^D-avvjFTs zb*!E>uttV}2*j(kFb(lct$6=T8*67#7GoWF{c9KNhW)Gu@x&`wAKvbapb3^@X_kSM zpJM}TB~B-)0?GVe8ojwvlaOqwE^C880lpmR-lTvTbZT+rh@z^=v2G z#dfm~usj=QH?TeIMs^e1%Wh^9Y!dWyn(1tY?PL4d0d@=2t}A7qEw zo$Ls^iydWmvt#T->>l=EcAVYI?qeTe_p{$&A4R=}~ryJ;px8{wBWs(+ak*ctXb`wIIiJIh{RUt?cq-(WAYKW6jnKeCtD%j}!%PuMH$ zPuaKFx7l~tcUh7BC-!ITd+ht{RrVVDbM`v>3-E^j%+9g@!hXnp#Qu`~m2xFed4C_r zX@~v(8>f@ z^K^!%vpk*S=>eXemG|%WfGs83cc(#vc`*}9Ovq_#!@obuBGd!E+*&NRf@a!bd zPVwwC&+0ro!?XK%u8-&Xc`m_oNuEpbT$<-HJeTFU9M28#+$7IU@!T}e={z^XbNl!} zA0O!F0|`Emkm zHOZ%@_|!C?()rX3pW4T#`}lM}pHA@UB%e<4=`^3t@aZg{&hhC1K0V2&r}*?VpVs;G z44>Y|^**lmb3MWJB-c}1PjfxP^(@zOTp!>FWY?#-KFwiu)Mto(FudR2RY_h7N?a=_ zyYd^xHEqk+73YpE1TKJCP=e1W%5egj8?mFeloRAV??P{s?&NM!x< zXm4a005N+Y6@X4bOM5s*w%T8^-qJ!;x^~iM&?WzC9lcfYveKkp=s=Nir4{<3RTUKQmsl*>#sPK=L_ zHx^j;_;{qCY|qb(kM|VRxVAwnnA#^XAoIxfe8C(UE?6SN82)&HP4pB@@d(DH>1WJS z!y4U@ofoP`3d+QWg4z{E>4Y?vVhesuxa#NFn9G7tZ|J7SUocRb(1oMDj4G0iE*kj zv0e<&7JuGat&D6K?g}pg+8$pH_$t{7>&6g9Fxv@j!->cwErNiO(nydjXpIFdYa3NKRZDLrPK=)_eZU*Udc=*J`nOaMC z;c$0jE5PK#+`QdA1%Lbuqci|GQyPq)Q7Ns9pD|HdA3tNJv>|@RLTO|CjFr-+_!%3e zq4*g)rOk1rP}BV{7)T2S(u@W)4204!2102o2102B1EI7H1EI7X1EDmEflwO5Kq&3N zKq&2uYpVpFcf~P(_k=crMVO#Pn?zdZB&6z&7rMF&UDz&hVCp8I)K&LOWHJ{aI`y74 zfG<6Tp2am_fkM2i!2Epz%Dt6PS$=CpTuX~__Mr~jaOHLd6}alKs9XtrRnXe?Ly_E> z70i#B^kd!_=v5z?0M<_CdJ2hnZ*WylA^F>?0>h?JJ%y!E0_|F_wuyEoKzPlG6PqHN zKne1o*PwUUu1SVSN%Wrv2?+rE@h_?r>?7SXCwe2Aw(11h$}HX1dSx306WT;AtuR5G zdF_t;SGcBXjbFhF!5hYhiNM)FDA6B!jBLc#!YVG`C)m`iTT*d8GNDHb>d2%H8pB5> z8~6r`3`8wzXbaTZbVmBMRJYd ziuDeU8)Fc$e~xpta2BEhJE9 zQ@oHuGD=X}0Jv%!!L!P6x+YHOSQrIZH^-k>ly%5#L55N0+W7NKlw605DA`JNhH+~f z)uGIGszaF_REIKSRA&g8>!}W9c2XV6?4ml9*-drUBJ%;NLzz6)q0Bhdq09|bX9Sr& zREIJ*QXR_NM0F^$m+GuR=4PrxnF*>xnMtZcnW=aoy9nlKx+n~ySQoif$ju0RLh))` z?28w2i?#RDg{XZ%vdqYRqR@Tr+G9AMsVLf0GmB@H{k&9( z$MeMEdX%D4)$7*{jm=ME&&yC9P z5Iif6Z;~z1Ves>XqTo5s;51bGZ?#U*(Z8WluQScPTCKR04^gV`*3_0;xaw6`H2dQAVS%Dq4X|gY2a8zpT7?rYl=nrE^r*8M62n6<51-) zbynb5S0dELz_CRMSC3!?)zGWZ6^+q6Rmd)Y*8ZBUCJ<}6r;#h%J5x)=g(6r@tvg%QbyuGN*SfhP>NBf2*-2qU8YRMQ6|b} z;F$KM%Hy~<3adCsiN(GjYLsD{siZ5nVVe@DOMA2KAY~Rx2cd;R)a$P(!%7Qt%L)sk z@+zaU28|pPHEKq2X;IXiqOz$`nZ+~8GK)(eFN}&G6dToVYFXLL^xJNmg3>8eI%w9E zK{E==(8dTQUv@MLhxx@buqz6b&|WD*SrPXC?#a{f^yB2XXq?mKjKrag%Hx!QN(%nt zF~&G05e;>Du=J>LGs=p}rWY2(MWsi@4NMsr9~*~Smp7+esHiC8(M2gHqewnEbuuXM zABBsBrL&5PXGFyf!iMu=%xEE=ZeZ7e70)c3F)%nfq6_oCcYtzkr`1MTZzU9?0QF*CfW*)7K1+6`zJgVd<6P3we@&Yj6RAm~7d6y!czsZgF& zo>Jy1)yhJMn59aMvO;-UaVvGov&t%^L0PM;S2ie{lr73OrAgVTJg4k}8rZA6r0iE( zl>^Ev%3XlkfxQ4KXr?WRVk*Q!0#o@%6eoqB`XTXm>W>P>32 z+E?wT#;CWdgVb0xUQJY!)l@ZIyIlaY3g)!hB{L%Rm;@bYK8iw`jk3PtyUMRi`AuSjk-d8T6L>+>a*%9 zwLx90u2(mxo764pHnmCJslK58mwHYWaq$U>Ny#axX>qY}adGi+32}*WNpZ<>DRHTB zX>qx6d2#u11#yLOQ{rReWO4N=iyn=sX$fhGX-R3xX(?%`X=!P> zX?bb+X$5J8X;X4zbK`R3a}#nCbCYtDb5n9tbJKEjbMtcZa|?2(lt(<>luU@)VRFGVdQjl7ZR*+keSCC&&P*5m^=>NN#xgfg(Dn?P4flQWzP#8$% z84yb?u*F@_s&^~*fCcYWSAuxzK|ZTNKx;rk>p(<}Aft^Sq|G3utstiDAg3K5sAly! z^?7v{2y3^xN8PKwsJ^7`Q}?SaYODIPdO$s>zM>vd538@Luc>Y7Z`9XSkNSpsL_Mm$ zsUB0`Qr}kJQQuYHQ{PuVP>-u8)DP8@>TlKGsi)MB)ZeQgtA9}csD7e;s{Tp+O#NIv zt$v}NQU9#|Mg3C!O8r{>M*XY$t@@q%H}&soJ4pKxB9cDXsV`ZAzG-WYZlE4Bz2V*riE+Ww5zoU?HcV`t-IDkvuQmwyB4YS z(yr64*KW{m)Ou^b(j1yoi_-dNH)%I((b_FqU(KcU)B0;M+5qiVZJ;(tsnc%LVzoFe zUQ5stwInTBOVLubG%Z~ltlh3dEbSp}v^GW?tBupfYY%IWXxZAM+GARdHbI-HoFTb;Go)k{B$pqOQiQUI{pWUN>k4Jhe?yuQ9y1MILy6)TSM_%7{{hw|abi?Qy z=H2k}jrZO-{>I09NA}L>eYm&(S2zD^!LR_Y|9CP@b8P0uCiBZ3fs*P%i`a_?% zK1=)TxoO?a%cJK;ABz6*maA^L_m+jXeAxH;zLWcY?YhzRtZS#M#r37@d_Q}?n11*4 z%kHlsJ}nvp_nZLZXJ*{fZuxmt!r=nao__3rwyzhCR}d2C)`j zc8l85!WXxMv_$fce9w!IEG_;8c3(DM?9aAFFfY%cKeZ#v8`AR(_jF|0qr&{rBFFCX zN4tE{E-TOBG5Rl6Y)3_rBVsuInb#N1nAac8^ax+OSM}BKoDhB%EsAj>4%;~H;Gx(Y zv=^bm;moGyMGm^iaWU4Wb5!K0=#UNI!9slFJKcYI{Yx6Wct7)+9}FzCPuTe^Jm*d3 z?!p|ryKlZG4Equu8(^0 z?rlSuA(};~{m#1{?aPFPl|EBeJImnj@lxGq@a}dI;Sc9Cm|p)v{cg6Gotymk%u|Mc zy7<^GhKcU_5uyJpiT5ls4)XE#cSW|&uV2IUKfKRXBjVha*(#PUgy(d$+Wj>m$I4d< z4`Z7;5EM zsp7?2%zL4^P*jl{qh=Ytxrf@jykoN_o{btrMf%nwxW}tKq7JM~CNHu}0 zz8bok{tiZ;8fKh2rH^}~=nw2PJH6-B8*doC z#ivk3e`DO9VJwxU7Tq~+oN;QHe(Kc0vy5x_oAi%iprZ^CWq#m9}4 zr}WB=3wE$(*1US##*GFq`kg)VZhd3r>M~Z$iWihrRvIUV=`X&x&BKncBW15W{-O~v zXv=J0v@cp^zG!o{`-Zvv<#r}c;c;DzpVEI_J#EocHkB3CPj4_V6k>n*Z4TTO<_bN| z-k$y1RKuU*Ptm8oHv4UMobhyi1GaQ#@EXzGzW32Bqu2;0(!~wf(s4Ly%cFa#Ihsc) zr$WHZ=d(Imz2~zqhrZ}YS`lB3l~xanOr$4e8b~TIogqC_eSNS%^H$7Tys+93^TZy} zlQ9>T$*<{^ja3^RzUM3(8yhz|eVW%RdRk}h7E^iM@@J}7EvTEf!f=b8b{;K;h*qXA zK`;HnxF@n-ScDhS&f5cn#1mi%ZQrf}9WAM;S>p76YF*;4S?TDw!?M!tUg_jxthVp* z{1)4{EASMn^oQx;R2^bgI}c34*6?`!(P0# ztl9Alt9|+zX0(YumW5A>5HW2+Mpa2=5u3mY))($5*-^6Zsr}6Gt+MQ6FE;LIGTfFO zJJ#=G``Ig%d#iR#_(X*8X$vunL@#K{Y zbjIEj*Brgc@Q=3~{oy@+4P(a2)r=<-&(m0>^blHHoY0)?=7$HS-J4fb`WSoI=xDXD z*Gpf`+mrU;!{4!g8C;9|T4)Z}`7Ha`S0)}g^2#em9424KfD2-{cH+db4wvt+HK>`K%$s#4xy7*gcJA45kR1*_qsVdDy%xHSZgILS)QiRT z!|4;lQ&WczPj!kIi}~mtk_H}AQh*{oBvb<85VYbA@#1<#jb5;5`t(HwMok6tAJ$V( z3_tDg9rpSUTZ+pu{a6C0@38N%g%-k*Ej$*N*9As{00u8gKEyEC`BrmW=%Axjk04o( z;(+e*e;J^{Z6+1^z7%cIV$xag2T_m5dx44|AzSU{u*4XvBw?|{TD-Nq+0l_@kq^U{ zfd1S|9AXS6Vd5)e9W)=9P(ez>e z|D(Mp*1c_@1u+C`u;{}%N7--K{)Rmpwrtq4dG%h<_15ZjbJxvnC}#zR*TRlfy*}k7 zW6DbpH$KFS2p4fKhEEa~M=7nV-AAt!w8;O=${bg&8;w<)CKsg8Y+5B_kmY2H)wOZ8J_ zN5*a&W;Cr?zm{+Eh3oFxr)!th8j}v{{tCatKJ=kcL!GSOxWvH|_Lm=?|0-mpi-%)# z{eINjL!A*z|M4Rb)ECV#^?*H7CgD+Nh1?as~4BgDxtwR>sTAp zS=lq?wX=vkQC8CR^Y>Au}aih*=HkItHXx+ZAW&0uHgQ+9ESW*Zn?U<=ujnkCB& z(Q8EUR{fLH8GNt^XZXty8K0&bGs;D;hSJ^DO$|*A4cHk&c&6@Nx4M2kGngA=*XH0v3OCrvg+U32OFpu^X_o z$mz%eO991t?Ed*(JM+!A`r9F#E^Qv?0PtPPsddTw0z4>t!kO3R^$nzvuw~1ZFEs{= zk-F`RTLR?T$0CKB|ADUT9h}uP3+}32US|yCxXZh|ZdonvvVGxy01p~u4Ppx? zNfC$5%g;t~?Q19oQ$67OYpyv_gq_0`8WV;k4E06(fi`^6rm&OR1gwMtf1t>eeP$JW zx7+D*2lTTXpoe*T@ONmSwpV*QhjIY&Xk?0hV75F^BU)`L+M$| zI<{d=?ONkAXcF5iwQHBInTuik(VxW%PoZG(`Z;T##BAh%|4oHB2MUq@e$JmDOA*W7xUFP+GDlEWOyOfdHL#%VFtLHk0aL>oqb=3`X9YY`oNX3ayTy}Zsyu&)T zp?aO8!(mz1(6G+g;RsYDE&_zY3Y*xHyS?}$bVpVV0nCA6*)9Nv(#HAvb2FM}?0kYi zbLrMu+sd{Ze1sKC1gPdAYY6LNT9%lVt686%g%6+rwJYzzsyFxXZMQJg`i zjEA>1&&LJb%i4H&^BP<^bt;>OuW7~==EZ&Un{i>-Dco1QM#mLBTe$5(CenhV#3OHp=L5aC?6+aMr34S)3pyq!n`I|KN;uEi=E{~*l}_Y? zw|TRz!IRU&Pk`XO0qVnvl)u@oHmkhi3YDriJKK5zY+wQ+@I4jPA1vm%*N78@?CxR8cq+BKU#(3LsX4^f) zG>K-4;n-%1nH+mQ6WefXGo2h4P&5-7aA25i;}BP9To@>_pPkKrwrbTP!0L9vNd-&N`?Qt~w@PCkx#I#DJdxMt8^pU`x z@YlfjlAJ--gRCp(UU~q*8q%p@e$z#AngELs$>U5wF2LIX*)TqXM87GSr6LUJITK?> z#lV=IUQ5v053aofMZtk*i9&mN>8LwdoFRY@xE6o}?CVi~NN+N-62Nvu9}qQib}^|N z@SNvcJF=iqZ6ALbVPt^NDw_;Snu&(u8e+Y7 z^yqt?*;aP%fzijS48D4#zHZs(QudUQE%g=H$ugfUbT4xo-=Q&9w551k)wZhUCC@YC zV-U#4mJi>2^FwEwm3=t*%@K`;Sp9)Mw{}hwTMtb^TFk-SmNjfuO>K=a(Cf9bJ+qt3 z8p|4sS3bdvAztV-npz-vpoRppD-y79fgN`x4K{!awaQ!&U3>*v8(r$ziCR6G;Vc zQo%dPn7DG9HG&5wB^4Fv)zzY2tYKn?A=3Db;zpi^?M7^A4#sDQdcLN*!4UWRM@k$> zgc}q&Cg_u9CCO3~V~{6=5Zw7zDMO`iEkLtGWRR`kSsE@T09G(fgTz`=5fQP~gr@sDLbk-_3w#{RMI7`&7 zBvd7|MP|ZB-I-|OTbZxBulu_r z_4?{f3)cos-nEN1ET}gIefPm}{n#<~_lJ&+ezQLtJ=z#Ca^Sa++fUZdhscIQVTDm+ z;kqcc^IoEtIEk$%zYg+_9Ihl3f@03J9l)66a42P%NZZQumxE8sAwUIsEIAcI&+ zfBq={%|F3k63}^>gP6x|+j60z0q;f2+ijQ{lB&#UF0l!WypaTU(7F|^WkX<0qS*w| z55g)-$DCw~95w>o-T;gy*^;m?O))r5;v~o)*>(>bI5`x$$F>EYTNuMOj~C$tJdS^S zS2q*%EFJ?$K}tBnnA993lR)4~whvZqT{AcT+}2I_L#(=L*&DN7Jw3Ejhh%9)?)jhj!j`R za~D4U#NMg>9#}r1Cgm^lPBP&3-OU#ng{Z_R|cOV%&mcy#+d>77?Q#$W&f(GnMyP8Tf4RaEVX>j3uFRiR3V)hy+ysmzPK&k!bBIG|ja0!VOiJ~lMb%F6g-Mpa_JH^E3v0uo`fA7d4F7z) zIAE==U)12}h_N)(*Ecx%fuO4s-oAjV({~u_Ai=LW4ggDnzdcFQ0?JDa5AU<2yllAi zy#&$WC6VkCb9p%!(KPL_TrLy5!{JPdDOgTsCB^{0$szZqG*{H)ak2>6Z{1Rj8BJ6C~CDa}~hN7;aFXc0O;4N=;fPz08;5m@5i ziEsIL{96hgwXq}6Rk7a)q(j8U3M5BdJeKT4jE#*L2EIDjP!x?JRgK4|Z<1k9#V#-0 zBv()h9j#Doh@Zg5la6s3ErWlYB&3Tx6R>8`8rgcCm-W0muySs5YU6b z9-iPi{v*!@f*}Yi(U7#>f|gsrfWyuV zzW@6=R}8lY;_R1%+et$ZotX9t_94E*B+o8*H>wbDc*=l$J4%#9I6%^q*X`EV*EF(5 zEZK#;0n?8IquhQwp>9+Unt}WVtog;bfH(`SDq^|@2M}oj>qyR!;j(2===ysgP0%#a zk~iqmHKV6ANhFDgP{GsC#rBLa^E=|43vSC0{yD8WwT`)xuO7pX>EbCj z0bpnE+B;2-_iJaZQT{Zz4%tz|n_7`81?p9m|ifZNpOY2LQ2 z*~zw7Y@JnW{CGt#y={xwkFZ7OXrxJwG&xR}3=&W%kvyl6Ri?eoA0r+M;g4bYU~$tj zS$Rv1eN0XMoL^5fCQs7mEvlZwo-!j9>)ED;`nATvgZiF5C!cN2+h6eX$ozZ*f-vTi zdYh>pglUZa$tR3=&-kRcdD_Ou>nm&Lu*wyN{~GbObcgC08BBElB;)9q&#Hdgv~%^2 z^;@?Z2M+3M>l-$+^=1&_DOORvXr3`?l3rAlxj3)2VE>8_T3XD;>+4rGvIeu>a<**6 zat0{3h%KmI1{iTr900zh6}Lw4Re$^L9~s^rwrbyLM1joVbsZW#^5w&tH0klBCC`*R z^Hc+4W~c+`lp^&{HdL%%w0_a1xotH@Tg`7bz5DJJ#%om8&ZYrlZE{4FJ^Pt^D@Tno z=j#e1Ut7QW(otVNvdKM9EDi#{r%E;4da z3rYY@xgnv*r*jx80S&pKRZSO-vdI!|FO{y|V5S#xy^!(6$2s3($JW2L!@aC-3A`T&8#Gq! zp1X}5Wrq&oYunu2RgH$rt1qivT({J{^R*3cGQ@R*Nnrl=P~k*sLI`(ayRb)ogHzlj z6l^y+DZoLlD+~p$JE<&#PDPUa(h4N&B!?rd1Ww0vrzXydpIEiL>fqi5z<`>#~JpNFmqun z5f=~?X&jw3Bp+;5TpT$&nBm?2@BdxH!gW|N#p(ao!8fo zLXo&N#*3-4{ls^HJ0~xgI*Co9a6FtfK`R}Or5skPOV|VDwS4h%Lr~t&MID{3+s-l3 zkE_Q|yDvF7_&PAPz;&-ug=a3-DyJwz6a8zG7U(d`Gp)B*{y&pcqwc{rZ zzKb{OEiE6c*k7=}VEF@6fCSuv=?fNAvIVObtY#ZmuQr}_fBjwN$pJC?V~?@hUw!P= z$3A7RzG}dER1-u71^XY_{0N{ojC{yJf*}%jdv!mO%iyCjZ4onAO45_~%NLD|BFZd6 zU5YW|wnx~c$7eqL%DA0FSqhs`Q?jIFQ}xD0TbXhCgc;!;{xzHqCxHqf9c29bL>!_& z7q9t>#Yy|*M@CH_vD~nIw6k!-1eR@#AhBg-uTMWXX{&MG;j&LEpFRnRR3hDKTMI@_ zM?Mu@n>hZ#>6t8(J-BP42bz~2v&Q63$Oj-}Esnx|!tpiGF1gmt9NaiWFg2$rggM-2 zX>uYHis6ET#>%*o{Fgp;;~pGZkj~QC(Ea1yq2!%5ZySU?S(s2f#N==t|Lua!95k+c zd0mYwe|IDbAsq^)8js1g+kSu)BqtKZ1!GuZ!Tt9cybbUN6x*b1RVf>=nr8e=LRKt&Am7KttP~DM?F&vG2p-}FU}x!0mZE{a z0y+pCnED4ZCH0T#x0AVyBoiq#K2xfzTf#(zh_)9_*VFGC4;NmD5mcTWN)+2T2)>Yq zy=m_og}WZecxk$RY{LG#*D;U19%UCIrnHz#6Cc$r_{%5T7Ti|E-ZdhQeU zec!zF*O&fktS#nM@IZ2G~apy$t%;kLyig^3mVL6kMkbky1 z8j_tAZ=ADwmU{_Xz~&pa=R_51Raw{?xO`VG*j~9AxlV5$IPm712PThpu;R)&3ue`r zb$J!)p&DCRW7vjoU$D8dnVD559~kW{W^*cMEm%^6Rzb2=qRL85x>p*uy4Bk^%2rX$ zF?#ak(awlx;gf-98;X#k!3?vI%pA&zvzHbc-uZg%j{5DJ@Y%KTI2`;hR&B1_ zTv=bnN?GdEvg}FOlSbah#8pPAx5>&*@7mUOu+!_^JXZmQeN-eaDEtz+Nc@ai#Kxhxw(7?33w)iF4OAd_@m(VASU zPsLh+d7rat}dTRi8YyGAhNs4ca*Owf`7*4 zwYY0|iWmdLm

=q+oq7+tRRgr-9Vc(Lh=j6D4m!A>yC8%GnaP7{>EZ zX-pf@FJa{XJP#(u2LqqMU@wxK*gp@RI%Nz)Cil1@MXAUql8E#os&k%ZryhS}tU+!w z>9z16Hz-^mcBo!f4A~8e2ds3 z&cO2VMT!&rgg+8S7IJraDbK`0mQqOhIZ?*T#B+fQ(sxP4LH{J`Bc%*8f;>BtVQ{e! z?6*NAV;&_i^dFY)R`P{8C~r8&YP#5-_90GjzqEF28zgpiOJ6Iw)*QB5DSygpgG{yB zZk5V|mftjmV1|4Q4$mtp%5$Riygfy&4&Qi7>z+NWPTpM_oIu;KH$9OqtH`B%_d#Xi zu`OSI`oVV)B~VecE;QLvrv%j>=h`zIF8faA!5Dkq8bRA2Xw7wp0| zUi26%dOmDSx1!w>qVJ!gTE-uk^z!tVr?-?JVux7E)|Yp^yz9Wh7SEr4Jb@@APd9d1 zMbFnok0Zk7F)CK+=d(hWu^G=!+dgf3VawD*_npb+S1sZ_41SnL1mdRViczLztKEF3 z!Ib}`@_+&{5ft7b#Q~Tk6R%(tfJ=IS(rhouxu=P?orJU2_7X)O=+z1^A9<{4N?-DN zaSYpC5~(>AvQrsrm5OW#xf5s_i8M`jg6vbe806et>4vWU2lEDM1T$!UNMA}z^0FmF zMw(ngB#XBe?a6bT*Doel#v@(hm(K|ANF0XD7}#52DdbEM6XwW6EFlhYf!2`_IsGAr zvGa+ozam?R3$rCC!tFwC2Qrgvan%FD=*%{&x^Eb=P-5)1Ta*D|9a)jKK0^kC+42=> z!JCzHQQ5XNa5v3R4B*o!1RQRh)*&ul)~p~hEY13>QZ8uFw9K*bA{r46zR1YGilP8F_Xw6bMUB{ z4;CDs1S?3Q6;{|NA_2}?dW}b5wRPSHF;xI_I5h~`2B1DD1<8UKP{`$JzJZMTV4ClF zdxo74!5bpjhT)YM_%rYZ7~V(lV3~t%8|1dh1#d&%i4>h}cnJaTJMb8p^betuO{5zL z1o;jlv?E_qKrldh*U40Gw^d^tw}c^n3fsim%$gQ%s(^QIQ^nuJxOFA#N_NcKQNN>p z?Q@HEEZR}PuV+n0)7B=EYY4fL7H*E_2bpux#>%y`<$94cG#jQ+(IETWl3T^N3N(49 zqM~$RF*9J(pS5mb8`suvG}u{wuvtQ5yz5Y0-qhqoEVgMszaCxgnD<;sy;0%TE0$Nz zTTp@f#3sDn1S{EB)9wx~0vMMN3Z%mwvqYr8Lfm}?tb4Hfz}$UC>=eDBxNZiUei_US zx`G_fv*(vKR~vi2)645iYfEd5l`=~}7kXD>N5rI9LaEHfJoi!C%B8pj=uHj9}Wg(wmndeUV#b|UDAV)Y&Z zfRy$@;tUobDOdRinxhwthKBi)BZr3hXG3D%73QCBCPktaP@{Cg$kd|1Jw2_ql-0Ot z$udfp9|N957A(C3;!BBKy7ZDV+im`GmsvHI=OFiW*NVsS4-%vC_eJy zTTzdDBV(;_45D;|S^ACD*6fX>x}8hWbuh2E(~wM`(hKNhXc!NRyo zCB2kHNuPxO&1q73Gmx4u91RKw6Fm!rdXM2r)4zR-YcKF{#=9{dI{n*GhUar#sJ|7x z_M@5s_;x!RR{lV~@kX+K`1#j2yv^Xnee%!~hUbj_!2Ub8Wym^|tUtgMYbt+(`gv9M z6U;IGHQog*HpD^Eq8Ajf5&H`^&w*HC*y=ZLHh3#Ps5e(Xk0d7!`xe>Mv`28RX1x&u zoK5JoyBiRUV%38yvizpm2 z(`yYEB?A6Pd)Dw<1@@8ZPlS>dUZ6=L}CXP~r@~)LaVY#s)J) zo#8U3?Yby7y=LlzEGJec1TR@UoFsD4XG~Jq87{8}EK#Y!!h`-!ywnizg$~0Jm5P{Q zr-HsuJ)Au5ofDNWv)RHg7}T8y=LF!F;r7dI=pdSgO2fvhukr{I zF&schP6Qb_z)6U2Ai|0#Fgpvr1W9T~+DG!)KqOE>;pBorgdm(U5`tM-PLz^82;3`? zE_fROig4+E^3U$76@0Tz-CYxG})-B(dRFjKX-BUq$#7z9)MuHBw*zX$1g|K;fJT9{{6r9$S+^-e2tDf zpZ{-d2kQp+o$Ck7{@t@t{m%Dvu1oj-Cv9}T=l|mPN__^)g8TotAN*om=eoZ%*3NbQ zljHxbonLxRD!=R+o>7(s_E)R}`s#dN=i|=LtG(8ByuVbh^F4H|{?PS4D*I3Gy|k_W f%X4~$E_2;^J#ifP;CI~=<%5iE_!YyhznS - - - - -Created by FontForge 20120731 at Tue Jul 1 20:39:22 2014 - By P.J. Onori -Created by P.J. Onori with FontForge 2.0 (http://fontforge.sf.net) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.ttf b/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.ttf deleted file mode 100755 index fab604866cd5e55ef4525ea22e420c411f510b01..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28028 zcmdtKd3;;feJ6U)xmZaM3$bwn2@oW}1>CTclqiYRLQA$5Y6)oBGM7s&UL;16CB=~) zH%=W_6UVZgVeQ0|xQgR(6Hfyvk(0O_J9V>Qn%H&oG)e0>@i>{hWS-onNvmm7eN1S+ zzjH1~P?8h3Z~l59fphM;=bq(ve&@HJt1v}T9Lj@=s?4rmzvGs>e#M?e$-DSAY}wuu zPnxz(DGIB>^~Cf&le3EBXMcd}6Zj5KA3GXEIX;t*;HP5m?7n+mKJ@c`Tz^VYD(~Jm zd1MylPFz2T)UxmH5A7ZOe}4HVio)j=WvpiZ%%sNt@lV$&%8rY;pWcrG(tJLATS5ef5?>;m=`T3U~TdOF!ucQC(+%tJ%mOWkhLq)lj+7BL_yl3W< z|K$8OuAf04Cua{GIr?|bL{U+0Z%`D&^z7l8*&pAf{=TBzgX+qM@uk@--(Pw5FDd=Y zzv;PiF*WcaJFOVej)kLlWmcx_K_#l7Hdl-))s-Jiaq+Wt?>bHS=G)5KZ>d2Pj^cL) zspv_s6cktVJbfGVdn<57wHg$I5=3giAFkhi>*`hfDp#)t<$c^@rlkfMM*)4yKjpoZ zm;e7O&j~k_zvW&)&a7B2n1DOHt25zBxS|PHxb6pE|LkYEcj28n_7e#qH3-ZzD|Xba zuyCr&LatB>-zH{GA;V(qa?!?47iYCXp*YJ<^ZA9f8oR8`&1u?oZB#99!|V;=FIv_H zHB=}yp=sKjTsBRN!=aeIVp3RFXLZmQUKG&EInIE&niKmm!2v$!20ko9;D~#VS11nc$`+=KtG~yf>$N>ebwp;yRE`v zGH}Jv)#<|c{rH;oR1LoSw#IV{&!ba4$LBE(`n=!v1WX7n_@h>+xl&r**uQ0L1!}B7 zt%+QDbF_1>eooBQh?%++pHi_R?rNvaVp0_&7C-Jcx2Da0VHnH(`yji@Q4AK*~y%C}@R$UciWpw&Fz=BN&REs|Hb5 z;$@}9KzIq9aGHV#O5h8E}wr4JV`QcE{(tKyortc-Ac zv8~hc$>PQ3trZG48duddZHX0S*S59PQlWs6zK{7a+O3K5cJSm-tA>$kafivtXzwF&by768I+`}rql(K|3%uZ`sLDML~eis`agzI^b!&%^)q#exy z{uPQ>X;RvWcC-W=e9lS}(GIuYlzx?4YHksgUImQXzoMzdf+Q*$Kg_9fyOSJZs$*<<+E(%oGdnwYpO{(HB(_-7zv zf{W|>&!PC0imz2WsU5X!4}vIr{4C;UXb`h{hi!c4o#Kn{u+t~=S@!wOPZV$8Jb5y& z2B{D?Kb}81xtV=Fdw=ovEV7czOS)@RtV$L75Hy$i0P=${%0+O6L9*X{n_ULtT`Uma zcpe2nR-kN&c4Mx7aJ`5UC-`?oL-n;aHU{{!w7-%2v5+p0DI98!q+H=t!kzY;Lk8jw z9$!4Yk|kTp^6XKUi`{*~_MqmmFZ`|Dqdj=ZUUQlSi+|q{2y_IPLnLaD+1c-X(xDa4 z*gYOQJE*Z**8?vU0$$A%qWMuB6`;a#{Ho zt(sfqBHoMjtCFy>n+Y~b9K*m+LKs3S=}r*hvY}^>Jv{vG+rtlQg~72wVC>ju4rR7% z$sGF3*uqQggM&0jfww#&+H;~s;H}GHHxf>{6Grf~aLOFbL^J-3H)Hl@=HhJ6PkvH7 z8{f2PZf?^i$TM?l@X8ZUUAdwcfOZf$EZYxWC7`sT-KIvruTtPDUw=L zK&%PU2IwJhOkYnG7;3ptY2dV;w43plfJ`Z{ovO3g_gK62-G8vEK~3AYZ{eI3GQtww z@naTIz&YGdTO;7iFb!-NY#O#Y?0Lu^g&BK5+2eYB9kt&Chy zfn`Q4M6*FP82LQSjArinLqVwK=$geu>6<*q=jB~2_&j$6Ca}PZ|3b3InB*GPsR8WC zdaR*a?n&0fd}iig5CvB;D?tY9&>S72HQ@i#6f+u&|KzB3ZAsgz*zsapcJtE*H?CND z(=BR1jTz0wKd7>$x43E@tfF{qbN1lV&EbE1ts7D9GGDu?OG5h7FYwkgf$VxLUl*#P#m;wC zHy9Wj9BCPLIK2U%W3wr4q*}&xM$b{3ll^&h&^+u5hcn=JN7hh-m1 zUgY!Eg_o@Ci6@G-`&Hk0cZbvNW=`vi*luVYA0ZEs-s1)rt%np7R@|$dpbgX{mqGDrvr8pyH$VUJ#p{eOwmGZp&nc8YPIm z*Gqe^tGyMQPwYJa8z?`>2;_3sX zzCdyw-DiScxfm(eg1j!u3zB9pwPDrk6lbXw+0Ifwq8%#>vD54{>7}xcq{~ehO9(P< zALw#-N2Ix$ldJ~$!4UT~G4MeLq#}SSf<4y5q~rirF2v3jJ*|iQU?^1886#}I!lG_d zy_LnY6<*bzuBw=0M&@l~+a$}X0^=JH6Hh1O9908c; zM24g{$zMn|S**+aX1^KBA#1BaN`;`eysqH2ZYzW2g4@MeR3kJH8QJdA7^F_c%u#cc zmXKPcMWmFrIxV;^*H-~nwrliPJmz0iUom!V^aVD&sCQ=N^)>B~OnXf`8B7acfS?sM zmz3BmqjPhm|D_g7CAdXH6XO%~$OS3Oav@MHWMv=`v3~r7K+uWp8xx>F#1a-+V=~Qv zF`Fvw#f$dJO~t?4#4h8)Ub%1#ziJRv9mOb#dp8scdT}K`RcWVwm*fsJ=wJ=-+Y5Wh zGJU7C+glS}pWhtmVI_r!+kTVJ|0Z8Nt2IYPTY8;k8V}vL`9e!*w5``x2K!p@dCP@J zqnH~wX@C(UGlzwx3v(o{l^9}fkQ-uq0ZwKx(D*cab^n>pe(Nic3yZ&MI5y^bY@=#m zChiT)6$*16H3+kob7x;&O`PP)cwb`d*sjCS9UuZw1#tWlj0FyOKb%#EBWezp zhTw;O0^xfl3+sJ9S}43FdcO5a0lN@{qts`ip!YX)1!5)OjlKwvrS4OW{UP*~#rX;) zLrhdQof|3+jUA&&@p;+iP!1Gv*WqPju2dQ^X0J`?3GTQb93RXd05g{0xYX{I58ra< zxsHL3+B2+|0JqcwWX>adoK4B}{xgMZ`yyPBV^*P;I)DpR6~ul(>sW%pJYe>Rqpbslp0X^vu63MFpo-IU6@N$SCoJNeMx8o)D97z!m@tlv(mI$ z_AG!vnmwd~S*c6Nr=`uUyzkPujZ5P;`h{gy@;nS%@0}F40_I7`LvmCU{JmdUsjOGF zD6ZA^jT?rC1_x4ou{Mulf>DEz2bSiv6fL2=39bdS7w9i&4y4JXSQw%|!el_I9Z4Q$ zDG01&A!rFgAP3Afg8NXMc4GO(m%!D$adxC5fK3AAxq__%vqFqG8iev2JRu*qp@Q62 zfsQZ1C?)F0siXs&TJQ_8rz^0}Objx#D+!&*3+C6HBEhQw1xxi?E8e|SfZ(UwmBEXM z-nk+5LH4QfkP#RTmL(%kiReXDqq~HZ*U&u@<+Kk8UVSa)6Kpn4BkiDNptUIDJ=SY@ zkBcBzYMiV{WwxV*=RsldIPBMY8zuXlUxEGF<1E?hVZYXuO{sF?wJ0zat_j%kx*L8!tfj+p%JQRk~3}w^rf?yJY zV*aWYrv`*%%l5>JXW1UopyOI`2*sdC8Wo|OnqPt!t+O9|CrR+?>x$HS#99MhC8K(2 ztxNDSC)1fhPHLFk45>^sQo2`KrV{UaMSyb7V^>v+&%V1B#*MK-)2&Wo$pGuMh#??- z+z~K1Z#9v)+g`idzW#bVq1{gMoUr|qNgVcP>@oPGNQ;2&gN*d=zAY>uP$%G?qB$?& znJS(q+O69ljM647X$7?cVnO&T+z#}dTz3P!v*_0-o^!(wrnZ&|G}6Dq_LPY(g6PNI zDl5^)A=|6O>OzmUsWc9Nn`{cOo`#dH{)|vzg>p(T)qv(28GVPgfc0(R^Y45C`{3jk z>T)^vff3@4BL`@XVqJxtWK=AQ4deCDx>mdFRTV_l$&Uk@0RAA#w-SjGUnp%cc6wng zBttUz3)V#z9g-ypia;Rj1pHGUpea|MCNrcm2%6F;>`Bn~;(lO%I2D0PEi9;hV_O|{aD zG1j=HZ0Bz@2u7Al4yhUFui#VCE=icjV$D@;{Qkf@_DBwYjSE z@S!s+2@6-AIdr(Qs<<)W9Xp22I@sW81Nda{lRBinMQvcmvc4D} zLItj=PwpZ>n%0P559kRR$zm|JUk0@#-)zO#%47#`7_zwdl2=Xt!c9Pe*D}}|AjerQ zSP+{a>434-Yiz}?7I-fQ38W)|0rEo`T{eJzko;$_w15_n{Aa|Ner3bK;auwcn7 zxeVbVCyG*_N#y3{=jP@k*ikeVv6rAH&cn8{Xj_C90qGUeiw7c17z>i|lF2F>$|NGG zFl^?G=caFSZhrNtCbr30Jnv@h&bMy;*x_A!?!5cO^i{?EZD*nOm1baR{Lbv5ag7`~ zoA1lsvs+u;qCND-)US|#M873|N!As}KR)pK63>MEvy5i~s2TlB_7w8{(;Aj&1IcNN zAM~-r$Nn{PC0fHWl|TF5vZ0hKf0u0d-g2pwEq|L_`u^ogj2cV2#AB?2SJ*2o0=ED* zL{5Nvli2|hJ;Dug8es@&;u^Geaw7soNFmp*NZ3jGRS(Qa0oVHAJ**PA7H>2(F}oq$ zOy-CoQ%U@a#>sm~*h2PD$fRlZM11<@b$u;XtI5A**Td^JeEhZzE|+R+?;gEHdq^0b z3Ki820dJ#Sa9chfO08aR_L^Y{2RpcEEkB)iT#W{No=m1waKkbWTZrM=(#$fcZch%=s7o$M7zP?Z2(a; zB$=R);Sl8umil$6&d!xy{U7 zTUQUS8Qxr6ke7R>^aAXYC7e;gu_0d=q+9}5vm3<^{F*cC(ti4K+YnD2cX6hz4P z!uKNNd&!H<2{pmgL?(!72E_9eo zSG~XB4RmEhJ~vdTc1F5Iz6)NG+)&>wj$`oJ3_5Pd}~f^(Nh*@hrj7 z1gjn9B;`XFAPDnS$e(eAGO&FCD06e{GT<^xUOjOsFK*CArCIO>xBjqf3eVHCV)IgC z)Cd(6FN(%!EKBsu49#*U_V2b0(dBldRNYQLU(#_1KMyUGDW*?jv_%{gXX~s6RWmv zu4+v?2YNR>)Xx2Z#@@bq#+n*kRaHjMTE^5$lUwb7HQaAh(-zfgc3OR~RF&doVs1y+ zYOwn~7HDPFBkNgnMPpjER{0JDeIo;&8ne5-(Gd%^RaRHkR(Sm;V`Y`On!E3*XtG(D zN%d5jDt&6Cd~JwZQ#_fJ-TjR0kx*c~A^yrF#gUQwv1DUFM*E(|dMFi}xyUNZGLT0Id4ixx*U!xSYmhON8Q9@Isb_MOI zQfk3JD!$fO=e3)Nzajpi%y{b(9$e{YDJi0EKIaBSdfpp=|29`w<6gMa%?EXb(p|hj z1d45PlmE8(mfL+nS0HtI1^h{XUeyu3f_MXOgizX{x1_`sI)|1btjHi?WVtC_kpmw- zwit{nag?!sX^y-0lUF8{0{=MR_U%(oxug#5u4*_^P~05cHzr zYmrc$uR`El99|uAB#`Sm5{0vh#o}=cSo9X ziN3x>U{y!QDt1I90Tl4u>VbjPC!RT>C)$dwE0VpvN%|ry;iJc6k^JP7G_m9uGYQ5i z42LNMx?n_*M~Dds3jtGw%WxJZM4&fb^Xc-Z&@90ZE#n}xH|H^K?F2PgiU8cPzG*X;t<{~s@Ewc#f%^JAcM5Di|8`8 zt)i0RFNzmsgatb-<1vb}%dhXOu5I)p%B$7pyVM&>MF{e|PB~fa2F@KDSj3l;*s{#GqTM7HF%D=1OirTVkeS`pN&nEGQGf zH<%OJD%}g%OE8$*N;K~M+ek?Ek@QZ=K{797A#g_8M^L@QFL6qlBUVX~c4TH2DRftS z1b-$Ond~tXaYJ&gcXf4ltPN6Z17uhyqG1h+MJQWB&(EN5FpJ-r7h+IAP&slo!ADEf z^Tt`kgNZ7TUv8XYs6w97>53j_Vr6P8kqpd!*b?5bt9S~%0;F7}5P?W(7@-wX9l%d=znfr%CJ4UDvf z0&J@Ey?1+whJ!}P_Nt|w7QO*-LIrHK39dq6`Js5_95n~<#OEk<95W@!_{x=n7RMK2 zd8s`CD?jlZ8z-IvKWGYV0Z@q$6U`BC@J7k43WpDZLn-k5GBQOQAcsyg#4r*Ipio9c zP+$$N7F9%~gOi2PZd0A$HRN;fm=U9+Z&pMvM508voY3C|NIgC}UlXe^X}0PW9j;EB zW;EY2{`hNb&z+~i*UqTH*B;-s)r8xfu8tMeHqBsd#}mbSPv42dG;f?)T7UHI6#fpc zOW2-;t-#I^I0!>aiG{+{EbLCg0>xx-lp4&R%$|PWU@&Owy#L-OvL|mAf~roRAr4^Y z_z~mXO}wZx+En9mn8_apw4m8}L#<#dTp$Ta(Oj@2*=@;o21_yny8b=XdlV?<*`^&veDfVWp&KJeGyLt_=znKkl`P~Kc#4@ z499g_ddY_YQ55{%%4XPZk^pu>Y4Mg>6C}e||^>sa*Z2KnZ52N|HnG0$F z`G&|dLRS0Ictm~a3n*_t;UX(CV)#q#-_~f>Ap_1oY%e$hAj8a(^$`M0)JOvzCB)@7lNe+IIY1- zo=lq;gL3r412BA%8V3g(5H3WXE?B&%CiB@X!h+g;(Ew(SARSWTIs%W~6~~^P9c+)^ z^_Yjx8wT4Ah*(CPG7k;>8HMV^Nv9KvU;N;6)priIw-4S~{oKL04BsKRE&4jp z09c=gfI(1c!91En)k2qA3?+ukYH6&bZ%DawSqSkJ5R`@I5i5=O1kY9(I9#+r45iUP zB*og3@Clru@mxKxR$w12o=IT3g<2?Bpk~bJyY$?eRc&v4^tnq<^7&P3p1b5b@#LlF zKKcgmhVVezd;C~u8|f(wVMmD+h#?X>0T}j1$-^FId&mw4vM2uWBWPghg3?lZ0&fCn z&neo2W=)zNoR=wsdFjG6WPs_B;xzpA#sBsDdd}d?wo2 zxy~oXeDy!@moVoT`iN2=iZp{$KdYD@q7d+772=l>3u#7Jq#sw@4>KUdK*s*)*};K< zD=qs*TPD`sYBt+z%vTy%Ah5Hscqz^j$umjo(RKH4{n;~HnGa{`Ag*0*8Qs@1xo!{K z>rTr*H*RZ0%vka7lBW~Nr0s*K`pnO^GN+^oa?hy3My}H&3Nk`qUpOUBgK5&b3{E6+ z1b$sN1C6!8lia9u5RHvA)p}i3A|8Yh5rQ&ArxZ2i&@$Pmg~)GS)XhrwQ{d@{8!^!554>LAvO5K>rXuKdhv6bW;n7<)3zPK z9EB}PoDri~XFAj55uweCwy3afX9&4U5x#ErIu1m|-LNbCo{*2!V9DHo01S3noRFa4 zmL)qd+1Y()yBa6JRO!b-=tdf_B0aA;%39@dFt(?zrud^7*7o2FuRZ?ZY33~M`@4&2 zoCQ&fM_Bv5JKe87^!RJrnDehLUF^7Ty>8dJ`m~_0!iPw9on>ct#GZDUqb^B=WcclE zLQ5i36wFmZR>(p~#lDuOb@Vej1qc+vdV-@T(1@19Uc_KX*q1^@T3xM+_Gpm*MLTjc z2(jGH%jq^$TTovd-6P$T4r}T*LK2IFu@GcS@Ed6>R7H$mjpV0v3QWbukrt99M3;=z zIfCS4%8*R`;85Eh$RNqC)}hGI=xfEdUIQvYJY~w}rcL+JVc)@h;ik<^eW%ABf9X5yRtP?g%n=#HJ^ukG6EmyxUY=0CxJ|y&w}&`CR3b!1<_R2-3!m}wu(y%k+T+m zZY>n7tj>zrP}_RkjV>F=*m{c3SoFD4e1=87T0&n67J{Z=6Q)_163G85zB0H_ z(Au8}+P-+khxyz%%_9z{L=g$8nz%U7zo^<6@lATSdmFMx z=dG$^7oYz?@vE($YK=UsHGF;dO)NW7{HKxJpJ>gdK2|UKk!QvFLEoBmTqB7Jhkz08 z;EiX7I1r9d8V5om&}x$?k_S_^Uem`#Y=r0kg^X z3srSmOE<*@&%MXpYait~Q35z~@=dZ|1J0yBSuS+P9D>(@7K@?U4HT;ads=450zws` zlRP+siGytb_CG(cX0WrP*tznTr1iQwGKO|lpKDWheV}UV-mO)E z`u?^Qh11sQ;s<08&r4-__E|l6m~NEfcoSQzI+C`&Rjc}J%>y@!_+c9fCBocXAf``O z((HmO!?LTgy-zes*t$ul2_w{1@^hTkF~i86N+8%3NGkltgNSp$Vf?4QZ1NQfwcWwz zoJS=im`4^#ef% z$Fjp-9N{ieN`jAgn#Q)oYbum#!N+`Vd!;zz=!zSB)!2%>C5-TE3Nu5Bt$3ET|L`M) zXNrIO?CUI2`11W@$1sSG{IK|=v(GZmGg|S@*YE$bb_|;Hk{nP0nn*DTz};Yj-$Q{( zz+HFTK<#&Pvt}$20%^zDIukuy*M=p+L9mCer!h%P-&e-=Dcd zd-&&%Ja*|rBpHlgj|u+pQLG^Fgs0ZF-fP0 zO@ev6y&&wQSBe*fbS*A;q+Og71>FE3$v#kx^PGr*cUK6y0jdBVRWixKEt3ur`eK8^ zZLsMlAoyCWsW{XWi*bq`Tz|LI_4ZRB*-*~!M`06>G@)GEH8S_T(q2FxHq1xZ-*MKR z+Dd|UN{^ZLE``^G0$t{$BoUA^*&jm(}czG*v{jdvpQ*XlUZ*!1?F zZ|g~=dbWN0t)|8!3%Btt_g#2mV@s1UYkEa`}7TW_;u$D?h#yiIX# zP2f=Z$+;+Ci{KMi885SW&_!riG61xao5WJRr(K1GuPAc@k!@df< z3%=;Jt5;-`y)a9{Dk)=z;fpSFUJ1>r6c=1l4NAn|+VawM=|20g5UYPIez{8|#h;6i zC25S&gR~dEU0y?0N4N?VZVr2W9e@7{jA2)adP41?rJgqjDNB!`AOM`^3=%+y;A7fL%L+^HAY0{O1?gW7mBC+sS zg;MolS0cwW+7k1NNA#tF?!UXJZYP>`?JAVE^eRRW-GGoGzksjj8MI7=*yAdty{o?6`3 z+}LcNSuA^;WQ5+|)84wapH#SqzEiC_i_dx- zjS+`+ZbKP<$(S&knbTN=Jsm2i;1j}%F5-)EDifq!+RugY{F<|e4p2bM$0=euDO_O5 zUY1OQ1=9XaVGS2k!Z^$YvIkILEwt;w&k1)u2#!Yf1CmC_a7MOz8LYwfET&k2()xj4 z5=L7tc&c$;P_VkiJ_u1FDHR+_y#E5?T72IV*dGgPN!2A0hgj9vF$yy;*F&)9Dj_9? zF(>TxNK2r`h0P-Ps8n!ivxM}6<&-y;<;mYghm~Kn@=1{te=HN>_rXc)Vk1s5{}cf@ zGA)oMOnNY!AB6u)JW|pdk|;Z&6@f?g#G)-t4RtzCq4VYRZU-o97>h_T4w({DhDe6_ zrx5eBEUma;E$}J)6yKsBF{%Pa3qokUP$7RY%2)6j6?`@8ZYb@VMptxJ9x2AC(?r0D z-dRC!odBFd4PGZ10{|y7UErMqh!>&}EQeJ&+(-^8dK4Ji1iVaXO0NhL$H6hxHaHA#NfZiL> z0@~PuBecS%LHj)lr5vv)0Zo9xI!q@FGDCDoBSNoIAmYF_4-Y>~azSfk>LVYSQkx@n zHEVY6TvJn58|vr`*3ukF2(GC8qc_ghS~ZjFu20P^kE00*-yN+t;&?1_ zAL@M@ukB`etEERI*cM*gv-V3slWmsB; z*hOEK8nYN!M5Px6s4QY&04kWm!Y=nVt96?jFEJqLh)Ba?`@hECw1N}Yp?$x*s-k4u z6PkN8U5%Hfkq#gA>FyeK{EaWB9{u`P9!q^OcWF8`x_jrw^b5KcbkErC-DCF@FAnYO z>Dl?qlKvxLr;?wGBIPU>8ta5DgI>qxO$ZW7=0lSEVL>Kafuc(iJQ{RN7ADmv_I30Y z-)_h?1h8-1PZVDgasV_c+(bmm88%cvxwm2AvEJ{#OL$FRY15;&?SiL5a(5$gS(n{$yiNQiv|mJiq2XmbB6LtV%ZnFb z>e8>l6tQsyO~HCE`Z%MYC3qJ>TO<6Ou-m=2pHm1lh?%FL47`gAx(K)w!rD>^;rFx{ z_bvK84O?!7-}5`fZ*JRQcd04CA_RuK_IPd^Vor1)=su$*hNlmJHLdVl)RFQ1-KbT< znX)lb3|hy(c8qiw_kD~_gd31|_P38LE#Gy(YM<(?_)+Q($BO@@R07lRS@wQUc^A=0St)(r{b2RV>%P}q%j>+K{O@Y# zy~au9*WJSyMVX%7unzF6{JHXc`FO$4m(BOR>Xko3d7L#{_8gVH-)FCF>;L36jbRzA z%hwZm{o{l8$){wMTa^>algc-hpTqZfGn-lxVE@EzyqRbDX0Gx3_$T>`U}Med z4)vH?P=9H#8Fm>SFnrPQKMn61W5yxl9^=!-ADV)uoav`#pE+m#l=)}o%NCQR#?oOq zVVSeMX!*Y7rqtF@l3^cDs7b=m7|sWD<7`BVym{@Y&&Rs z#&)sFR5elcVAa!A->UitdyD;;{fzwu`w#6!N7}L3vDfi2$1{$-f2db8eJy$^Z|K7%jf zyV-Zx_oT1jd)MFWf3n6`^JL8%wQaR4YA0$xTKmP?AJi7>R@CjU`)b|y>)xunTyLvy zsb5jQqh70jp#JIlUo|KVS#Zz?8_qWr19br{@QJ`nfxm5RZd~1XTjQr1Uv2zlQ*+a? zrf&v^f+vD!gD(ev82nYJF?3t#Oz2yopElPu4>wOVpKAVU^Sj}i@agcY;h(nHTQ;`L zwmjYPot7)D$=3T?pKg6KVu-AdJQ?}xNHIDTor<1_J|F#WZ8dG{+h*HdZKuFn;+sEJ z_9GI3K3x2g4>MhPx5z87i~Y$W9UfL5*7FRWr~j(wDGKBN)$^*-!Ups_PD8RIdfuqm z*=O`T-k!r=g*3$sBoz}z$vlGv;=ky54r|8$t>;x`RQZ*jHz?KY4n1#F8rc1M-lX{0 z7nKp^Fy8h&sT{?xrUaEK)H#6sar_>|%!4>ja|q=}MS2+T z2Ae@y9QAvVwxPyR{LLx@uvPUad-b}M%DUak5tMeLg&EX?GCp#6X7cEa7M%J}aBKI* z?%4w(UQ9batSpXD>?kQfc>*z1;_Aj-rj5 zlxfismg1)ALkE!@&`T&)4xsD+(%&}n0gQg9m>13SZUK=#lu>z~(gnL)7iQUud=d>U z8`wZ_=fR@~j@~_^^#uoleO;NZcyAwSUEiFtSW!`Sp^L)+#sM*M>ZDu$261!d@R0+D z4hH+W@rUa}fanZH*R_0Nhh}FEc9mu)u~E7D5XO0<&reZ^Q^1Tfl^O6xCll;d7Q8X8 zf>kPOm34s524K!j%*Lufn;guEXr*fAW*+8cKG=b3SS_n#^$Y>PA9Iw!Sf-uimhgA*f1Mm zYuP%so^4>G>?XDmFD$;9-NH7rEo>{>#>Uuowu9|tyVwU{IODvpM#M>`C?% z`!xFudz$?R_F48h_6++Yc9wmfJUnc=!^5d1n*1oz7+3E^S%u4%ksW{ z-Z#nnrg+~p@6&kS4DZ{^$5T9>=J5=VXL-Dz$0vDwipQsUT;uT> z9^cCoy*$weuQE?0cp}LYDV|94M207_Jkie+lRPoS6Vp7Q@x%;I?B&T`p6uhvI8P>c zGRc!E1YPlDh9|Q;+0T=cJUPXa(>$s1f@<6PbJ`~=BX4XgXW~4Q;F%=PqgQ9Fd}@kMP4g*@PtEYDy?nZtPxtZZ zIG;}N=_H>{@#!?5&hY6hpYG?=lYDxLPfzn{jZe?;>AhU*w`~4l|1WJN*uYz)E%B3gjC&tIe>+`I0d_0_2w&rHW$Gh@sEVwS1 zH?&S-K*o`+xx6tvoHvDsG5qm7o9N0LVquIcsGT!T4F~Ct>^xsFl2<0y<<*W5N=JgH zf~U~(xn5)IscpH5t@V>*@|#un=G|;W9iN26)56 zlXFPd2MoSSKc1O1cJf5ZDb?O3z_inc)p6R#&A`I ztFF8Q%{T=}f`Gs@hMl*MOaxC&1oL(Ptt;=0ZQ7ALXVBJ;x8$p4!Y8`&uGpq+xlP+; zVSNbYZc$zxJEu5CcIM7G93y!)Ih=QN5`qG4htJvQrwTuL=EF*;ty^>F2x|eX;Zs;# z>b4^k#$%;?y}VD40PpGUIA*c|aRt$vF2nIrF6a%5O4FjRHJr-Oc@Vq02`8y|qBUpq9 zTC_=|`F298&RD*qGv9&j5(B1g07~6(zl0~VVWLyNwFdB|E8n%a2F#a_b>x}1S3tSD z94gCi^~8cHG0tApVe78nuAl-p92S);zOM>eyLKp?J=ep$m`NYzje*|qkqKb!WVS0G zk9GT3bmbGjt12*T8r73n3dPqN><(_Aoe2=$bn4WG@CHzV9OyOZ9ky$NAyN|kr$9n{ zz<&ITDtYTj=gg_@a4@*y6xvEJ-41rkHu46viCV$@1a0Qk+j3vwK{Z(a6}%9?P=mY~HN@&3D2JDSMB;$3hqQyx(+$sivU$77&VM~1hOELt5AbK}O zbQpwJ05n-qoVQ^227~Lv8>ll{t$qPAnt%>bWk;?%xB^U%Mywa2u_ch3T5)v~ZY{D^ zxlq?5*F;!f8H}+jKcJ6bq_i{>#CNX+Txlr>W8q*oL2W&#?uzm5bDhkCjkjX47^}Hd zymGNv)Gj@`tjPYLas1& zMK?By9OD`g3lQiEz|xCYmQXO-Y| zQ;g6tKMJsJjGb4MHOOp2hEe9`*m)*OZb3$rY^FNHxV44qP-ZLDq0Ba_LzywEGla}` zszaF_REIJ3CWBKf2?R|71YVQ|0s(nD@ zsOp`ueE(wAyXZnxy<6m{>OCSyRS(AU1B+D;(S@iwD{@rzgCa*&568X&|7J-t8t%+n zX7Xyw))T~Px)cc5g)s;q?2{nMQly?erx=GJFm%Y&vMl`uxQA7g=s8tcd#;5&vJJxG tBe`>`w)R|vu3oY{2>a6NN2Vb$p$g>T@pFo;#)kMsZl diff --git a/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.woff b/imgr/imgrpkg/static-content/open-iconic/font/fonts/open-iconic.woff deleted file mode 100755 index f9309988aeab3868040d3b322658902098eba27f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14984 zcmZ8|b8seK(C!=Cwr#($lZ~BhY}>Y-jcwc5*vZBlYh&9^ZhqhW{ZvpRobEY2 zRim2jc2|&)0Du6#g(m`l^xtUf0|3Fv_;2t37YPYfIRF6U=Qof04SefskYWWDCf0Ax zvBgA?Sg zQ{3X4{N{ANb;56uL&kuESlGIFd~-hEx-kF%7M7U{z_qbA{?BgvJGPPkQ1m-q%+}E3 zdtHw2HU7t!7$h5R$XB`1U|?VZ2x4oEo(?{~<9cW^U`%1|L<`O49o%ya3Cchk?TQjvHN{6At8vTKtqH+gT24Lz@);yzA(}YXmPMtu?=J) zB`AsehXP=+al-fk06b49&+lmeAMwbpQMYtnkU%E5*g+%ehk}td81f)!!euyQg~T*2 z)@9npKco9a9KNs1`!r1D7wjizEmb+j<)@`LL%3o_S^DOxFhSl--hj14 zM#H5aHC`i!yXJ}d7a=RP@L93co8&-xe2dITtXa!y%MBkDB~oaSX8=|B+}p%5@uonM zn_)dskE5dgxwy$B7UDtO_s#N{dQ@IiYRc?**2_dj%d{C+ob@a*k&~f+QCmvu@MvPv zXAzzv=m(mV@f35IWRg%#BWNS#Yb*+XqhW64orn;jVCARAp6(CT+dJl6*AU;? zM*P*yjc8Zknkp&+s)x#G((ur2&&kDr+QHf9@3~dEGc~r>L7*Gzy1Zi26w8WWema4O9nUHF1Ay`VkG|KN;jIkW!y|Iqm z_{%A18!12g;hLL=>v$cmr4i55J7qcYXU=B~yAkp<@s~C6tv|V{8@vThN7>Ar*+kUT zG#R!Mo!W$4Nb=yBdJDs4I&6_7L__a`awb5B)C3Ey=!p>9V1OES1_-UBB15l>gAY6! zgAcgD1lD&~n=am~Xzs0?{DhP>B#)UnBu6*&eKAo@JpMbD(YyVmvxqj z&@&kK=UwrH$rMA@KCPr0_vdj`DwkaL#P-jJHm=bJ?i!1 z8}!q?ktnS3m!tlo1#^A;Kj@_YSVeWK>j|c&ToS7G_GF@PG48OmO z9f5EK30J^t+iqJy*#ApP50`b1Itps9p(Y}?<(r0xM8Llb@Vv_bC)p7#QQo3mf&A%)o+*0URgNCG za4$QHzx$SKgZ`gRt#R0@*1!twSlSHhsoh;QsLMm8r|!LTG;ZrmyWdoHUi$My zm|}07P^J|LaHp^NgRiGf&NR(l5NXAon_%#8@W<{J!y{jdzW4$&DU}1qKxKQX)8XSL z?2mV_=`AIG5HC-7@$7A6{NO&-ydr#n74Uj&pF-Z$8y{E$zC4yusOM~M_{>Se`eA&?^+`>z6+^^e z-9zRTW5i&l^d`h>3TNz)Nke3o@P4#IaDYO_;5OYM^K&LQe2?L@Z-9NqAh8)@a0oa2 zBgZE0*v2lzCWIB9Dg+PnN60WgJt9X9;>y;|Kz%P)#Ht|n&;k+1CZVGLZfL=$4YG(l)XI zh)7x3yd;LHCXIWu%}triolkzfz}&Mv;H7!jBuw@gw*s$C$eu=Qa`1sc z5B}ui$H!Ce4T7GYUs-(D)QtlbRq-=L`#jXs?`*z*GJpGBAOxgH)eXYY$Hg~AG4DOq z=I=cl`sYCiMJzXE)U-~?69#ZqtZ&+AQf<3#MTmlm%g{%Umm_j2vh91ay zqv1Eg^xKZrziV{;&zZQAcXh9BJ$2;6V~=dAB!U$EAp{B=FqE%)N^YkP%oiRBdy5yc}^m({p@zFIc>%w~m)m9mf}!-OfW5B#m6e+P`6X=P7dmh0oT$%qeiyr_JA?e>=;4&-SO=&B8d&53>ph7P{!2UjA~-<}+y zPd{`k0wz%CSu^`360$||g)I7cO(uA+j+wedG2^l`$+y$zR;9Uh)P|Z7YDCGkDr?Emz*2pk z=&{N3d}iyDCb5)=dbZCriD^F425+7nvY$^RexMM&Y@~fu_8dox`Rv=J+(Qc9 zWn-qPasT@eA02E~FvN~G5E{6FE|YOYXW<6Lr~;=-HsGPY*-BMa)A~nN0YuSZvNR`; z?3GZSJ9gTT=B1hQ>?q8Z$4Lc+-+cJDeA2{i2Y;$GDd|}~D%QeStOPVz3q!BG*3_3< zsN9j}+#54rC}E;sx!5Odt+_wQl@-R;EOL%rm7PhG84}(HzEmEj=aMrK zIbG|+mgHB(oqX}A(s99tu1a)pigk_tAoUw~m?aQ&b3GAeI>XD0@EuIa$5l*WS1n*g zVJzBC98rNH+I+s$#v@W|d9@)RcYCycT4=Se+q`R8J-~u{;9-d3WS5+P6N)5m6Yiaf zW5r-x?=Ll_GwMmLqv7bF{L`WyIobWu>Q~t8YF*XhO1GVnn(*7@JyIqu1`U@KGOlS7 zDkIuCSkaEPKx|W0eg3B=i?9iL1FUT5wishps-be9I&>pL2hh8|-SBPq^WaW#5tOE~ zT}eCEtSL~gqcqjWVd7I9gOLIKbVX?4W{OO%%C0HvcP#h>_@M-fc}T%}R9KJL<`U9V zXu1u!HS7X0Ez~@YB)L|YW@u9W5-|tHX@2Vd^Q|Yoj6j=D&m1~FnIk%im7$;J?kgN=T59<}6@^cfW2XSeDIy;+ z;ETOlaWdwo5OPoV_ct=W{O6{#XMgMJ$9oeE-~m`CjpUZsw{hJ#0gvO&c?Cy}%w9Ms zF1qLs5n#X6OVn!u32_b_qY`#EKw4CB&te~7XZY(jWdCXUQ92kuUn~8)qF)SI2<%X% z$*37c99~#|tO)1lveW3!TBbb0&BE?sJ2VN2b`;e?d02KJA-GD}T=1K%plNHtYUYXp zgJD%O29qwCKm_~M0K>`K8^SP{D*2gCTZu`SM9S}-Ykw9zDoswD2oi?2TS?0j|YT&|8hjXaQoPL@9w`)i%-M<8&28g z`*F!&y{zlqjf@rLrt~FRSN5BK<&28)W4m>{vp08~u*1zMt6=`$Tiv_$EYw^6mW-W< zt8zy&d5h9t;u3Jj2lY=`hj8Cq$z7Jwz83FVg8EUT_;y_|+qcUF=C!0ITJ*U22Lx;V! zcKoPS=n8#~`Z=P6J*6*B$?-V%RjyUCCvVVwdl4E(WA=YtevNLvY$%)5Bc}Fw#;j-I z0#n6dHjW;Da&pE??)2+d3EbXdopfMeK@6A7^s%KeI88UNE8A_UQz9pRg$VLmUKJVl z4I&pPU<9*3OS$nt9-xj5K$8UbcV(lbl*jMiig1b^fo^TkNqIjEk~>Q^*t@Y56IUj>ezm7Kz-yTs!n(QG%R6u)`W@o3~fE4rr$BH|lu!66Zt>E+mol2P_*O ziCJ0f=UY}ApdzPxn7#+JwBo&4_`u(lc$Y5=bBVwn<&r;>yAaRJ-31VEoTj>*61yyd zp3YVTLPv?QW5862ulNZ1OgO37-b6gtqu(;CiQAmQ# zCr+Ycyg+WEcZ!?X&fSUptp-8 zOKi8O!M8Q-*Qu1ps0AggluG*V^1Nk{%4)ki%nw(VY+snRW|#=(2QwJB9_$3%HZg&v zGierEtLuJ=$|~f4f4fwK5=?TPAjUyj8Yew=i=kkkgavOh6g$X3)xPOz)zymuI+`8M zw>dd|>IZAe!R{&|(y{JJk1V~blgfVPyc@hkWl%sl(2&%1_ zBayVylj>~>f=ABwi~c<+Iw4?r-Y>*Ha5S^04!G0F`%{@_*=~3GPH#N7wy(VW#9K~% z^A}g?O}_Q?lKt*@WTk_H-hSSv3-$^pR130pW(KZ(yEogRXYxqJ=3(mI^u9}QZvQ-a z((-M|R_NJHj9Leb)GgW74j^HIe+xHZ9kE0~@bpOQ{p$rbO7MWSD}JS|^sjCkYlGuC zUORP_Sk^=&Xl>}jo)cc3(U8>A$EKMhU3Op5&q?!5bIRWKQy#{mHJe~z zpD_@@wKexPN7*mrUJtXFETM6Et`^w$d}C!Oti(ItQxZ<}ac+wqpcwP31>V3Xy^R=>z5USMBZKK+o&=70h3Nk7J|rhq`+&2=kGz zbKt(1>sMjxt*%JtH0X1QUjjrO+!WGqJ~>^oI7Jo_J)Kc&*z0~air!w9jp!g4?wfgq zJL+up-MtWP-#IVzI~_ZIvZ7?AAS3Z;mPEnwP_cT! z*JJkw8oBTf-J3$s=O1WSr-_ar>?Lq(5SfWB(V-~fojAhaKW3_-Gv)6Cs%N6kHOpSA zcS_*;`P_me1{t2on+Vr1a$ReDFnK`uz3Z3nG7l^pUjIFTxC`QjIs zw*4v<4CwC+ww4{v+O69!bR4?vCk|s{UsX-Jfap8;>_AXh$l|f<;E74Cz!jC7G9IXy zRd53A1wnR`fLa1lq+bZjJc+3|#A70PRV!DqsMBI+{Y`^Fjxpas$8>UHzBCi7^C*i6 zK(hW0jN5kPJk|E<^L0~z;qgZas_$AoR&%@#wjhOvWDm=21DL3NucshN z&4&0NC>nxBdAUC#X!+LbzQ^kjjbhE1k1OVX7~$`<-c{$9+pA7>tr~|B)r7k3PQii)1bP3cLR~PA43g zv4&593)87tEg~Q62W|9|3QnF4m?e!IAcZS5Ibl^1YcsARB`ADY4@045znu~7a01Rh z>+l$JuFC|4z7hK3+kCD|DCv!`W2+C<_BhK-N=Y> zl~TeiuMqwCt^g2?J(W(R_x%hzZ2vT01(hBOkf{W6GNbOatvp{|VWfZ@Gaj%s85B1e z{1-eVWEKKhhEWhGjoh&iS!ze1fT3o7ow#1s4uhlLS<=;VminN4iuf0PSxB_tM4{Q*zUBpS#fqtC8M||{+PW- z5(wRsj(WEBgf#w`o)_kNV2gkk)eH-#tUQ@!r1^IZh&ZD0`?tbafwU1|CVhznf zNcNSz+~+>zhi)M#9b%<-D2l7HP?UKitR+ZD(RSuH;DtL1{iZh<2ucun!sawL z`=q-fJdKD;G+Bv51liqQ+tU(A>7MJhhOnA&5qu5Rl=-K7=a^Bc5AfVym}bjN8}a31 zSC+FQ2;YpbwsQh&KyheTK+B>WMu-W!SdTKbq+HdKtis?NxkRxZ$qSeOCGaBhz|Z(DEp*18 z1VY0=kluAfiGjwwj;QdjMMGCGU*OjKSx<7Ei}Qj)i@i@!ss5pK%B8wKW43@}FZc$1 z-YoNXL5^b2WSlRy4ve@Z5jq~L&dXc<&fA`H7{ix;`+e}9bh&Hz9biU!LH$`ro>n{E z60{dR1cz+zB{R$pgoATCvTD1<7#BtK@y^5If#X$}l~ytQCQx-!#mp8tbkW2!!BzcyD)40=2|*Yu0mzK2QhCp1h#(R@$2;3wHfiXgEyLjy>&XZ{&M zX|0LbwAC69Uagm>U>z2#~Po-F%98OE1a8pWC?$^=_E$3P3gIXP#XRT!S%HmE3Nof?Q8}oXNel$6zZ6o5zeox?V*DP z#;gc)w7}{?5S6x8>d);zSK@Bkb2cjyb4fpGEQY8yvG{d=<)f#aeV&c7cz}dINU$Mi z(%?!S-H5nn;V;BHL`q}2RFUQG#`yzUbSbPC|xe%Okxc%);L zG_IfQ50^C{^A+S3h12axEIV`>eqL^5>t|45rId@hnBdprP!y7Z)cQ%p(8ARJ5fkIp zsXBB>UB(p=2!Bb&w+Ydbzv(Zoq=hleRCOX?9E-CqQnFv*KyBvL5g10fl#6st3l1r^ z{nu}0VD+#h3EPFLP)&G6MVtXL zojBMIJEED*owWecK9Axcvs^)EyxTG6kCj#khg~RI92J@%q-I~YswpGSNItHCSVz-Z z$aI%XJe@qt>YU7K`DFEY%(uxUQNk=Y1!MdKB!^j3lDhl& zB*r^qUR%{ANk;qd1q6@ttEMdwk?leq$2=`&Sl6|!Y!1R}KfWg7%;x6J6}JEmGNXFm zg|_y^m62>BRdyx`Y%_8b#P`(XCq2~>tsGTcLL!`UA*V>h`1J*&%T zdIHFYXJMi^OA7M~hfB<*ZueY+JM&>+Qfs#=kiLtfx0Ft)66%I_u?evJL21EhB1K~o z`y+e<;GfX>bBQsII2~e7232`QBzVq9t<1BI9gB&3v^Ec(tsL>=LHPD(3RZhi>+eHu zd|8z;=K=UNDEvmBsN1(=_6jNRl;dDjM9kO}*MC(c^F3lY{V&6y`f`AQZw?~-MqNy@ zTjAUYNJv+3iVw0y+J$1+cV)GLRf00|eV_EtDGG}ZM`MgKy1E3@Y68%4IWb*yvmw;1 zW4+u|$L@h*3@+;&b&FewrGx#rG#a-Y6k`B#0lUWXJ{=|geA4hq+^u1speQWAISOkxN6G2HT#(@9Tx^dB9XN_J?3OOn|~ zl$aAWj7%vg4nFC>fH5@o+O&Bq=Yw0FizVKxE{rDu<>BtzXAf=xem*|A%c3k`_IB1; zS?QAC^M3G%gl?zt#n9;@+H;`p^q*0YcXU&pIoTNQ@}1(qL22#*r= zZZi_}Yy%6t5zSkDn-$(McjvFXR9jx!dN;Or+L1<0IbO;R%_-O(w+5pxh#!$=qJ4Y4 zYD|XROqif~U`MF-?cxEZyv;j173tj z-YY(e%y5_KiS|+MCa32c^uh!YtRyu#U+7JX-2>9+vtNsXrX)PoX~9gbOv0o7fgfj} zB`?g8I*)BLm-MV-8F|9RS6zfd%mWs5oU49T_0Hc?R!?L211om!o0F5?OCs*R=6-{c#%b^7GQ}uK~jPH z!qWw1S0j(t4IW+yW|v#OYAN)jCMFo4AluBz$FX=j+Sk*9N}jv6sek`8*blveRYyK6 z@$$QlJR0o@v$S+f-zsLw0nh#kUV&fD{$c1Ky*FirKmqzg+)FWg)*qYr#!&xh)r5FM zyIhdtLDGe=z-F!B!f`gKQ;5@DmkA~JFJ)}&q2vWU*3SVpi6R6uxf)tZkEGzFa5#xh zgxWZZW?URJ?Z)bcPP-?uZsE@O`(e|((Jc)+yo;i4MIL;)hlm(2w741^jymCajG}`Y z0+9`yJ4PswEoFzGwoK&Bt{R)>WKNgeyhyZZrCWq%%VuYWOSZTCmc7B@AINXaIYw>g zD(_7~W$3#FFPFybE@REcF<7d=>Bl!Qs|)m~SLEeCXQD;JBti`=eSRQFLEkCdcI{wy zZh^j@{zDOlr}L}zgS3@RiQBzf2Jwro|}z zp(8`DShFcww4*$ph=`Zv&Qf;2lWqEvw#uf03PUx5*6Zt_ixy%t9Lsse#_!)n3$--l zOf$;2nUJKM8%rIVj%qU1>XT_ym2MR4aaD{P*8oOSZgIqcWfWlkoR%D~ll0=66q}CTgR^m^OW6AzkH7eH)iozB+LoEQPHk( z#`+MS)QEj`X~>v7ZPYe^*p)Xt3}Ja0T^Df?O^X*F|EApS<~55@Q05SkK0sF+UD=#y zt7#A&M)vf*n^sI0F~cOr_VJvOH0Xd?%4c zS9%8jMQZ#au03wIpvh_4m~jGGx}6aI{d!htmWrf+Ec501JY=~N`(k@SGWn!aRsfxN){B8UN2djrCZY-c;VfAmwKt~0mYbZs}* zN)bzhWb*t}1j2|hWp6O^-@hIy=snZ+vUl(7haLy(cRSqP)j6yC>k9j)-0U_2f`oC* zDq6$j2-(gxSw{;!Dp96XDiCcn<=s}RfXP?}T|Y2spwLwsB6ETb1}TfF=R{7Hzpnh5 zA8mde1`9$mIOIAp6)$HGzWUmv@fqHkz82Ew-Q~St6-GJ%T zoE#?-c3l0~iaA9*ZHhlS4{FA<9Xf40OlkBmvD;}@=7o63Ay)&<*d*Y$1s;!ljpE;>z#T%*x>L7ZnjI45Ij{?bC*!?k!+qG ztdZ3sm+s_sl6t;4RC2XWn51!HZA6K~SFd{_-)wmP_l?z2qE~E~<2OIQ+O+`I`?nv4 zTY=XT@qB)6R50(?106eq%h-+tvkEe1h`*@lmM&+x3DEC^osEhDdqcgXu%ke2MH&Xk z1C-O3ZCc_QBqYIvgg?eabiv}wJFj##c2D8mmh`lixXcu@YxCQrG8!B!t|Fs3VzCQ; z9hr_t$>&PsMb)7~T9Gy2%f@h*+#5)SQ1_;4J^h9y10)bshZ z;l2nhm_6Q$h;b}ZWEkFj``_4Ccc@<0bZ^yIU;nEXlUv%4ty-&3ERH>Fs*hBk2V4(@zX=>s`_S;> znv9FMT_}=x6fgK5Eocs51k=oLfx-1*kl`Xt-`Wy>}^8>`FDC3BHmx0tiP7SUAm<*Y2o55|>ORCS?h9s0JBXbw;#Cph$cb&794ji= z+q>GiW^0_In6F@|`Go$PG?<~CdAy08(5Tw{%|4#eF}0z$P|{heEvSj_fb)BSxH5<| z05&!eJ_hd`J6pRTn3-`De*kX~6ob6;5$76=(raIQ zLf|D#m~aFvX;k~)4ngj9jDkYEH>=9Bl0Y4lFbo2hwZ;8SM5yle*pjPB#+xSFQmlZS zx-6>M44W~rAali^78Y#mRKbxFx=eMiUEa9z(ucTGd4XT}DvL>5sH(2)4?_+6KO;-8 zrn@NfBWJqrmF0aeV)74j{RNieoN=x1WWDtZBl&cYz_p4>6*bDFG3D`jit{?pN}=Kb zA$HRnUz77!U1Y__9o>Mc9eAhu-xJAe)|vDDd>|D0$V1~)51#MF`!ucYiH0PDBh7hd zP@~9L9U6_>0ITN)i|*;n^J#Cuv4^nl9;%&+iqY3>S?5D)G#pDe#$!hX0bHuh9I~vq zA2D4T@VATH2!##Rj~ya`D*lSE^NQsk@^8~~tHFwqGoQhqMQ94Y#*!-iK3j^ml#r&i zOqazq3pA5ARb?ZISzwF}DezJS|A=-F4_sjNEx`+yGyRH{IhD+PA05?2fF70oRRvbTyn=GafV{2>-SOR5)yp}dOVJQnupdB__2H{ zi%Re7Q-_+nW%M@Y$ImbA3k6IhfhQs^_th%;8QPSFoVu@2dYLVA7&B7wEV3z3DWY|4`dJ^1W>(H5b9w2ewH26TeK*KTVdYH@0yhXow`Vt zEiQb%wNti%zh@KY^!l}LTgdz&+oC$>Osld`vBzQUXWP=M-9c}NQL_(n4;71kn5XGo zmVOZ3ksQkzy(!yLlj|9MYY%lc=Ah@ZOz?K%F2w`tdy65K9JF()4*MSTo^&Wn?TB3P zh4PYQtzNI2laZ^V1u@2%VYXofo#$f9?} z{g5ky{arkjo0YZngdjFBkKC`Vo`@ZkWNC`C_ZF7g_;LQ^=gJK60isc0nfD||;QbLh zqm?XPW>-Ds0dZJbpO zb}am_%z^ldSG0U6@a*@mqlI3hkR}r6(>VCjfiSOI46I~*s;(97Ro)8+>zQ@jlv$49PArKvxkxgwBdB;#)2(4-!CdDVF!4L+<>%U)0rggTDio~bmuS8 z*DD7#>a9n~qz&fVQ)Srb$Y8w@3@3OW!=V6HjEqk8@ilHta1dF<-HO!0i~(!}5~#<= z!n4PX!FG>le~I^w5dGJxZstqGGH1pB;o}eE(Eh6Be7L8vtB>x7O+Oo_hROX4XeF%iNrNuDbMF%%Fj5&tjH zZ7s_!M;$vi4iUxIB2MrA(l$%5jD^&&(JiBh?Iq~B=emhrk`8_i{Ffx(xx%$@JBb4$SlNt~?WQ(N zrbFis>F-n+Ewf$L%LDR}95)U!ev7AlHLtPc>%(EeK6Xt72Nfmhq@VH#)l!BvMwO(w<36$uo$fW(#UmwvEP`o}J zPq{_b+bON@JG)PrK_|W_HmDM^PA|s$o1Y4khOl?^I?z#%nE! z{XC7pZ{9)DmQ?j7%D20V@pyT&Qdj#Tq9{+FAHx6pAWx)0Eu9L z5P*=4FobZ6NRH@+n21=7xPVTSv+KMKCW`On=9T!~!Jpg?S1Asw@0mRV42*4P_1jnSrl*M$yOvfC< ze8(ciO2@{;PRE|bp~m6EF~AAJsl@q<^NGucYk}L0JBj-b_Z|-(j~tH=PZiGu&krvf z?;0O~55)h8AAsM8|4D#LU_uZ>@SEVAkd#n}P=_#?aDecVh?K~UsE=5H*n_x`xQBR& z_?m=}M294iWQb&!6qi(l)POXKw3+ms44W*0Y=CT+9Fbg_+<`ose1!a!f}O&PBAa53 z5}Zw{%81H?s+?+r8k<^z+JSn2=DS1cf3GEvp@e?oJ^-k!K_hm=RJ*f~ zEPy^8)bGD}--KRiQ5NiBg;%7?zy1B=B*CHtc5B`!uGQRYFqnRBRXcLS z5pE{wla8bepSRui&#pNdE4gXH30(*{{GCl_2&(6MoneF?{$&T+Oa5g?MnXO=2THwJ zNyu0l{80#UvlT~tQNytW?0(Xc(S$a90`+1L4jIB^YnjWGh~q2PwiAbQyrJWIs()GM z-LTx|QI(~BF!yZyu3jYOyxi)d6q1}%F&nsTiNOoMg)@>4DswO zd7&f@=3|L%Ce-$h8rp+jmYY_uB#UFDQ4=Lb^GwKDnU=3`E4&nCwr*b=o=B|s^hs1R#V!agd6;mD@GGo*1m^2txCCYJ=jET}Lb#)NzldN#7*)#TZtJX7)bZh()DN<&DULB-z4J%ASOCDOS zi0&0yIg1V%+Atv2pu!%dK1bsWTZ|X)or9^6BWGs)3I=Y28W_*KeR-jvY4B^gK*h{y^sAn)+SUTnDOF`orBX|!{9+a4 zVtJ-&laFDBi^D=mo7d6d<;Dz!8i#DF~u*T d`d@*P)=+z2O9=Gccp2C_0H}G=_V0V@{{Zm~b;kez diff --git a/imgr/imgrpkg/static-content/popper.min.js b/imgr/imgrpkg/static-content/popper.min.js deleted file mode 100644 index 8a17212f..00000000 --- a/imgr/imgrpkg/static-content/popper.min.js +++ /dev/null @@ -1,5 +0,0 @@ -/* - Copyright (C) Federico Zivolo 2019 - Distributed under the MIT License (license terms are at http://opensource.org/licenses/MIT). - */(function(e,t){'object'==typeof exports&&'undefined'!=typeof module?module.exports=t():'function'==typeof define&&define.amd?define(t):e.Popper=t()})(this,function(){'use strict';function e(e){return e&&'[object Function]'==={}.toString.call(e)}function t(e,t){if(1!==e.nodeType)return[];var o=e.ownerDocument.defaultView,n=o.getComputedStyle(e,null);return t?n[t]:n}function o(e){return'HTML'===e.nodeName?e:e.parentNode||e.host}function n(e){if(!e)return document.body;switch(e.nodeName){case'HTML':case'BODY':return e.ownerDocument.body;case'#document':return e.body;}var i=t(e),r=i.overflow,p=i.overflowX,s=i.overflowY;return /(auto|scroll|overlay)/.test(r+s+p)?e:n(o(e))}function i(e){return e&&e.referenceNode?e.referenceNode:e}function r(e){return 11===e?re:10===e?pe:re||pe}function p(e){if(!e)return document.documentElement;for(var o=r(10)?document.body:null,n=e.offsetParent||null;n===o&&e.nextElementSibling;)n=(e=e.nextElementSibling).offsetParent;var i=n&&n.nodeName;return i&&'BODY'!==i&&'HTML'!==i?-1!==['TH','TD','TABLE'].indexOf(n.nodeName)&&'static'===t(n,'position')?p(n):n:e?e.ownerDocument.documentElement:document.documentElement}function s(e){var t=e.nodeName;return'BODY'!==t&&('HTML'===t||p(e.firstElementChild)===e)}function d(e){return null===e.parentNode?e:d(e.parentNode)}function a(e,t){if(!e||!e.nodeType||!t||!t.nodeType)return document.documentElement;var o=e.compareDocumentPosition(t)&Node.DOCUMENT_POSITION_FOLLOWING,n=o?e:t,i=o?t:e,r=document.createRange();r.setStart(n,0),r.setEnd(i,0);var l=r.commonAncestorContainer;if(e!==l&&t!==l||n.contains(i))return s(l)?l:p(l);var f=d(e);return f.host?a(f.host,t):a(e,d(t).host)}function l(e){var t=1=o.clientWidth&&n>=o.clientHeight}),l=0a[e]&&!t.escapeWithReference&&(n=Q(f[o],a[e]-('right'===e?f.width:f.height))),ae({},o,n)}};return l.forEach(function(e){var t=-1===['left','top'].indexOf(e)?'secondary':'primary';f=le({},f,m[t](e))}),e.offsets.popper=f,e},priority:['left','right','top','bottom'],padding:5,boundariesElement:'scrollParent'},keepTogether:{order:400,enabled:!0,fn:function(e){var t=e.offsets,o=t.popper,n=t.reference,i=e.placement.split('-')[0],r=Z,p=-1!==['top','bottom'].indexOf(i),s=p?'right':'bottom',d=p?'left':'top',a=p?'width':'height';return o[s]r(n[s])&&(e.offsets.popper[d]=r(n[s])),e}},arrow:{order:500,enabled:!0,fn:function(e,o){var n;if(!K(e.instance.modifiers,'arrow','keepTogether'))return e;var i=o.element;if('string'==typeof i){if(i=e.instance.popper.querySelector(i),!i)return e;}else if(!e.instance.popper.contains(i))return console.warn('WARNING: `arrow.element` must be child of its popper element!'),e;var r=e.placement.split('-')[0],p=e.offsets,s=p.popper,d=p.reference,a=-1!==['left','right'].indexOf(r),l=a?'height':'width',f=a?'Top':'Left',m=f.toLowerCase(),h=a?'left':'top',c=a?'bottom':'right',u=S(i)[l];d[c]-us[c]&&(e.offsets.popper[m]+=d[m]+u-s[c]),e.offsets.popper=g(e.offsets.popper);var b=d[m]+d[l]/2-u/2,w=t(e.instance.popper),y=parseFloat(w['margin'+f],10),E=parseFloat(w['border'+f+'Width'],10),v=b-e.offsets.popper[m]-y-E;return v=ee(Q(s[l]-u,v),0),e.arrowElement=i,e.offsets.arrow=(n={},ae(n,m,$(v)),ae(n,h,''),n),e},element:'[x-arrow]'},flip:{order:600,enabled:!0,fn:function(e,t){if(W(e.instance.modifiers,'inner'))return e;if(e.flipped&&e.placement===e.originalPlacement)return e;var o=v(e.instance.popper,e.instance.reference,t.padding,t.boundariesElement,e.positionFixed),n=e.placement.split('-')[0],i=T(n),r=e.placement.split('-')[1]||'',p=[];switch(t.behavior){case ce.FLIP:p=[n,i];break;case ce.CLOCKWISE:p=G(n);break;case ce.COUNTERCLOCKWISE:p=G(n,!0);break;default:p=t.behavior;}return p.forEach(function(s,d){if(n!==s||p.length===d+1)return e;n=e.placement.split('-')[0],i=T(n);var a=e.offsets.popper,l=e.offsets.reference,f=Z,m='left'===n&&f(a.right)>f(l.left)||'right'===n&&f(a.left)f(l.top)||'bottom'===n&&f(a.top)f(o.right),g=f(a.top)f(o.bottom),b='left'===n&&h||'right'===n&&c||'top'===n&&g||'bottom'===n&&u,w=-1!==['top','bottom'].indexOf(n),y=!!t.flipVariations&&(w&&'start'===r&&h||w&&'end'===r&&c||!w&&'start'===r&&g||!w&&'end'===r&&u),E=!!t.flipVariationsByContent&&(w&&'start'===r&&c||w&&'end'===r&&h||!w&&'start'===r&&u||!w&&'end'===r&&g),v=y||E;(m||b||v)&&(e.flipped=!0,(m||b)&&(n=p[d+1]),v&&(r=z(r)),e.placement=n+(r?'-'+r:''),e.offsets.popper=le({},e.offsets.popper,C(e.instance.popper,e.offsets.reference,e.placement)),e=P(e.instance.modifiers,e,'flip'))}),e},behavior:'flip',padding:5,boundariesElement:'viewport',flipVariations:!1,flipVariationsByContent:!1},inner:{order:700,enabled:!1,fn:function(e){var t=e.placement,o=t.split('-')[0],n=e.offsets,i=n.popper,r=n.reference,p=-1!==['left','right'].indexOf(o),s=-1===['top','left'].indexOf(o);return i[p?'left':'top']=r[o]-(s?i[p?'width':'height']:0),e.placement=T(t),e.offsets.popper=g(i),e}},hide:{order:800,enabled:!0,fn:function(e){if(!K(e.instance.modifiers,'hide','preventOverflow'))return e;var t=e.offsets.reference,o=D(e.instance.modifiers,function(e){return'preventOverflow'===e.name}).boundaries;if(t.bottomo.right||t.top>o.bottom||t.rightwindow.devicePixelRatio||!fe),c='bottom'===o?'top':'bottom',g='right'===n?'left':'right',b=B('transform');if(d='bottom'==c?'HTML'===l.nodeName?-l.clientHeight+h.bottom:-f.height+h.bottom:h.top,s='right'==g?'HTML'===l.nodeName?-l.clientWidth+h.right:-f.width+h.right:h.left,a&&b)m[b]='translate3d('+s+'px, '+d+'px, 0)',m[c]=0,m[g]=0,m.willChange='transform';else{var w='bottom'==c?-1:1,y='right'==g?-1:1;m[c]=d*w,m[g]=s*y,m.willChange=c+', '+g}var E={"x-placement":e.placement};return e.attributes=le({},E,e.attributes),e.styles=le({},m,e.styles),e.arrowStyles=le({},e.offsets.arrow,e.arrowStyles),e},gpuAcceleration:!0,x:'bottom',y:'right'},applyStyle:{order:900,enabled:!0,fn:function(e){return V(e.instance.popper,e.styles),j(e.instance.popper,e.attributes),e.arrowElement&&Object.keys(e.arrowStyles).length&&V(e.arrowElement,e.arrowStyles),e},onLoad:function(e,t,o,n,i){var r=L(i,t,e,o.positionFixed),p=O(o.placement,r,t,e,o.modifiers.flip.boundariesElement,o.modifiers.flip.padding);return t.setAttribute('x-placement',p),V(t,{position:o.positionFixed?'fixed':'absolute'}),o},gpuAcceleration:void 0}}},ge}); -//# sourceMappingURL=popper.min.js.map diff --git a/imgr/imgrpkg/static-content/styles.css b/imgr/imgrpkg/static-content/styles.css deleted file mode 100644 index b39be2f0..00000000 --- a/imgr/imgrpkg/static-content/styles.css +++ /dev/null @@ -1,85 +0,0 @@ -/* - Copyright (c) 2015-2021, NVIDIA CORPORATION. - SPDX-License-Identifier: Apache-2.0 -*/ - -.table td.fit, -.table th.fit { - white-space: nowrap; - width: 1%; -} - -body { padding-top: 70px; } - -.no-margin { margin: 0; } - -pre.code { - background-color: #e9ecef; - border-radius: .25rem; - padding: 20px; -} - -.clickable { - cursor: pointer; -} - -span.jstExpand, span.jstFold { - cursor: pointer; -} - -#btn-back-to-top { - position: fixed; - bottom: 20px; - right: 20px; - display: none; -} - -/* jsontree.css */ - -.jstValue { - white-space: pre-wrap; - /*font-size: 10px; - font-weight: 400; - font-family: "Lucida Console", Monaco, monospace;*/ -} -.jstComma { - white-space: pre-wrap; -} -.jstProperty { - color: #666; - word-wrap: break-word; -} -.jstBracket { - white-space: pre-wrap;; -} -.jstBool { - color: #2525CC; -} -.jstNum { - color: #D036D0; -} -.jstNull { - color: gray; -} -.jstStr { - color: #2DB669; -} -.jstFold:after { - content: ' -'; - cursor: pointer; -} -.jstExpand { - white-space: normal; -} -.jstExpand:after { - content: ' +'; - cursor: pointer; -} -.jstFolded { - white-space: normal !important; -} -.jstHiddenBlock { - display: none; -} - -/* End of jsontree.css */ From 49da6f1afe7be27b825d0c3321c532ba1161d9ed Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 17 Sep 2021 15:12:54 -0700 Subject: [PATCH 093/258] Added in placeholders for leases in iclient's embedded HTTP server --- iclient/iclientpkg/api.go | 12 +++++++ iclient/iclientpkg/globals.go | 4 +++ iclient/iclientpkg/html_templates.go | 16 +++++++++ iclient/iclientpkg/http-server.go | 51 ++++++++++++++++++++++++++++ imgr/imgrpkg/html_templates.go | 2 +- 5 files changed, 84 insertions(+), 1 deletion(-) diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index ec505e39..cb2edba0 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -40,6 +40,18 @@ // This will return a JSON document that matches the conf.ConfMap used to // launch this package. // +// GET /leases +// +// This will display the state of every lease. +// +// POST /leases/demote +// +// This will trigger the demotion of any Exclusive Leases held. +// +// POST /leases/release +// +// This will trigger the release of any {Exclusive|Shared} Leases held. +// // GET /stats // // This will return a raw bucketstats dump. diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 8c828bcf..20ad4b1e 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -42,9 +42,13 @@ type configStruct struct { type statsStruct struct { GetConfigUsecs bucketstats.BucketLog2Round // GET /config + GetLeasesUsecs bucketstats.BucketLog2Round // GET /leases GetStatsUsecs bucketstats.BucketLog2Round // GET /stats GetVersionUsecs bucketstats.BucketLog2Round // GET /version + PostLeasesDemoteUsecs bucketstats.BucketLog2Round // POST /leases/demote + PostLeasesReleaseUsecs bucketstats.BucketLog2Round // POST /leases/release + AdjustInodeTableEntryOpenCountUsecs bucketstats.BucketLog2Round // (*imgrpkg.RetryRPCServerStruct)AdjustInodeTableEntryOpenCount() DeleteInodeTableEntryUsecs bucketstats.BucketLog2Round // (*imgrpkg.RetryRPCServerStruct)DeleteInodeTableEntry() FetchNonceRangeUsecs bucketstats.BucketLog2Round // (*imgrpkg.RetryRPCServerStruct)FetchNonceRange() diff --git a/iclient/iclientpkg/html_templates.go b/iclient/iclientpkg/html_templates.go index 1b1de2c2..09e2d48d 100644 --- a/iclient/iclientpkg/html_templates.go +++ b/iclient/iclientpkg/html_templates.go @@ -30,6 +30,9 @@ const indexDotHTMLTemplate string = `

+ Version %[1]v @@ -70,6 +73,16 @@ const indexDotHTMLTemplate string = `
+
+
+
Leases
+

Examine leases currently active on this iclient instance.

+
+ +
@@ -106,6 +119,9 @@ const configTemplate string = ` + Version %[1]v diff --git a/iclient/iclientpkg/http-server.go b/iclient/iclientpkg/http-server.go index a865c8e2..725419e7 100644 --- a/iclient/iclientpkg/http-server.go +++ b/iclient/iclientpkg/http-server.go @@ -111,6 +111,8 @@ func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, reques switch request.Method { case http.MethodGet: serveHTTPGet(responseWriter, request, requestPath) + case http.MethodPost: + serveHTTPPost(responseWriter, request, requestPath) default: responseWriter.WriteHeader(http.StatusMethodNotAllowed) } @@ -132,6 +134,8 @@ func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, req responseWriter.Header().Set("Content-Type", "text/html") responseWriter.WriteHeader(http.StatusOK) _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion))) + case "/leases" == requestPath: + serveHTTPGetOfLeases(responseWriter, request) case "/stats" == requestPath: serveHTTPGetOfStats(responseWriter, request) case "/version" == requestPath: @@ -180,6 +184,18 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ } } +func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Request) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.GetLeasesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + responseWriter.WriteHeader(http.StatusNotImplemented) // TODO +} + func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { var ( err error @@ -222,3 +238,38 @@ func serveHTTPGetOfVersion(responseWriter http.ResponseWriter, request *http.Req logWarnf("responseWriter.Write([]byte(statsAsString)) failed: %v", err) } } + +func serveHTTPPost(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { + switch { + case "/leases/demote" == requestPath: + serveHTTPPostOfLeasesDemote(responseWriter, request) + case "/leases/release" == requestPath: + serveHTTPPostOfLeasesRelease(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusNotFound) + } +} + +func serveHTTPPostOfLeasesDemote(responseWriter http.ResponseWriter, request *http.Request) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.PostLeasesDemoteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + responseWriter.WriteHeader(http.StatusNotImplemented) // TODO +} + +func serveHTTPPostOfLeasesRelease(responseWriter http.ResponseWriter, request *http.Request) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.PostLeasesReleaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + responseWriter.WriteHeader(http.StatusNotImplemented) // TODO +} diff --git a/imgr/imgrpkg/html_templates.go b/imgr/imgrpkg/html_templates.go index c1b4c561..fc54e747 100644 --- a/imgr/imgrpkg/html_templates.go +++ b/imgr/imgrpkg/html_templates.go @@ -76,7 +76,7 @@ const indexDotHTMLTemplate string = `
Volumes
-

Examine volumes currently active on this imgr node.

+

Examine volumes currently active on this imgr instance.

  • From 740f310b18a039ab50ece9627a0bba7f726d1268 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 17 Sep 2021 17:37:33 -0700 Subject: [PATCH 094/258] Beginnings of iclient HTTP server's "Leases" web page ...currently just clones the configTemplate --- iclient/iclientpkg/globals.go | 42 ++++++++-- iclient/iclientpkg/html_templates.go | 59 ++++++++++++++ iclient/iclientpkg/http-server.go | 116 ++++++++++++++++++++++++++- 3 files changed, 207 insertions(+), 10 deletions(-) diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 20ad4b1e..a5774eab 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -40,6 +40,26 @@ type configStruct struct { HTTPServerPort uint16 // To be served on HTTPServerIPAddr via TCP } +type inodeLeaseStateType uint32 + +const ( + inodeLeaseStateNone inodeLeaseStateType = iota + inodeLeaseStateSharedRequested + inodeLeaseStateSharedGranted + inodeLeaseStateSharedPromoting + inodeLeaseStateSharedReleasing + inodeLeaseStateSharedExpired + inodeLeaseStateExclusiveRequested + inodeLeaseStateExclusiveGranted + inodeLeaseStateExclusiveDemoting + inodeLeaseStateExclusiveReleasing + inodeLeaseStateExclusiveExpired +) + +type inodeLeaseStruct struct { + state inodeLeaseStateType +} + type statsStruct struct { GetConfigUsecs bucketstats.BucketLog2Round // GET /config GetLeasesUsecs bucketstats.BucketLog2Round // GET /leases @@ -105,14 +125,16 @@ type statsStruct struct { } type globalsStruct struct { - config configStruct // - logFile *os.File // == nil if config.LogFilePath == "" - retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" - fissionErrChan chan error // - httpServer *http.Server // - httpServerWG sync.WaitGroup // - stats *statsStruct // - fissionVolume fission.Volume // + sync.Mutex // serializes access to inodeLeaseTable + config configStruct // + logFile *os.File // == nil if config.LogFilePath == "" + retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" + fissionErrChan chan error // + inodeLeaseTable map[uint64]*inodeLeaseStruct // + httpServer *http.Server // + httpServerWG sync.WaitGroup // + stats *statsStruct // + fissionVolume fission.Volume // } var globals globalsStruct @@ -246,6 +268,8 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err globals.fissionErrChan = fissionErrChan + globals.inodeLeaseTable = make(map[uint64]*inodeLeaseStruct) + globals.stats = &statsStruct{} bucketstats.Register("ICLIENT", "", globals.stats) @@ -276,6 +300,8 @@ func uninitializeGlobals() (err error) { globals.retryRPCCACertPEM = nil + globals.inodeLeaseTable = nil + globals.fissionErrChan = nil bucketstats.UnRegister("ICLIENT", "") diff --git a/iclient/iclientpkg/html_templates.go b/iclient/iclientpkg/html_templates.go index 09e2d48d..b72c600e 100644 --- a/iclient/iclientpkg/html_templates.go +++ b/iclient/iclientpkg/html_templates.go @@ -150,3 +150,62 @@ const configTemplate string = ` ` + +// To use: fmt.Sprintf(leasesTemplate, proxyfsVersion, inodeLeaseTableJSONString) +// %[1]v %[2]v +const leasesTemplate string = ` + + + + + + + Leases + + + +
    + +

    + Leases +

    +
    
    +    
    + + + + + + + +` diff --git a/iclient/iclientpkg/http-server.go b/iclient/iclientpkg/http-server.go index 725419e7..7793c5d4 100644 --- a/iclient/iclientpkg/http-server.go +++ b/iclient/iclientpkg/http-server.go @@ -11,6 +11,7 @@ import ( "log" "net" "net/http" + "sort" "strconv" "strings" "time" @@ -184,16 +185,127 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ } } +type inodeLeaseTableByInodeNumberElement struct { + InodeNumber uint64 + State string +} + +type inodeLeaseTableByInodeNumberSlice []inodeLeaseTableByInodeNumberElement + +func (s inodeLeaseTableByInodeNumberSlice) Len() int { + return len(s) +} + +func (s inodeLeaseTableByInodeNumberSlice) Swap(i, j int) { + s[i].InodeNumber, s[j].InodeNumber = s[j].InodeNumber, s[i].InodeNumber + s[i].State, s[j].State = s[j].State, s[i].State +} + +func (s inodeLeaseTableByInodeNumberSlice) Less(i, j int) bool { + return s[i].InodeNumber < s[j].InodeNumber +} + func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Request) { var ( - startTime time.Time = time.Now() + err error + inodeLease *inodeLeaseStruct + inodeLeaseTable inodeLeaseTableByInodeNumberSlice + inodeLeaseTableJSON []byte + inodeLeaseTableIndex int + inodeNumber uint64 + startTime time.Time = time.Now() ) defer func() { globals.stats.GetLeasesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - responseWriter.WriteHeader(http.StatusNotImplemented) // TODO + globals.Lock() + + globals.inodeLeaseTable[1] = &inodeLeaseStruct{state: inodeLeaseStateNone} // UNDO + globals.inodeLeaseTable[2] = &inodeLeaseStruct{state: inodeLeaseStateSharedRequested} // UNDO + globals.inodeLeaseTable[3] = &inodeLeaseStruct{state: inodeLeaseStateSharedGranted} // UNDO + globals.inodeLeaseTable[4] = &inodeLeaseStruct{state: inodeLeaseStateSharedPromoting} // UNDO + globals.inodeLeaseTable[5] = &inodeLeaseStruct{state: inodeLeaseStateSharedReleasing} // UNDO + globals.inodeLeaseTable[6] = &inodeLeaseStruct{state: inodeLeaseStateSharedExpired} // UNDO + globals.inodeLeaseTable[7] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveRequested} // UNDO + globals.inodeLeaseTable[8] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveGranted} // UNDO + globals.inodeLeaseTable[9] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveDemoting} // UNDO + globals.inodeLeaseTable[10] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveReleasing} // UNDO + globals.inodeLeaseTable[11] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveExpired} // UNDO + + inodeLeaseTable = make(inodeLeaseTableByInodeNumberSlice, len(globals.inodeLeaseTable)) + inodeLeaseTableIndex = 0 + for inodeNumber, inodeLease = range globals.inodeLeaseTable { + inodeLeaseTable[inodeLeaseTableIndex].InodeNumber = inodeNumber + switch inodeLease.state { + case inodeLeaseStateNone: + inodeLeaseTable[inodeLeaseTableIndex].State = "None" + case inodeLeaseStateSharedRequested: + inodeLeaseTable[inodeLeaseTableIndex].State = "SharedRequested:" + case inodeLeaseStateSharedGranted: + inodeLeaseTable[inodeLeaseTableIndex].State = "SharedGranted" + case inodeLeaseStateSharedPromoting: + inodeLeaseTable[inodeLeaseTableIndex].State = "SharedPromoting" + case inodeLeaseStateSharedReleasing: + inodeLeaseTable[inodeLeaseTableIndex].State = "SharedReleasing" + case inodeLeaseStateSharedExpired: + inodeLeaseTable[inodeLeaseTableIndex].State = "SharedExpired" + case inodeLeaseStateExclusiveRequested: + inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveRequested" + case inodeLeaseStateExclusiveGranted: + inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveGranted" + case inodeLeaseStateExclusiveDemoting: + inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveDemoting" + case inodeLeaseStateExclusiveReleasing: + inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveReleasing" + case inodeLeaseStateExclusiveExpired: + inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveExpired" + default: + logFatalf("globals.inodeLeaseTable[inudeNumber:0x%016X].state (%v) unrecognized", inodeNumber, inodeLease.state) + } + inodeLeaseTableIndex++ + } + + delete(globals.inodeLeaseTable, 1) // UNDO + delete(globals.inodeLeaseTable, 2) // UNDO + delete(globals.inodeLeaseTable, 3) // UNDO + delete(globals.inodeLeaseTable, 4) // UNDO + delete(globals.inodeLeaseTable, 5) // UNDO + delete(globals.inodeLeaseTable, 6) // UNDO + delete(globals.inodeLeaseTable, 7) // UNDO + delete(globals.inodeLeaseTable, 8) // UNDO + delete(globals.inodeLeaseTable, 9) // UNDO + delete(globals.inodeLeaseTable, 10) // UNDO + delete(globals.inodeLeaseTable, 11) // UNDO + + globals.Unlock() + + sort.Sort(inodeLeaseTable) + + inodeLeaseTableJSON, err = json.Marshal(inodeLeaseTable) + if nil != err { + logFatalf("json.Marshal(inodeLeaseTable) failed: %v", err) + } + + if strings.Contains(request.Header.Get("Accept"), "text/html") { + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(fmt.Sprintf(leasesTemplate, version.ProxyFSVersion, string(inodeLeaseTableJSON[:])))) + if nil != err { + logWarnf("responseWriter.Write([]byte(fmt.Sprintf(leasesTemplate, version.ProxyFSVersion, string(inodeLeaseTableJSON[:])))) failed: %v", err) + } + } else { + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(inodeLeaseTableJSON))) + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write(inodeLeaseTableJSON) + if nil != err { + logWarnf("responseWriter.Write(inodeLeaseTableJSON) failed: %v", err) + } + } } func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { From bf48003202017493edbfbbd625249c8f3539a177 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 19 Sep 2021 14:53:12 -0700 Subject: [PATCH 095/258] Added LeaseTable display to iclient (w/ Demote/Release buttons) --- iclient/iclientpkg/html_templates.go | 66 ++++++++++++++++++++++++++-- iclient/iclientpkg/http-server.go | 64 ++++++++++++++++----------- 2 files changed, 102 insertions(+), 28 deletions(-) diff --git a/iclient/iclientpkg/html_templates.go b/iclient/iclientpkg/html_templates.go index b72c600e..075e193d 100644 --- a/iclient/iclientpkg/html_templates.go +++ b/iclient/iclientpkg/html_templates.go @@ -160,6 +160,7 @@ const leasesTemplate string = ` + Leases @@ -195,16 +196,75 @@ const leasesTemplate string = `

    Leases

    + +
    
         
- + diff --git a/iclient/iclientpkg/http-server.go b/iclient/iclientpkg/http-server.go index 7793c5d4..c537fe23 100644 --- a/iclient/iclientpkg/http-server.go +++ b/iclient/iclientpkg/http-server.go @@ -205,6 +205,8 @@ func (s inodeLeaseTableByInodeNumberSlice) Less(i, j int) bool { return s[i].InodeNumber < s[j].InodeNumber } +var numUNDOs = uint64(1) + func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Request) { var ( err error @@ -222,17 +224,19 @@ func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Requ globals.Lock() - globals.inodeLeaseTable[1] = &inodeLeaseStruct{state: inodeLeaseStateNone} // UNDO - globals.inodeLeaseTable[2] = &inodeLeaseStruct{state: inodeLeaseStateSharedRequested} // UNDO - globals.inodeLeaseTable[3] = &inodeLeaseStruct{state: inodeLeaseStateSharedGranted} // UNDO - globals.inodeLeaseTable[4] = &inodeLeaseStruct{state: inodeLeaseStateSharedPromoting} // UNDO - globals.inodeLeaseTable[5] = &inodeLeaseStruct{state: inodeLeaseStateSharedReleasing} // UNDO - globals.inodeLeaseTable[6] = &inodeLeaseStruct{state: inodeLeaseStateSharedExpired} // UNDO - globals.inodeLeaseTable[7] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveRequested} // UNDO - globals.inodeLeaseTable[8] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveGranted} // UNDO - globals.inodeLeaseTable[9] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveDemoting} // UNDO - globals.inodeLeaseTable[10] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveReleasing} // UNDO - globals.inodeLeaseTable[11] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveExpired} // UNDO + for i := uint64(0); i < numUNDOs; i++ { + globals.inodeLeaseTable[(13*i)+1] = &inodeLeaseStruct{state: inodeLeaseStateNone} // UNDO + globals.inodeLeaseTable[(13*i)+2] = &inodeLeaseStruct{state: inodeLeaseStateSharedRequested} // UNDO + globals.inodeLeaseTable[(13*i)+3] = &inodeLeaseStruct{state: inodeLeaseStateSharedGranted} // UNDO + globals.inodeLeaseTable[(13*i)+4] = &inodeLeaseStruct{state: inodeLeaseStateSharedPromoting} // UNDO + globals.inodeLeaseTable[(13*i)+5] = &inodeLeaseStruct{state: inodeLeaseStateSharedReleasing} // UNDO + globals.inodeLeaseTable[(13*i)+6] = &inodeLeaseStruct{state: inodeLeaseStateSharedExpired} // UNDO + globals.inodeLeaseTable[(13*i)+7] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveRequested} // UNDO + globals.inodeLeaseTable[(13*i)+8] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveGranted} // UNDO + globals.inodeLeaseTable[(13*i)+9] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveDemoting} // UNDO + globals.inodeLeaseTable[(13*i)+10] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveReleasing} // UNDO + globals.inodeLeaseTable[(13*i)+11] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveExpired} // UNDO + } // UNDO inodeLeaseTable = make(inodeLeaseTableByInodeNumberSlice, len(globals.inodeLeaseTable)) inodeLeaseTableIndex = 0 @@ -242,7 +246,7 @@ func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Requ case inodeLeaseStateNone: inodeLeaseTable[inodeLeaseTableIndex].State = "None" case inodeLeaseStateSharedRequested: - inodeLeaseTable[inodeLeaseTableIndex].State = "SharedRequested:" + inodeLeaseTable[inodeLeaseTableIndex].State = "SharedRequested" case inodeLeaseStateSharedGranted: inodeLeaseTable[inodeLeaseTableIndex].State = "SharedGranted" case inodeLeaseStateSharedPromoting: @@ -267,17 +271,19 @@ func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Requ inodeLeaseTableIndex++ } - delete(globals.inodeLeaseTable, 1) // UNDO - delete(globals.inodeLeaseTable, 2) // UNDO - delete(globals.inodeLeaseTable, 3) // UNDO - delete(globals.inodeLeaseTable, 4) // UNDO - delete(globals.inodeLeaseTable, 5) // UNDO - delete(globals.inodeLeaseTable, 6) // UNDO - delete(globals.inodeLeaseTable, 7) // UNDO - delete(globals.inodeLeaseTable, 8) // UNDO - delete(globals.inodeLeaseTable, 9) // UNDO - delete(globals.inodeLeaseTable, 10) // UNDO - delete(globals.inodeLeaseTable, 11) // UNDO + for i := uint64(0); i < numUNDOs; i++ { + delete(globals.inodeLeaseTable, (13*i)+1) // UNDO + delete(globals.inodeLeaseTable, (13*i)+2) // UNDO + delete(globals.inodeLeaseTable, (13*i)+3) // UNDO + delete(globals.inodeLeaseTable, (13*i)+4) // UNDO + delete(globals.inodeLeaseTable, (13*i)+5) // UNDO + delete(globals.inodeLeaseTable, (13*i)+6) // UNDO + delete(globals.inodeLeaseTable, (13*i)+7) // UNDO + delete(globals.inodeLeaseTable, (13*i)+8) // UNDO + delete(globals.inodeLeaseTable, (13*i)+9) // UNDO + delete(globals.inodeLeaseTable, (13*i)+10) // UNDO + delete(globals.inodeLeaseTable, (13*i)+11) // UNDO + } // UNDO globals.Unlock() @@ -371,7 +377,10 @@ func serveHTTPPostOfLeasesDemote(responseWriter http.ResponseWriter, request *ht globals.stats.PostLeasesDemoteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - responseWriter.WriteHeader(http.StatusNotImplemented) // TODO + logWarnf("serveHTTPPostOfLeasesDemote() TODO") + numUNDOs++ + + responseWriter.WriteHeader(http.StatusOK) } func serveHTTPPostOfLeasesRelease(responseWriter http.ResponseWriter, request *http.Request) { @@ -383,5 +392,10 @@ func serveHTTPPostOfLeasesRelease(responseWriter http.ResponseWriter, request *h globals.stats.PostLeasesReleaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - responseWriter.WriteHeader(http.StatusNotImplemented) // TODO + logWarnf("serveHTTPPostOfLeasesRelease() TODO") + if numUNDOs > 0 { + numUNDOs-- + } // UNDO + + responseWriter.WriteHeader(http.StatusOK) } From 25cb2ed54aaff5bcf9368c6211a62ed5a7162f8e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 19 Sep 2021 20:21:44 -0700 Subject: [PATCH 096/258] Added iclient lease management / request mechanism ...local locks will be layered upon leases... --- iclient/iclientpkg/globals.go | 26 +++- iclient/iclientpkg/lease.go | 255 ++++++++++++++++++++++++++++++++++ 2 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 iclient/iclientpkg/lease.go diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index a5774eab..e3f3b3e1 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -4,6 +4,7 @@ package iclientpkg import ( + "container/list" "io/ioutil" "log" "net/http" @@ -43,7 +44,7 @@ type configStruct struct { type inodeLeaseStateType uint32 const ( - inodeLeaseStateNone inodeLeaseStateType = iota + inodeLeaseStateNone inodeLeaseStateType = iota // Only used if (*inodeLeaseStruct).requestList is non-empty inodeLeaseStateSharedRequested inodeLeaseStateSharedGranted inodeLeaseStateSharedPromoting @@ -56,8 +57,26 @@ const ( inodeLeaseStateExclusiveExpired ) +type inodeLeaseRequestTagType uint32 + +const ( + inodeLeaseRequestShared inodeLeaseRequestTagType = iota + inodeLeaseRequestPromote // upgraded to inodeLeaseRequestExclusive if intervening inodeLeaseRequestRelease results in inodeLeaseStateNone + inodeLeaseRequestExclusive // upgraded to inodeLeaseRequestPromote if intervening inodeLeaseRequestShared results in inodeLeaseStateSharedGranted + inodeLeaseRequestDemote // + inodeLeaseRequestRelease // +) + +type inodeLeaseRequestStruct struct { + sync.WaitGroup // signaled when the request has been processed + listElement *list.Element + tag inodeLeaseRequestTagType +} + type inodeLeaseStruct struct { - state inodeLeaseStateType + sync.WaitGroup // Signaled to tell (*inodeLeaseStruct).goroutine() to drain it's now non-empty inodeLeaseStruct.requestList + state inodeLeaseStateType + requestList *list.List // List of inodeLeaseRequestStruct's } type statsStruct struct { @@ -125,12 +144,13 @@ type statsStruct struct { } type globalsStruct struct { - sync.Mutex // serializes access to inodeLeaseTable + sync.Mutex // Serializes access to inodeLeaseTable config configStruct // logFile *os.File // == nil if config.LogFilePath == "" retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" fissionErrChan chan error // inodeLeaseTable map[uint64]*inodeLeaseStruct // + inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits httpServer *http.Server // httpServerWG sync.WaitGroup // stats *statsStruct // diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go new file mode 100644 index 00000000..969e2491 --- /dev/null +++ b/iclient/iclientpkg/lease.go @@ -0,0 +1,255 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +import ( + "container/list" +) + +func inodeLeaseRequestWhileLocked(inodeNumber uint64, inodeLeaseRequestTag inodeLeaseRequestTagType) { + var ( + inodeLease *inodeLeaseStruct + inodeLeaseRequest *inodeLeaseRequestStruct + ok bool + ) + + inodeLease, ok = globals.inodeLeaseTable[inodeNumber] + + if !ok { + inodeLease = &inodeLeaseStruct{ + state: inodeLeaseStateNone, + requestList: list.New(), + } + + globals.inodeLeaseWG.Add(1) + inodeLease.Add(1) + go inodeLease.goroutine() + } + + inodeLeaseRequest = &inodeLeaseRequestStruct{ + tag: inodeLeaseRequestTag, + } + + inodeLeaseRequest.Add(1) + + inodeLeaseRequest.listElement = inodeLease.requestList.PushBack(inodeLeaseRequest) + + if inodeLease.requestList.Len() == 1 { + inodeLease.Done() + } + + globals.Unlock() + + inodeLeaseRequest.Wait() + + globals.Lock() +} + +func (inodeLease *inodeLeaseStruct) goroutine() { + var ( + inodeLeaseRequest *inodeLeaseRequestStruct + inodeLeaseRequestListElement *list.Element + ok bool + ) + + defer globals.inodeLeaseWG.Done() + + for { + inodeLease.Wait() + + globals.Lock() + + inodeLeaseRequestListElement = inodeLease.requestList.Front() + if nil == inodeLeaseRequestListElement { + logFatalf("(*inodeLeaseStruct).goroutine() awakened with empty inodeLease.requestList") + } + + inodeLease.requestList.Remove(inodeLeaseRequestListElement) + + if inodeLease.requestList.Len() == 0 { + inodeLease.Add(1) // Make sure we stay awake + } + + globals.Unlock() + + inodeLeaseRequest, ok = inodeLeaseRequestListElement.Value.(*inodeLeaseRequestStruct) + if !ok { + logFatalf("inodeLeaseRequestListElement.Value.(*inodeLeaseRequestStruct) returned !ok") + } + + // TODO: For now, just grant the requested operation + + switch inodeLease.state { + case inodeLeaseStateNone: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateNone...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateSharedRequested: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateSharedRequested...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateSharedGranted: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateSharedGranted...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateSharedPromoting: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateSharedPromoting...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateSharedReleasing: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateSharedReleasing...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateSharedExpired: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateSharedExpired...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateExclusiveRequested: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateExclusiveRequested...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateExclusiveGranted: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateExclusiveGranted...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateExclusiveDemoting: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateExclusiveDemoting...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateExclusiveReleasing: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateExclusiveReleasing...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + case inodeLeaseStateExclusiveExpired: + switch inodeLeaseRequest.tag { + case inodeLeaseRequestShared: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestPromote: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestExclusive: + inodeLease.state = inodeLeaseStateExclusiveGranted + case inodeLeaseRequestDemote: + inodeLease.state = inodeLeaseStateSharedGranted + case inodeLeaseRequestRelease: + inodeLease.state = inodeLeaseStateNone + default: + logFatalf("inodeLease.state == inodeLeaseStateExclusiveExpired...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) + } + default: + logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) + } + + inodeLeaseRequest.Done() + } +} From a11d4ecc2b40d2b9a987df949bb0c40f30b76f13 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 21 Sep 2021 19:39:18 -0700 Subject: [PATCH 097/258] Fixed InodeLeaseTable in iclient --- iclient/iclientpkg/html_templates.go | 48 +++++++++++++--------------- ihtml/static-content/styles.css | 5 +++ 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/iclient/iclientpkg/html_templates.go b/iclient/iclientpkg/html_templates.go index 075e193d..a56f815b 100644 --- a/iclient/iclientpkg/html_templates.go +++ b/iclient/iclientpkg/html_templates.go @@ -196,6 +196,7 @@ const leasesTemplate string = `

Leases

+ -

+      
     
     
     
@@ -231,9 +230,9 @@ const leasesTemplate string = `
       const addMarkup = function(json_data) {
         let markup = "";
 
-        markup += "      Demote All";
-        markup += "      
"; markup += " Release All"; + markup += "
"; + markup += " Demote All"; markup += "
"; markup += "
"; From 8ad7de9f7cd52ec7b2ae492656e08ba29974f77b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 23 Sep 2021 10:50:48 -0700 Subject: [PATCH 099/258] Added tracing to iclientpkg/fission.go "upcalls" --- iclient/iclientpkg/fission.go | 236 ++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 8d17e060..8730ab4e 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -58,11 +58,17 @@ func (dummy *globalsStruct) DoLookup(inHeader *fission.InHeader, lookupIn *fissi startTime time.Time = time.Now() ) + logTracef("==> DoLookup(inHeader: %+v, lookupIn: %+v)", inHeader, lookupIn) + defer func() { + logTracef("<== DoLookup(lookupOut: %+v, errno: %v)", lookupOut, errno) + }() + defer func() { globals.stats.DoLookupUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + lookupOut = nil errno = syscall.ENOSYS return } @@ -72,6 +78,11 @@ func (dummy *globalsStruct) DoForget(inHeader *fission.InHeader, forgetIn *fissi startTime time.Time = time.Now() ) + logTracef("==> DoForget(inHeader: %+v, forgetIn: %+v)", inHeader, forgetIn) + defer func() { + logTracef("<== DoForget()") + }() + defer func() { globals.stats.DoForgetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -85,11 +96,17 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis startTime time.Time = time.Now() ) + logTracef("==> DoGetAttr(inHeader: %+v, getAttrIn: %+v)", inHeader, getAttrIn) + defer func() { + logTracef("<== DoGetAttr(getAttrOut: %+v, errno: %v)", getAttrOut, errno) + }() + defer func() { globals.stats.DoGetAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + getAttrOut = nil errno = syscall.ENOSYS return } @@ -99,11 +116,17 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis startTime time.Time = time.Now() ) + logTracef("==> DoSetAttr(inHeader: %+v, setAttrIn: %+v)", inHeader, setAttrIn) + defer func() { + logTracef("<== DoSetAttr(setAttrOut: %+v, errno: %v)", setAttrOut, errno) + }() + defer func() { globals.stats.DoSetAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + setAttrOut = nil errno = syscall.ENOSYS return } @@ -113,11 +136,17 @@ func (dummy *globalsStruct) DoReadLink(inHeader *fission.InHeader) (readLinkOut startTime time.Time = time.Now() ) + logTracef("==> DoReadLink(inHeader: %+v)", inHeader) + defer func() { + logTracef("<== DoReadLink(readLinkOut: %+v, errno: %v)", readLinkOut, errno) + }() + defer func() { globals.stats.DoReadLinkUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + readLinkOut = nil errno = syscall.ENOSYS return } @@ -127,11 +156,17 @@ func (dummy *globalsStruct) DoSymLink(inHeader *fission.InHeader, symLinkIn *fis startTime time.Time = time.Now() ) + logTracef("==> DoSymLink(inHeader: %+v, symLinkIn: %+v)", inHeader, symLinkIn) + defer func() { + logTracef("<== DoSymLink(symLinkOut: %+v, errno: %v)", symLinkOut, errno) + }() + defer func() { globals.stats.DoSymLinkUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + symLinkOut = nil errno = syscall.ENOSYS return } @@ -141,10 +176,16 @@ func (dummy *globalsStruct) DoMkNod(inHeader *fission.InHeader, mkNodIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoMkNod(inHeader: %+v, mkNodIn: %+v)", inHeader, mkNodIn) + defer func() { + logTracef("<== DoMkNod(mkNodOut: %+v, errno: %v)", mkNodOut, errno) + }() + defer func() { globals.stats.DoMkNodUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + mkNodOut = nil errno = syscall.ENOSYS return } @@ -154,11 +195,17 @@ func (dummy *globalsStruct) DoMkDir(inHeader *fission.InHeader, mkDirIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoMkDir(inHeader: %+v, mkDirIn: %+v)", inHeader, mkDirIn) + defer func() { + logTracef("<== DoMkDir(mkDirOut: %+v, errno: %v)", mkDirOut, errno) + }() + defer func() { globals.stats.DoMkDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + mkDirOut = nil errno = syscall.ENOSYS return } @@ -168,6 +215,11 @@ func (dummy *globalsStruct) DoUnlink(inHeader *fission.InHeader, unlinkIn *fissi startTime time.Time = time.Now() ) + logTracef("==> DoUnlink(inHeader: %+v, unlinkIn: %+v)", inHeader, unlinkIn) + defer func() { + logTracef("<== DoUnlink(errno: %v)", errno) + }() + defer func() { globals.stats.DoUnlinkUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -182,6 +234,11 @@ func (dummy *globalsStruct) DoRmDir(inHeader *fission.InHeader, rmDirIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoRmDir(inHeader: %+v, rmDirIn: %+v)", inHeader, rmDirIn) + defer func() { + logTracef("<== DoRmDir(errno: %v)", errno) + }() + defer func() { globals.stats.DoRmDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -196,6 +253,11 @@ func (dummy *globalsStruct) DoRename(inHeader *fission.InHeader, renameIn *fissi startTime time.Time = time.Now() ) + logTracef("==> DoRename(inHeader: %+v, renameIn: %+v)", inHeader, renameIn) + defer func() { + logTracef("<== DoRename(errno: %v)", errno) + }() + defer func() { globals.stats.DoRenameUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -210,11 +272,17 @@ func (dummy *globalsStruct) DoLink(inHeader *fission.InHeader, linkIn *fission.L startTime time.Time = time.Now() ) + logTracef("==> DoLink(inHeader: %+v, linkIn: %+v)", inHeader, linkIn) + defer func() { + logTracef("<== DoLink(linkOut: %+v, errno: %v)", linkOut, errno) + }() + defer func() { globals.stats.DoLinkUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + linkOut = nil errno = syscall.ENOSYS return } @@ -224,11 +292,17 @@ func (dummy *globalsStruct) DoOpen(inHeader *fission.InHeader, openIn *fission.O startTime time.Time = time.Now() ) + logTracef("==> DoOpen(inHeader: %+v, openIn: %+v)", inHeader, openIn) + defer func() { + logTracef("<== DoOpen(openOut: %+v, errno: %v)", openOut, errno) + }() + defer func() { globals.stats.DoOpenUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + openOut = nil errno = syscall.ENOSYS return } @@ -238,11 +312,21 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R startTime time.Time = time.Now() ) + logTracef("==> DoRead(inHeader: %+v, readIn: %+v)", inHeader, readIn) + defer func() { + if errno == 0 { + logTracef("<== DoRead(readOut: &{len(Data):%v}, errno: %v)", len(readOut.Data), errno) + } else { + logTracef("<== DoRead(readOut: %+v, errno: %v)", readOut, errno) + } + }() + defer func() { globals.stats.DoReadUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + readOut = nil errno = syscall.ENOSYS return } @@ -252,11 +336,17 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoWrite(inHeader: %+v, writeIn: &{FH:%v Offset:%v Size:%v: WriteFlags:%v LockOwner:%v Flags:%v Padding:%v len(Data):%v})", inHeader, writeIn.FH, writeIn.Offset, writeIn.Size, writeIn.WriteFlags, writeIn.LockOwner, writeIn.Flags, writeIn.Padding, len(writeIn.Data)) + defer func() { + logTracef("<== DoWrite(writeOut: %+v, errno: %v)", writeOut, errno) + }() + defer func() { globals.stats.DoWriteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + writeOut = nil errno = syscall.ENOSYS return } @@ -266,11 +356,17 @@ func (dummy *globalsStruct) DoStatFS(inHeader *fission.InHeader) (statFSOut *fis startTime time.Time = time.Now() ) + logTracef("==> DoStatFS(inHeader: %+v)", inHeader) + defer func() { + logTracef("<== DoStatFS(statFSOut: %+v, errno: %v)", statFSOut, errno) + }() + defer func() { globals.stats.DoStatFSUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + statFSOut = nil errno = syscall.ENOSYS return } @@ -280,6 +376,11 @@ func (dummy *globalsStruct) DoRelease(inHeader *fission.InHeader, releaseIn *fis startTime time.Time = time.Now() ) + logTracef("==> DoRelease(inHeader: %+v, releaseIn: %+v)", inHeader, releaseIn) + defer func() { + logTracef("<== DoRelease(errno: %v)", errno) + }() + defer func() { globals.stats.DoReleaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -294,6 +395,11 @@ func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoFSync(inHeader: %+v, fSyncIn: %+v)", inHeader, fSyncIn) + defer func() { + logTracef("<== DoFSync(errno: %v)", errno) + }() + defer func() { globals.stats.DoFSyncUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -308,6 +414,11 @@ func (dummy *globalsStruct) DoSetXAttr(inHeader *fission.InHeader, setXAttrIn *f startTime time.Time = time.Now() ) + logTracef("==> DoSetXAttr(inHeader: %+v, setXAttrIn: %+v)", inHeader, setXAttrIn) + defer func() { + logTracef("<== DoSetXAttr(errno: %v)", errno) + }() + defer func() { globals.stats.DoSetXAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -322,11 +433,17 @@ func (dummy *globalsStruct) DoGetXAttr(inHeader *fission.InHeader, getXAttrIn *f startTime time.Time = time.Now() ) + logTracef("==> DoGetXAttr(inHeader: %+v, getXAttrIn: %+v)", inHeader, getXAttrIn) + defer func() { + logTracef("<== DoGetXAttr(getXAttrOut: %+v, errno: %v)", getXAttrOut, errno) + }() + defer func() { globals.stats.DoGetXAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + getXAttrOut = nil errno = syscall.ENOSYS return } @@ -336,11 +453,17 @@ func (dummy *globalsStruct) DoListXAttr(inHeader *fission.InHeader, listXAttrIn startTime time.Time = time.Now() ) + logTracef("==> DoListXAttr(inHeader: %+v, listXAttrIn: %+v)", inHeader, listXAttrIn) + defer func() { + logTracef("<== DoListXAttr(listXAttrOut: %+v, errno: %v)", listXAttrOut, errno) + }() + defer func() { globals.stats.DoListXAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + listXAttrOut = nil errno = syscall.ENOSYS return } @@ -350,6 +473,11 @@ func (dummy *globalsStruct) DoRemoveXAttr(inHeader *fission.InHeader, removeXAtt startTime time.Time = time.Now() ) + logTracef("==> DoRemoveXAttr(inHeader: %+v, removeXAttrIn: %+v)", inHeader, removeXAttrIn) + defer func() { + logTracef("<== DoRemoveXAttr(errno: %v)", errno) + }() + defer func() { globals.stats.DoRemoveXAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -364,6 +492,11 @@ func (dummy *globalsStruct) DoFlush(inHeader *fission.InHeader, flushIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoFlush(inHeader: %+v, flushIn: %+v)", inHeader, flushIn) + defer func() { + logTracef("<== DoFlush(errno: %v)", errno) + }() + defer func() { globals.stats.DoFlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -378,6 +511,11 @@ func (dummy *globalsStruct) DoInit(inHeader *fission.InHeader, initIn *fission.I startTime time.Time = time.Now() ) + logTracef("==> DoInit(inHeader: %+v, initIn: %+v)", inHeader, initIn) + defer func() { + logTracef("<== DoInit(initOut: %+v, errno: %v)", initOut, errno) + }() + defer func() { globals.stats.DoInitUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -403,11 +541,17 @@ func (dummy *globalsStruct) DoOpenDir(inHeader *fission.InHeader, openDirIn *fis startTime time.Time = time.Now() ) + logTracef("==> DoOpenDir(inHeader: %+v, openDirIn: %+v)", inHeader, openDirIn) + defer func() { + logTracef("<== DoOpenDir(openDirOut: %+v, errno: %v)", openDirOut, errno) + }() + defer func() { globals.stats.DoOpenDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + openDirOut = nil errno = syscall.ENOSYS return } @@ -417,11 +561,17 @@ func (dummy *globalsStruct) DoReadDir(inHeader *fission.InHeader, readDirIn *fis startTime time.Time = time.Now() ) + logTracef("==> DoReadDir(inHeader: %+v, readDirIn: %+v)", inHeader, readDirIn) + defer func() { + logTracef("<== DoReadDir(readDirOut: %+v, errno: %v)", readDirOut, errno) + }() + defer func() { globals.stats.DoReadDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + readDirOut = nil errno = syscall.ENOSYS return } @@ -431,6 +581,11 @@ func (dummy *globalsStruct) DoReleaseDir(inHeader *fission.InHeader, releaseDirI startTime time.Time = time.Now() ) + logTracef("==> DoReleaseDir(inHeader: %+v, releaseDirIn: %+v)", inHeader, releaseDirIn) + defer func() { + logTracef("<== DoReleaseDir(errno: %v)", errno) + }() + defer func() { globals.stats.DoReleaseDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -445,6 +600,11 @@ func (dummy *globalsStruct) DoFSyncDir(inHeader *fission.InHeader, fSyncDirIn *f startTime time.Time = time.Now() ) + logTracef("==> DoFSyncDir(inHeader: %+v, fSyncDirIn: %+v)", inHeader, fSyncDirIn) + defer func() { + logTracef("<== DoFSyncDir(errno: %v)", errno) + }() + defer func() { globals.stats.DoFSyncDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -459,10 +619,16 @@ func (dummy *globalsStruct) DoGetLK(inHeader *fission.InHeader, getLKIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoGetLK(inHeader: %+v, getLKIn: %+v)", inHeader, getLKIn) + defer func() { + logTracef("<== DoGetLK(getLKOut: %+v, errno: %v)", getLKOut, errno) + }() + defer func() { globals.stats.DoGetLKUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + getLKOut = nil errno = syscall.ENOSYS return } @@ -472,6 +638,11 @@ func (dummy *globalsStruct) DoSetLK(inHeader *fission.InHeader, setLKIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoSetLK(inHeader: %+v, setLKIn: %+v)", inHeader, setLKIn) + defer func() { + logTracef("<== DoSetLK(errno: %v)", errno) + }() + defer func() { globals.stats.DoSetLKUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -485,6 +656,11 @@ func (dummy *globalsStruct) DoSetLKW(inHeader *fission.InHeader, setLKWIn *fissi startTime time.Time = time.Now() ) + logTracef("==> DoSetLKW(inHeader: %+v, setLKWIn: %+v)", inHeader, setLKWIn) + defer func() { + logTracef("<== DoSetLKW(errno: %v)", errno) + }() + defer func() { globals.stats.DoSetLKWUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -498,6 +674,11 @@ func (dummy *globalsStruct) DoAccess(inHeader *fission.InHeader, accessIn *fissi startTime time.Time = time.Now() ) + logTracef("==> DoAccess(inHeader: %+v, accessIn: %+v)", inHeader, accessIn) + defer func() { + logTracef("<== DoAccess(errno: %v)", errno) + }() + defer func() { globals.stats.DoAccessUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -512,11 +693,17 @@ func (dummy *globalsStruct) DoCreate(inHeader *fission.InHeader, createIn *fissi startTime time.Time = time.Now() ) + logTracef("==> DoCreate(inHeader: %+v, createIn: %+v)", inHeader, createIn) + defer func() { + logTracef("<== DoCreate(createOut: %+v, errno: %v)", createOut, errno) + }() + defer func() { globals.stats.DoCreateUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + createOut = nil errno = syscall.ENOSYS return } @@ -526,6 +713,11 @@ func (dummy *globalsStruct) DoInterrupt(inHeader *fission.InHeader, interruptIn startTime time.Time = time.Now() ) + logTracef("==> DoInterrupt(inHeader: %+v, interruptIn: %+v)", inHeader, interruptIn) + defer func() { + logTracef("<== DoInterrupt()") + }() + defer func() { globals.stats.DoInterruptUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -539,10 +731,16 @@ func (dummy *globalsStruct) DoBMap(inHeader *fission.InHeader, bMapIn *fission.B startTime time.Time = time.Now() ) + logTracef("==> DoBMap(inHeader: %+v, bMapIn: %+v)", inHeader, bMapIn) + defer func() { + logTracef("<== DoBMap(bMapOut: %+v, errno: %v)", bMapOut, errno) + }() + defer func() { globals.stats.DoBMapUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + bMapOut = nil errno = syscall.ENOSYS return } @@ -552,6 +750,11 @@ func (dummy *globalsStruct) DoDestroy(inHeader *fission.InHeader) (errno syscall startTime time.Time = time.Now() ) + logTracef("==> DoDestroy(inHeader: %+v)", inHeader) + defer func() { + logTracef("<== DoDestroy(errno: %v)", errno) + }() + defer func() { globals.stats.DoDestroyUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -566,11 +769,17 @@ func (dummy *globalsStruct) DoPoll(inHeader *fission.InHeader, pollIn *fission.P startTime time.Time = time.Now() ) + logTracef("==> DoPoll(inHeader: %+v, pollIn: %+v)", inHeader, pollIn) + defer func() { + logTracef("<== DoPoll(pollOut: %+v, errno: %v)", pollOut, errno) + }() + defer func() { globals.stats.DoPollUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + pollOut = nil errno = syscall.ENOSYS return } @@ -580,6 +789,11 @@ func (dummy *globalsStruct) DoBatchForget(inHeader *fission.InHeader, batchForge startTime time.Time = time.Now() ) + logTracef("==> DoBatchForget(inHeader: %+v, batchForgetIn: %+v)", inHeader, batchForgetIn) + defer func() { + logTracef("<== DoBatchForget()") + }() + defer func() { globals.stats.DoBatchForgetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -593,6 +807,11 @@ func (dummy *globalsStruct) DoFAllocate(inHeader *fission.InHeader, fAllocateIn startTime time.Time = time.Now() ) + logTracef("==> DoFAllocate(inHeader: %+v, fAllocateIn: %+v)", inHeader, fAllocateIn) + defer func() { + logTracef("<== DoFAllocate(errno: %v)", errno) + }() + defer func() { globals.stats.DoFAllocateUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -607,11 +826,17 @@ func (dummy *globalsStruct) DoReadDirPlus(inHeader *fission.InHeader, readDirPlu startTime time.Time = time.Now() ) + logTracef("==> DoReadDirPlus(inHeader: %+v, readDirPlusIn: %+v)", inHeader, readDirPlusIn) + defer func() { + logTracef("<== DoReadDirPlus(readDirPlusOut: %+v, errno: %v)", readDirPlusOut, errno) + }() + defer func() { globals.stats.DoReadDirPlusUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + readDirPlusOut = nil errno = syscall.ENOSYS return } @@ -621,6 +846,11 @@ func (dummy *globalsStruct) DoRename2(inHeader *fission.InHeader, rename2In *fis startTime time.Time = time.Now() ) + logTracef("==> DoRename2(inHeader: %+v, rename2In: %+v)", inHeader, rename2In) + defer func() { + logTracef("<== DoRename2(errno: %v)", errno) + }() + defer func() { globals.stats.DoRename2Usecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -635,11 +865,17 @@ func (dummy *globalsStruct) DoLSeek(inHeader *fission.InHeader, lSeekIn *fission startTime time.Time = time.Now() ) + logTracef("==> DoLSeek(inHeader: %+v, lSeekIn: %+v)", inHeader, lSeekIn) + defer func() { + logTracef("<== DoLSeek(lSeekOut: %+v, errno: %v)", lSeekOut, errno) + }() + defer func() { globals.stats.DoLSeekUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() // TODO + lSeekOut = nil errno = syscall.ENOSYS return } From 22ab4916e404b6e27419950ea1d575ce1382aef5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 23 Sep 2021 11:02:26 -0700 Subject: [PATCH 100/258] Added configurable output of package fission logging --- iclient/dev.conf | 1 + iclient/iclient.conf | 1 + iclient/iclientpkg/api.go | 1 + iclient/iclientpkg/globals.go | 6 ++++++ iclient/iclientpkg/log.go | 4 +++- 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/iclient/dev.conf b/iclient/dev.conf index 1552d6a3..f7e58afc 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -19,5 +19,6 @@ RetryRPCCACertFilePath: # Defaults to /dev/null LogFilePath: # Defaults to "" LogToConsole: true TraceEnabled: false +FUSELogEnabled: false HTTPServerIPAddr: dev HTTPServerPort: 15347 diff --git a/iclient/iclient.conf b/iclient/iclient.conf index fb19984d..ee154d37 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -19,5 +19,6 @@ RetryRPCCACertFilePath: caCert.pem LogFilePath: # Defaults to "" LogToConsole: true TraceEnabled: false +FUSELogEnabled: false HTTPServerIPAddr: iclient HTTPServerPort: 15347 diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index cb2edba0..b56357ba 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -25,6 +25,7 @@ // LogFilePath: iclient.log // LogToConsole: true // TraceEnabled: false +// FUSELogEnabled: false // HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) // HTTPServerPort: # Defaults to disabling the embedded HTTP Server // diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index e3f3b3e1..8478fd0b 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -37,6 +37,7 @@ type configStruct struct { LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled LogToConsole bool TraceEnabled bool + FUSELogEnabled bool HTTPServerIPAddr string HTTPServerPort uint16 // To be served on HTTPServerIPAddr via TCP } @@ -264,6 +265,10 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err if nil != err { logFatal(err) } + globals.config.FUSELogEnabled, err = confMap.FetchOptionValueBool("ICLIENT", "FUSELogEnabled") + if nil != err { + logFatal(err) + } globals.config.HTTPServerIPAddr, err = confMap.FetchOptionValueString("ICLIENT", "HTTPServerIPAddr") if nil != err { globals.config.HTTPServerIPAddr = "0.0.0.0" @@ -315,6 +320,7 @@ func uninitializeGlobals() (err error) { globals.config.LogFilePath = "" globals.config.LogToConsole = false globals.config.TraceEnabled = false + globals.config.FUSELogEnabled = false globals.config.HTTPServerIPAddr = "" globals.config.HTTPServerPort = 0 diff --git a/iclient/iclientpkg/log.go b/iclient/iclientpkg/log.go index a128c018..36dd17eb 100644 --- a/iclient/iclientpkg/log.go +++ b/iclient/iclientpkg/log.go @@ -98,6 +98,8 @@ func newLogger() *log.Logger { } func (dummy *globalsStruct) Write(p []byte) (n int, err error) { - logf("FISSION", "%s", string(p[:])) + if globals.config.FUSELogEnabled { + logf("FISSION", "%s", string(p[:])) + } return 0, nil } From 3a86bfb6b0a20f6bb70c0d801f50aa19ce2884bd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 26 Sep 2021 16:42:54 -0700 Subject: [PATCH 101/258] Reworked lease/lock globals in iclientpkg ...this is only the struct linkages... coding it up remains... --- iclient/iclientpkg/globals.go | 36 +-- iclient/iclientpkg/impl.go | 20 ++ iclient/iclientpkg/lease.go | 505 +++++++++++++++++--------------- iclient/iclientpkg/retry-rpc.go | 12 + 4 files changed, 315 insertions(+), 258 deletions(-) create mode 100644 iclient/iclientpkg/retry-rpc.go diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 8478fd0b..9d4f99e7 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -58,26 +58,27 @@ const ( inodeLeaseStateExclusiveExpired ) -type inodeLeaseRequestTagType uint32 - -const ( - inodeLeaseRequestShared inodeLeaseRequestTagType = iota - inodeLeaseRequestPromote // upgraded to inodeLeaseRequestExclusive if intervening inodeLeaseRequestRelease results in inodeLeaseStateNone - inodeLeaseRequestExclusive // upgraded to inodeLeaseRequestPromote if intervening inodeLeaseRequestShared results in inodeLeaseStateSharedGranted - inodeLeaseRequestDemote // - inodeLeaseRequestRelease // -) +type inodeLeaseStruct struct { + sync.WaitGroup // TODO (fix this): Signaled to tell (*inodeLeaseStruct).goroutine() to drain it's now non-empty inodeLeaseStruct.requestList + state inodeLeaseStateType // + heldList *list.List // List of granted inodeHeldLockStruct's + requestList *list.List // List of pending inodeLockRequestStruct's +} -type inodeLeaseRequestStruct struct { - sync.WaitGroup // signaled when the request has been processed - listElement *list.Element - tag inodeLeaseRequestTagType +type inodeHeldLockStruct struct { + parent *inodeLockRequestStruct + exclusive bool + listElement *list.Element } -type inodeLeaseStruct struct { - sync.WaitGroup // Signaled to tell (*inodeLeaseStruct).goroutine() to drain it's now non-empty inodeLeaseStruct.requestList - state inodeLeaseStateType - requestList *list.List // List of inodeLeaseRequestStruct's +type inodeLockRequestStruct struct { + sync.WaitGroup // Signaled when the state of the lock request has been served + inodeNumber uint64 // The inodeNumber for which the latest lock request is being made + exclusive bool // Indicates if the latest lock request is exclusive + listElement *list.Element // Maintains position in .inodeNumber's indicated inodeLeaseStruct.requestList + locksHeld map[uint64]*inodeHeldLockStruct // At entry, contains the list of inodeLock's already held (shared or exclusively) + // At exit, either contains the earlier list appended with the granted inodeLock + // or is empty indicating the caller should restart lock request sequence } type statsStruct struct { @@ -152,6 +153,7 @@ type globalsStruct struct { fissionErrChan chan error // inodeLeaseTable map[uint64]*inodeLeaseStruct // inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits + inodeLockWG sync.WaitGroup // Signaled as each active inodeLockRequestStruct has signaled service complete with .locksHeld empty httpServer *http.Server // httpServerWG sync.WaitGroup // stats *statsStruct // diff --git a/iclient/iclientpkg/impl.go b/iclient/iclientpkg/impl.go index 70e898a7..c718e486 100644 --- a/iclient/iclientpkg/impl.go +++ b/iclient/iclientpkg/impl.go @@ -13,6 +13,16 @@ func start(confMap conf.ConfMap, fissionErrChan chan error) (err error) { return } + err = openRetryRPC() + if nil != err { + return + } + + err = startLeaseHandler() + if nil != err { + return + } + // TODO err = performMountFUSE() @@ -45,6 +55,16 @@ func stop() (err error) { // TODO + err = stopLeaseHandler() + if nil != err { + return + } + + err = closeRetryRPC() + if nil != err { + return + } + err = uninitializeGlobals() return diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 969e2491..7d3a89ea 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -3,253 +3,276 @@ package iclientpkg -import ( - "container/list" -) - -func inodeLeaseRequestWhileLocked(inodeNumber uint64, inodeLeaseRequestTag inodeLeaseRequestTagType) { - var ( - inodeLease *inodeLeaseStruct - inodeLeaseRequest *inodeLeaseRequestStruct - ok bool - ) - - inodeLease, ok = globals.inodeLeaseTable[inodeNumber] - - if !ok { - inodeLease = &inodeLeaseStruct{ - state: inodeLeaseStateNone, - requestList: list.New(), - } - - globals.inodeLeaseWG.Add(1) - inodeLease.Add(1) - go inodeLease.goroutine() - } +func startLeaseHandler() (err error) { + return nil // TODO +} + +func stopLeaseHandler() (err error) { + return nil // TODO +} - inodeLeaseRequest = &inodeLeaseRequestStruct{ - tag: inodeLeaseRequestTag, +func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { + inodeLockRequest = &inodeLockRequestStruct{ + inodeNumber: 0, + exclusive: false, + listElement: nil, + locksHeld: make(map[uint64]*inodeHeldLockStruct), } - inodeLeaseRequest.Add(1) + return +} - inodeLeaseRequest.listElement = inodeLease.requestList.PushBack(inodeLeaseRequest) +func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { + // TODO +} - if inodeLease.requestList.Len() == 1 { - inodeLease.Done() - } +func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { + // TODO +} - globals.Unlock() +// func inodeLeaseRequestWhileLocked(inodeNumber uint64, inodeLeaseRequestTag inodeLeaseRequestTagType) { +// var ( +// inodeLease *inodeLeaseStruct +// inodeLeaseRequest *inodeLeaseRequestStruct +// ok bool +// ) - inodeLeaseRequest.Wait() +// inodeLease, ok = globals.inodeLeaseTable[inodeNumber] - globals.Lock() -} +// if !ok { +// inodeLease = &inodeLeaseStruct{ +// state: inodeLeaseStateNone, +// requestList: list.New(), +// } -func (inodeLease *inodeLeaseStruct) goroutine() { - var ( - inodeLeaseRequest *inodeLeaseRequestStruct - inodeLeaseRequestListElement *list.Element - ok bool - ) - - defer globals.inodeLeaseWG.Done() - - for { - inodeLease.Wait() - - globals.Lock() - - inodeLeaseRequestListElement = inodeLease.requestList.Front() - if nil == inodeLeaseRequestListElement { - logFatalf("(*inodeLeaseStruct).goroutine() awakened with empty inodeLease.requestList") - } - - inodeLease.requestList.Remove(inodeLeaseRequestListElement) - - if inodeLease.requestList.Len() == 0 { - inodeLease.Add(1) // Make sure we stay awake - } - - globals.Unlock() - - inodeLeaseRequest, ok = inodeLeaseRequestListElement.Value.(*inodeLeaseRequestStruct) - if !ok { - logFatalf("inodeLeaseRequestListElement.Value.(*inodeLeaseRequestStruct) returned !ok") - } - - // TODO: For now, just grant the requested operation - - switch inodeLease.state { - case inodeLeaseStateNone: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateNone...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateSharedRequested: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateSharedRequested...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateSharedGranted: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateSharedGranted...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateSharedPromoting: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateSharedPromoting...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateSharedReleasing: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateSharedReleasing...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateSharedExpired: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateSharedExpired...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateExclusiveRequested: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateExclusiveRequested...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateExclusiveGranted: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateExclusiveGranted...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateExclusiveDemoting: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateExclusiveDemoting...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateExclusiveReleasing: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateExclusiveReleasing...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - case inodeLeaseStateExclusiveExpired: - switch inodeLeaseRequest.tag { - case inodeLeaseRequestShared: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestPromote: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestExclusive: - inodeLease.state = inodeLeaseStateExclusiveGranted - case inodeLeaseRequestDemote: - inodeLease.state = inodeLeaseStateSharedGranted - case inodeLeaseRequestRelease: - inodeLease.state = inodeLeaseStateNone - default: - logFatalf("inodeLease.state == inodeLeaseStateExclusiveExpired...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) - } - default: - logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) - } - - inodeLeaseRequest.Done() - } -} +// globals.inodeLeaseWG.Add(1) +// inodeLease.Add(1) +// go inodeLease.goroutine() +// } + +// inodeLeaseRequest = &inodeLeaseRequestStruct{ +// tag: inodeLeaseRequestTag, +// } + +// inodeLeaseRequest.Add(1) + +// inodeLeaseRequest.listElement = inodeLease.requestList.PushBack(inodeLeaseRequest) + +// if inodeLease.requestList.Len() == 1 { +// inodeLease.Done() +// } + +// globals.Unlock() + +// inodeLeaseRequest.Wait() + +// globals.Lock() +// } + +// func (inodeLease *inodeLeaseStruct) goroutine() { +// var ( +// inodeLeaseRequest *inodeLeaseRequestStruct +// inodeLeaseRequestListElement *list.Element +// ok bool +// ) + +// defer globals.inodeLeaseWG.Done() + +// for { +// inodeLease.Wait() + +// globals.Lock() + +// inodeLeaseRequestListElement = inodeLease.requestList.Front() +// if nil == inodeLeaseRequestListElement { +// logFatalf("(*inodeLeaseStruct).goroutine() awakened with empty inodeLease.requestList") +// } + +// inodeLease.requestList.Remove(inodeLeaseRequestListElement) + +// if inodeLease.requestList.Len() == 0 { +// inodeLease.Add(1) // Make sure we stay awake +// } + +// globals.Unlock() + +// inodeLeaseRequest, ok = inodeLeaseRequestListElement.Value.(*inodeLeaseRequestStruct) +// if !ok { +// logFatalf("inodeLeaseRequestListElement.Value.(*inodeLeaseRequestStruct) returned !ok") +// } + +// // TODO: For now, just grant the requested operation + +// switch inodeLease.state { +// case inodeLeaseStateNone: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateNone...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateSharedRequested: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateSharedRequested...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateSharedGranted: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateSharedGranted...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateSharedPromoting: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateSharedPromoting...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateSharedReleasing: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateSharedReleasing...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateSharedExpired: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateSharedExpired...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateExclusiveRequested: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateExclusiveRequested...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateExclusiveGranted: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateExclusiveGranted...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateExclusiveDemoting: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateExclusiveDemoting...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateExclusiveReleasing: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateExclusiveReleasing...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// case inodeLeaseStateExclusiveExpired: +// switch inodeLeaseRequest.tag { +// case inodeLeaseRequestShared: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestPromote: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestExclusive: +// inodeLease.state = inodeLeaseStateExclusiveGranted +// case inodeLeaseRequestDemote: +// inodeLease.state = inodeLeaseStateSharedGranted +// case inodeLeaseRequestRelease: +// inodeLease.state = inodeLeaseStateNone +// default: +// logFatalf("inodeLease.state == inodeLeaseStateExclusiveExpired...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) +// } +// default: +// logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) +// } + +// inodeLeaseRequest.Done() +// } +// } diff --git a/iclient/iclientpkg/retry-rpc.go b/iclient/iclientpkg/retry-rpc.go new file mode 100644 index 00000000..1164a9a4 --- /dev/null +++ b/iclient/iclientpkg/retry-rpc.go @@ -0,0 +1,12 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +func openRetryRPC() (err error) { + return nil // TODO +} + +func closeRetryRPC() (err error) { + return nil // TODO +} From 5d853f1ae468c993930fb48c97004c2be01f6604 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 27 Sep 2021 15:27:55 -0700 Subject: [PATCH 102/258] Coding framework for (*inodeLockRequestStruct)addThisLock() of iclientpkg instantiated ...TODOs mark the three places where lease requests must be issued... and their responses processed. --- iclient/iclientpkg/globals.go | 41 +-- iclient/iclientpkg/lease.go | 498 ++++++++++++++++---------------- iclient/iclientpkg/retry-rpc.go | 25 +- 3 files changed, 295 insertions(+), 269 deletions(-) diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 9d4f99e7..44721c63 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -16,6 +16,7 @@ import ( "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/retryrpc" "github.com/NVIDIA/proxyfs/utils" ) @@ -59,16 +60,16 @@ const ( ) type inodeLeaseStruct struct { - sync.WaitGroup // TODO (fix this): Signaled to tell (*inodeLeaseStruct).goroutine() to drain it's now non-empty inodeLeaseStruct.requestList - state inodeLeaseStateType // - heldList *list.List // List of granted inodeHeldLockStruct's - requestList *list.List // List of pending inodeLockRequestStruct's + state inodeLeaseStateType // + heldList *list.List // List of granted inodeHeldLockStruct's + requestList *list.List // List of pending inodeLockRequestStruct's } type inodeHeldLockStruct struct { - parent *inodeLockRequestStruct - exclusive bool - listElement *list.Element + inodeLease *inodeLeaseStruct + inodeLockRequest *inodeLockRequestStruct + exclusive bool + listElement *list.Element //Maintains position in .inodeNumber's indicated inodeLeaseStruct.heldList } type inodeLockRequestStruct struct { @@ -146,18 +147,20 @@ type statsStruct struct { } type globalsStruct struct { - sync.Mutex // Serializes access to inodeLeaseTable - config configStruct // - logFile *os.File // == nil if config.LogFilePath == "" - retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" - fissionErrChan chan error // - inodeLeaseTable map[uint64]*inodeLeaseStruct // - inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits - inodeLockWG sync.WaitGroup // Signaled as each active inodeLockRequestStruct has signaled service complete with .locksHeld empty - httpServer *http.Server // - httpServerWG sync.WaitGroup // - stats *statsStruct // - fissionVolume fission.Volume // + sync.Mutex // Serializes access to inodeLeaseTable + config configStruct // + logFile *os.File // == nil if config.LogFilePath == "" + retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" + retryRPCClientConfig *retryrpc.ClientConfig // + retryRPCClient *retryrpc.Client // + fissionErrChan chan error // + inodeLeaseTable map[uint64]*inodeLeaseStruct // + inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits + inodeLockWG sync.WaitGroup // Signaled as each active inodeLockRequestStruct has signaled service complete with .locksHeld empty + httpServer *http.Server // + httpServerWG sync.WaitGroup // + stats *statsStruct // + fissionVolume fission.Volume // } var globals globalsStruct diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 7d3a89ea..e2aaca5d 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -3,6 +3,10 @@ package iclientpkg +import ( + "container/list" +) + func startLeaseHandler() (err error) { return nil // TODO } @@ -11,6 +15,8 @@ func stopLeaseHandler() (err error) { return nil // TODO } +// newLockRequest is called to create and initialize an inodeLockRequestStruct. +// func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { inodeLockRequest = &inodeLockRequestStruct{ inodeNumber: 0, @@ -22,257 +28,253 @@ func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { return } +// addThisLock is called for an existing inodeLockRequestStruct with the inodeNumber and +// exclusive fields set to specify the inode to be locked and whether or not the lock +// should be exlusive. +// +// Upon successful granting of the requested lock, the locksHeld map will contain a link +// to the inodeHeldLockStruct tracking the granted lock. If unsuccessful, the locksHeld map +// will be empty. +// +// To collect a set of locks, the caller may repeatably call addThisLock filling in a fresh +// inodeNumber and exclusive field tuple for each inode lock needed. If any lock attempt fails, +// the locksHeld map will be empty. Note that this indicates not only the failure to obtain the +// requested lock but also the implicit releasing of any locks previously in the locksHeld map +// at the time of the call. +// func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { - // TODO + var ( + inodeHeldLock *inodeHeldLockStruct + inodeLease *inodeLeaseStruct + ok bool + ) + + globals.Lock() + + if inodeLockRequest.inodeNumber == 0 { + logFatalf("(*inodeLockRequestStruct)addThisLock() called with .inodeNumber == 0") + } + _, ok = inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] + if ok { + logFatalf("*inodeLockRequestStruct)addThisLock() called with .inodeNumber already present in .locksHeld map") + } + + inodeLease, ok = globals.inodeLeaseTable[inodeLockRequest.inodeNumber] + if !ok { + inodeLease = &inodeLeaseStruct{ + state: inodeLeaseStateNone, + heldList: list.New(), + requestList: list.New(), + } + + globals.inodeLeaseTable[inodeLockRequest.inodeNumber] = inodeLease + } + + if inodeLease.requestList.Len() != 0 { + // At lease one other inodeLockRequestStruct is blocked, so this one must block + + inodeLockRequest.Add(1) + + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + + globals.Unlock() + + inodeLockRequest.Wait() + + return + } + + if inodeLease.heldList.Len() != 0 { + if inodeLockRequest.exclusive { + // Lock is held, so this exclusive inodeLockRequestStruct must block + + inodeLockRequest.Add(1) + + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + + globals.Unlock() + + inodeLockRequest.Wait() + + return + } + + inodeHeldLock, ok = inodeLease.heldList.Front().Value.(*inodeHeldLockStruct) + if !ok { + logFatalf("inodeLease.heldList.Front().Value.(*inodeHeldLockStruct) returned !ok") + } + + if inodeHeldLock.exclusive { + // Lock is held exclusively, so this inodeLockRequestStruct must block + + inodeLockRequest.Add(1) + + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + + globals.Unlock() + + inodeLockRequest.Wait() + + return + } + } + + if inodeLockRequest.exclusive { + switch inodeLease.state { + case inodeLeaseStateNone: + inodeLease.state = inodeLeaseStateExclusiveRequested + + // TODO - this is where we issue imgrpkg.LeaseRequestTypeExclusive + + case inodeLeaseStateSharedRequested: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateSharedGranted: + inodeLease.state = inodeLeaseStateSharedPromoting + + // TODO - this is where we issue imgrpkg.LeaseRequestTypePromote + + case inodeLeaseStateSharedPromoting: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateSharedReleasing: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateSharedExpired: + // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up + inodeLockRequest.unlockAllWhileLocked() + globals.Unlock() + case inodeLeaseStateExclusiveRequested: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateExclusiveGranted: + // We can immediately grant the exclusive inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ + inodeLease: inodeLease, + inodeLockRequest: inodeLockRequest, + exclusive: true, + } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + globals.Unlock() + case inodeLeaseStateExclusiveDemoting: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateExclusiveReleasing: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateExclusiveExpired: + // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up + inodeLockRequest.unlockAllWhileLocked() + globals.Unlock() + default: + logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) + } + } else { + switch inodeLease.state { + case inodeLeaseStateNone: + inodeLease.state = inodeLeaseStateSharedRequested + + // TODO - this is where we issue imgrpkg.LeaseRequestTypeShared + + case inodeLeaseStateSharedRequested: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateSharedGranted: + // We can immediately grant the shared inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ + inodeLease: inodeLease, + inodeLockRequest: inodeLockRequest, + exclusive: false, + } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + globals.Unlock() + case inodeLeaseStateSharedPromoting: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateSharedReleasing: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateSharedExpired: + // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up + inodeLockRequest.unlockAllWhileLocked() + globals.Unlock() + case inodeLeaseStateExclusiveRequested: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateExclusiveGranted: + // We can immediately grant the shared inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ + inodeLease: inodeLease, + inodeLockRequest: inodeLockRequest, + exclusive: false, + } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + globals.Unlock() + case inodeLeaseStateExclusiveDemoting: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateExclusiveReleasing: + // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct + inodeLockRequest.Add(1) + inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + globals.Unlock() + inodeLockRequest.Wait() + case inodeLeaseStateExclusiveExpired: + // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up + inodeLockRequest.unlockAllWhileLocked() + globals.Unlock() + default: + logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) + } + } } +// unlockAll is called to explicitly release all locks listed in the locksHeld map. +// func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { - // TODO + globals.Lock() + inodeLockRequest.unlockAllWhileLocked() + globals.Unlock() } -// func inodeLeaseRequestWhileLocked(inodeNumber uint64, inodeLeaseRequestTag inodeLeaseRequestTagType) { -// var ( -// inodeLease *inodeLeaseStruct -// inodeLeaseRequest *inodeLeaseRequestStruct -// ok bool -// ) - -// inodeLease, ok = globals.inodeLeaseTable[inodeNumber] - -// if !ok { -// inodeLease = &inodeLeaseStruct{ -// state: inodeLeaseStateNone, -// requestList: list.New(), -// } - -// globals.inodeLeaseWG.Add(1) -// inodeLease.Add(1) -// go inodeLease.goroutine() -// } - -// inodeLeaseRequest = &inodeLeaseRequestStruct{ -// tag: inodeLeaseRequestTag, -// } - -// inodeLeaseRequest.Add(1) - -// inodeLeaseRequest.listElement = inodeLease.requestList.PushBack(inodeLeaseRequest) - -// if inodeLease.requestList.Len() == 1 { -// inodeLease.Done() -// } - -// globals.Unlock() - -// inodeLeaseRequest.Wait() - -// globals.Lock() -// } - -// func (inodeLease *inodeLeaseStruct) goroutine() { -// var ( -// inodeLeaseRequest *inodeLeaseRequestStruct -// inodeLeaseRequestListElement *list.Element -// ok bool -// ) - -// defer globals.inodeLeaseWG.Done() - -// for { -// inodeLease.Wait() - -// globals.Lock() - -// inodeLeaseRequestListElement = inodeLease.requestList.Front() -// if nil == inodeLeaseRequestListElement { -// logFatalf("(*inodeLeaseStruct).goroutine() awakened with empty inodeLease.requestList") -// } - -// inodeLease.requestList.Remove(inodeLeaseRequestListElement) - -// if inodeLease.requestList.Len() == 0 { -// inodeLease.Add(1) // Make sure we stay awake -// } - -// globals.Unlock() - -// inodeLeaseRequest, ok = inodeLeaseRequestListElement.Value.(*inodeLeaseRequestStruct) -// if !ok { -// logFatalf("inodeLeaseRequestListElement.Value.(*inodeLeaseRequestStruct) returned !ok") -// } - -// // TODO: For now, just grant the requested operation - -// switch inodeLease.state { -// case inodeLeaseStateNone: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateNone...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateSharedRequested: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateSharedRequested...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateSharedGranted: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateSharedGranted...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateSharedPromoting: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateSharedPromoting...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateSharedReleasing: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateSharedReleasing...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateSharedExpired: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateSharedExpired...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateExclusiveRequested: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateExclusiveRequested...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateExclusiveGranted: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateExclusiveGranted...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateExclusiveDemoting: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateExclusiveDemoting...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateExclusiveReleasing: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateExclusiveReleasing...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// case inodeLeaseStateExclusiveExpired: -// switch inodeLeaseRequest.tag { -// case inodeLeaseRequestShared: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestPromote: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestExclusive: -// inodeLease.state = inodeLeaseStateExclusiveGranted -// case inodeLeaseRequestDemote: -// inodeLease.state = inodeLeaseStateSharedGranted -// case inodeLeaseRequestRelease: -// inodeLease.state = inodeLeaseStateNone -// default: -// logFatalf("inodeLease.state == inodeLeaseStateExclusiveExpired...switch inodeLeaseRequest.tag unexpected: %v", inodeLeaseRequest.tag) -// } -// default: -// logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) -// } - -// inodeLeaseRequest.Done() -// } -// } +// unlockAllWhileLocked is what unlockAll calls after obtaining globals.Lock(). +// +func (inodeLockRequest *inodeLockRequestStruct) unlockAllWhileLocked() { + // TODO +} diff --git a/iclient/iclientpkg/retry-rpc.go b/iclient/iclientpkg/retry-rpc.go index 1164a9a4..ec85cdbb 100644 --- a/iclient/iclientpkg/retry-rpc.go +++ b/iclient/iclientpkg/retry-rpc.go @@ -3,10 +3,31 @@ package iclientpkg +import ( + "github.com/NVIDIA/proxyfs/retryrpc" +) + func openRetryRPC() (err error) { - return nil // TODO + globals.retryRPCClientConfig = &retryrpc.ClientConfig{ + DNSOrIPAddr: globals.config.RetryRPCPublicIPAddr, + Port: int(globals.config.RetryRPCPort), + RootCAx509CertificatePEM: globals.retryRPCCACertPEM, + Callbacks: globals, + DeadlineIO: globals.config.RetryRPCDeadlineIO, + KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, + } + + globals.retryRPCClient, err = retryrpc.NewClient(globals.retryRPCClientConfig) + + return // err as set by call to retryrpc.NewClient() is sufficient } func closeRetryRPC() (err error) { - return nil // TODO + globals.retryRPCClient.Close() + + globals.retryRPCClientConfig = nil + globals.retryRPCClient = nil + + err = nil + return } From 52126a7bea34678917b0f83cedc866a1ad7a3102 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 28 Sep 2021 17:50:15 -0700 Subject: [PATCH 103/258] Added Swift Auth and http.Client logic to iclient/iclientpkg/rpc.go et. al. iclient is now auth'ing with Swift --- iclient/dev.conf | 11 ++- iclient/iclient.conf | 11 ++- iclient/iclientpkg/api.go | 18 ++-- iclient/iclientpkg/globals.go | 97 +++++++++++++------ iclient/iclientpkg/impl.go | 4 +- iclient/iclientpkg/retry-rpc.go | 33 ------- iclient/iclientpkg/rpc.go | 162 ++++++++++++++++++++++++++++++++ 7 files changed, 261 insertions(+), 75 deletions(-) delete mode 100644 iclient/iclientpkg/retry-rpc.go create mode 100644 iclient/iclientpkg/rpc.go diff --git a/iclient/dev.conf b/iclient/dev.conf index f7e58afc..563a383b 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -8,9 +8,14 @@ FUSEAllowOther: true FUSEMaxBackground: 1000 FUSECongestionThreshhold: 0 FUSEMaxWrite: 131076 -PlugInPath: iauth/iauth-swift/iauth-swift.so -PlugInEnvName: SwiftAuthBlob -PlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +AuthPlugInPath: iauth/iauth-swift/iauth-swift.so +AuthPlugInEnvName: +AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +SwiftRetryDelay: 100ms +SwiftRetryExpBackoff: 2 +SwiftRetryLimit: 4 +SwiftTimeout: 10m +SwiftConnectionPoolSize: 128 RetryRPCPublicIPAddr: dev RetryRPCPort: 32356 RetryRPCDeadlineIO: 60s diff --git a/iclient/iclient.conf b/iclient/iclient.conf index ee154d37..08fad368 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -8,9 +8,14 @@ FUSEAllowOther: true FUSEMaxBackground: 1000 FUSECongestionThreshhold: 0 FUSEMaxWrite: 131076 -PlugInPath: iauth-swift.so -PlugInEnvName: SwiftAuthBlob -PlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +AuthPlugInPath: iauth-swift.so +AuthPlugInEnvName: +AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +SwiftRetryDelay: 100ms +SwiftRetryExpBackoff: 2 +SwiftRetryLimit: 4 +SwiftTimeout: 10m +SwiftConnectionPoolSize: 128 RetryRPCPublicIPAddr: imgr RetryRPCPort: 32356 RetryRPCDeadlineIO: 60s diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index b56357ba..0e5d6ea9 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -14,9 +14,14 @@ // FUSEMaxBackground: 1000 // FUSECongestionThreshhold: 0 // FUSEMaxWrite: 131076 -// PlugInPath: iauth-swift.so -// PlugInEnvName: SwiftAuthBlob -// PlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +// AuthPlugInPath: iauth-swift.so +// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here +// AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +// SwiftRetryDelay: 100ms +// SwiftRetryExpBackoff: 2 +// SwiftRetryLimit: 4 +// SwiftTimeout: 10m +// SwiftConnectionPoolSize: 128 // RetryRPCPublicIPAddr: imgr // RetryRPCPort: 32356 // RetryRPCDeadlineIO: 60s @@ -29,9 +34,10 @@ // HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) // HTTPServerPort: # Defaults to disabling the embedded HTTP Server // -// Most of the config keys are required and must have values. The exceptions are -// the HTTPServer{IPAddr|Port} keys that, if not present (or HTTPServerPort is zero) -// will disable the embedded HTTP Server. +// Most of the config keys are required and must have values. One set of exceptions +// are the HTTPServer{IPAddr|Port} keys that, if not present (or HTTPServerPort is +// zero) will disable the embedded HTTP Server. Another set of exceptions are the +// AuthPlungInEnv{Name|Value} keys of which exactly one must be present. // // The embedded HTTP Server (at URL http://:) // responds to the following: diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 44721c63..84e4bfbf 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -6,7 +6,6 @@ package iclientpkg import ( "container/list" "io/ioutil" - "log" "net/http" "os" "sync" @@ -27,9 +26,14 @@ type configStruct struct { FUSEMaxBackground uint16 FUSECongestionThreshhold uint16 FUSEMaxWrite uint32 - PlugInPath string - PlugInEnvName string - PlugInEnvValue string + AuthPlugInPath string + AuthPlugInEnvName string + AuthPlugInEnvValue string + SwiftRetryDelay time.Duration + SwiftRetryExpBackoff float64 + SwiftRetryLimit uint32 + SwiftTimeout time.Duration + SwiftConnectionPoolSize uint32 RetryRPCPublicIPAddr string RetryRPCPort uint16 RetryRPCDeadlineIO time.Duration @@ -151,6 +155,11 @@ type globalsStruct struct { config configStruct // logFile *os.File // == nil if config.LogFilePath == "" retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" + httpClient *http.Client // + swiftAuthInString string // + swiftAuthWaitGroup *sync.WaitGroup // != nil if updateAuthTokenAndStorageURL() is active + swiftAuthToken string // + swiftStorageURL string // retryRPCClientConfig *retryrpc.ClientConfig // retryRPCClient *retryrpc.Client // fissionErrChan chan error // @@ -167,8 +176,7 @@ var globals globalsStruct func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err error) { var ( - configJSONified string - plugInEnvValueSlice []string + configJSONified string ) // Default logging related globals @@ -203,36 +211,63 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err if nil != err { logFatal(err) } - globals.config.PlugInPath, err = confMap.FetchOptionValueString("ICLIENT", "PlugInPath") + globals.config.AuthPlugInPath, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInPath") if nil != err { logFatal(err) } - globals.config.PlugInEnvName, err = confMap.FetchOptionValueString("ICLIENT", "PlugInEnvName") - if nil != err { - logFatal(err) - } - globals.config.PlugInEnvValue, err = confMap.FetchOptionValueString("ICLIENT", "PlugInEnvValue") - if nil != err { - logFatal(err) - } - err = confMap.VerifyOptionIsMissing("ICLIENT", "PlugInEnvValue") + err = confMap.VerifyOptionIsMissing("ICLIENT", "AuthPlugInEnvName") if nil == err { - globals.config.PlugInEnvValue = "" - } else { - plugInEnvValueSlice, err = confMap.FetchOptionValueStringSlice("ICLIENT", "PlugInEnvValue") + globals.config.AuthPlugInEnvName = "" + globals.config.AuthPlugInEnvValue, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInEnvValue") if nil != err { logFatal(err) + } + } else { + err = confMap.VerifyOptionValueIsEmpty("ICLIENT", "AuthPlugInEnvName") + if nil == err { + globals.config.AuthPlugInEnvName = "" + globals.config.AuthPlugInEnvValue, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInEnvValue") + if nil != err { + logFatal(err) + } } else { - switch len(plugInEnvValueSlice) { - case 0: - globals.config.PlugInEnvValue = "" - case 1: - globals.config.PlugInEnvValue = plugInEnvValueSlice[0] - default: - log.Fatalf("[ICLIENT]PlugInEnvValue must be missing, empty, or single-valued: %#v", plugInEnvValueSlice) + globals.config.AuthPlugInEnvName, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInEnvName") + if nil != err { + logFatal(err) + } + err = confMap.VerifyOptionIsMissing("ICLIENT", "AuthPlugInEnvValue") + if nil == err { + globals.config.AuthPlugInEnvValue = "" + } else { + err = confMap.VerifyOptionValueIsEmpty("ICLIENT", "AuthPlugInEnvValue") + if nil == err { + globals.config.AuthPlugInEnvValue = "" + } else { + logFatalf("If [ICLIENT]AuthPlugInEnvName is present and non-empty, [ICLIENT]AuthPlugInEnvValue must be missing or empty") + } } } } + globals.config.SwiftRetryDelay, err = confMap.FetchOptionValueDuration("ICLIENT", "SwiftRetryDelay") + if nil != err { + logFatal(err) + } + globals.config.SwiftRetryExpBackoff, err = confMap.FetchOptionValueFloat64("ICLIENT", "SwiftRetryExpBackoff") + if nil != err { + logFatal(err) + } + globals.config.SwiftRetryLimit, err = confMap.FetchOptionValueUint32("ICLIENT", "SwiftRetryLimit") + if nil != err { + logFatal(err) + } + globals.config.SwiftTimeout, err = confMap.FetchOptionValueDuration("ICLIENT", "SwiftTimeout") + if nil != err { + logFatal(err) + } + globals.config.SwiftConnectionPoolSize, err = confMap.FetchOptionValueUint32("ICLIENT", "SwiftConnectionPoolSize") + if nil != err { + logFatal(err) + } globals.config.RetryRPCPublicIPAddr, err = confMap.FetchOptionValueString("ICLIENT", "RetryRPCPublicIPAddr") if nil != err { logFatal(err) @@ -315,8 +350,14 @@ func uninitializeGlobals() (err error) { globals.config.FUSEMaxBackground = 0 globals.config.FUSECongestionThreshhold = 0 globals.config.FUSEMaxWrite = 0 - globals.config.PlugInPath = "" - globals.config.PlugInEnvValue = "" + globals.config.AuthPlugInPath = "" + globals.config.AuthPlugInEnvName = "" + globals.config.AuthPlugInEnvValue = "" + globals.config.SwiftRetryDelay = time.Duration(0) + globals.config.SwiftRetryExpBackoff = 0.0 + globals.config.SwiftRetryLimit = 0 + globals.config.SwiftTimeout = time.Duration(0) + globals.config.SwiftConnectionPoolSize = 0 globals.config.RetryRPCPublicIPAddr = "" globals.config.RetryRPCPort = 0 globals.config.RetryRPCDeadlineIO = time.Duration(0) diff --git a/iclient/iclientpkg/impl.go b/iclient/iclientpkg/impl.go index c718e486..3b1e58fd 100644 --- a/iclient/iclientpkg/impl.go +++ b/iclient/iclientpkg/impl.go @@ -13,7 +13,7 @@ func start(confMap conf.ConfMap, fissionErrChan chan error) (err error) { return } - err = openRetryRPC() + err = startRPCHandler() if nil != err { return } @@ -60,7 +60,7 @@ func stop() (err error) { return } - err = closeRetryRPC() + err = stopRPCHandler() if nil != err { return } diff --git a/iclient/iclientpkg/retry-rpc.go b/iclient/iclientpkg/retry-rpc.go deleted file mode 100644 index ec85cdbb..00000000 --- a/iclient/iclientpkg/retry-rpc.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package iclientpkg - -import ( - "github.com/NVIDIA/proxyfs/retryrpc" -) - -func openRetryRPC() (err error) { - globals.retryRPCClientConfig = &retryrpc.ClientConfig{ - DNSOrIPAddr: globals.config.RetryRPCPublicIPAddr, - Port: int(globals.config.RetryRPCPort), - RootCAx509CertificatePEM: globals.retryRPCCACertPEM, - Callbacks: globals, - DeadlineIO: globals.config.RetryRPCDeadlineIO, - KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, - } - - globals.retryRPCClient, err = retryrpc.NewClient(globals.retryRPCClientConfig) - - return // err as set by call to retryrpc.NewClient() is sufficient -} - -func closeRetryRPC() (err error) { - globals.retryRPCClient.Close() - - globals.retryRPCClientConfig = nil - globals.retryRPCClient = nil - - err = nil - return -} diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go new file mode 100644 index 00000000..69ed3dac --- /dev/null +++ b/iclient/iclientpkg/rpc.go @@ -0,0 +1,162 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +import ( + "fmt" + "net/http" + "os" + "sync" + + "github.com/NVIDIA/proxyfs/iauth" + "github.com/NVIDIA/proxyfs/retryrpc" +) + +func startRPCHandler() (err error) { + var ( + customTransport *http.Transport + defaultTransport *http.Transport + ok bool + ) + + defaultTransport, ok = http.DefaultTransport.(*http.Transport) + if !ok { + err = fmt.Errorf("http.DefaultTransport.(*http.Transport) returned !ok\n") + return + } + + customTransport = &http.Transport{ // Up-to-date as of Golang 1.11 + Proxy: defaultTransport.Proxy, + DialContext: defaultTransport.DialContext, + Dial: defaultTransport.Dial, + DialTLS: defaultTransport.DialTLS, + TLSClientConfig: defaultTransport.TLSClientConfig, + TLSHandshakeTimeout: globals.config.SwiftTimeout, + DisableKeepAlives: false, + DisableCompression: defaultTransport.DisableCompression, + MaxIdleConns: int(globals.config.SwiftConnectionPoolSize), + MaxIdleConnsPerHost: int(globals.config.SwiftConnectionPoolSize), + MaxConnsPerHost: int(globals.config.SwiftConnectionPoolSize), + IdleConnTimeout: globals.config.SwiftTimeout, + ResponseHeaderTimeout: globals.config.SwiftTimeout, + ExpectContinueTimeout: globals.config.SwiftTimeout, + TLSNextProto: defaultTransport.TLSNextProto, + ProxyConnectHeader: defaultTransport.ProxyConnectHeader, + MaxResponseHeaderBytes: defaultTransport.MaxResponseHeaderBytes, + } + + globals.httpClient = &http.Client{ + Transport: customTransport, + Timeout: globals.config.SwiftTimeout, + } + + if globals.config.AuthPlugInEnvName == "" { + globals.swiftAuthInString = globals.config.AuthPlugInEnvValue + } else { + globals.swiftAuthInString = os.Getenv(globals.config.AuthPlugInEnvName) + } + + updateSwithAuthTokenAndSwiftStorageURL() + + globals.retryRPCClientConfig = &retryrpc.ClientConfig{ + DNSOrIPAddr: globals.config.RetryRPCPublicIPAddr, + Port: int(globals.config.RetryRPCPort), + RootCAx509CertificatePEM: globals.retryRPCCACertPEM, + Callbacks: &globals, + DeadlineIO: globals.config.RetryRPCDeadlineIO, + KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, + } + + globals.retryRPCClient, err = retryrpc.NewClient(globals.retryRPCClientConfig) + + return // err as set by call to retryrpc.NewClient() is sufficient +} + +func stopRPCHandler() (err error) { + globals.retryRPCClient.Close() + + globals.httpClient = nil + + globals.retryRPCClientConfig = nil + globals.retryRPCClient = nil + + err = nil + return +} + +func fetchSwiftAuthToken() (swiftAuthToken string) { + var ( + swiftAuthWaitGroup *sync.WaitGroup + ) + + for { + globals.Lock() + + swiftAuthWaitGroup = globals.swiftAuthWaitGroup + + if nil == swiftAuthWaitGroup { + swiftAuthToken = globals.swiftAuthToken + globals.Unlock() + return + } + + globals.Lock() + + swiftAuthWaitGroup.Wait() + } +} + +func fetchSwiftStorageURL() (swiftStorageURL string) { + var ( + swiftAuthWaitGroup *sync.WaitGroup + ) + + for { + globals.Lock() + + swiftAuthWaitGroup = globals.swiftAuthWaitGroup + + if nil == swiftAuthWaitGroup { + swiftStorageURL = globals.swiftStorageURL + globals.Unlock() + return + } + + globals.Lock() + + swiftAuthWaitGroup.Wait() + } +} + +func updateSwithAuthTokenAndSwiftStorageURL() { + var ( + err error + swiftAuthWaitGroup *sync.WaitGroup + ) + + globals.Lock() + + swiftAuthWaitGroup = globals.swiftAuthWaitGroup + + if nil != swiftAuthWaitGroup { + globals.Unlock() + swiftAuthWaitGroup.Wait() + return + } + + globals.swiftAuthWaitGroup = &sync.WaitGroup{} + globals.swiftAuthWaitGroup.Add(1) + + globals.Unlock() + + globals.swiftAuthToken, globals.swiftStorageURL, err = iauth.PerformAuth(globals.config.AuthPlugInPath, globals.swiftAuthInString) + if nil != err { + logFatalf("iauth.PerformAuth() failed: %v", err) + } + + globals.Lock() + globals.swiftAuthWaitGroup.Done() + globals.swiftAuthWaitGroup = nil + globals.Unlock() +} From ab1d5196ac7244b6db431eacf3b7040df7606c97 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 29 Sep 2021 15:53:55 -0700 Subject: [PATCH 104/258] iclient Lock granting (while not contending) is now functional Demonstrated in DoGetAttr() only... the Unlock operation is currently a no-op... but since lock request is shared, one can repeat e.g. stat /mnt --- iclient/iclientpkg/fission.go | 13 ++++ iclient/iclientpkg/globals.go | 1 + iclient/iclientpkg/lease.go | 130 +++++++++++++++++++++++++++++++--- iclient/iclientpkg/rpc.go | 56 ++++++++++++++- 4 files changed, 189 insertions(+), 11 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 8730ab4e..f369d9f6 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -4,6 +4,7 @@ package iclientpkg import ( + "fmt" "syscall" "time" @@ -106,6 +107,18 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis }() // TODO + fmt.Printf("UNDO: inHeader: %+v\n", inHeader) // UNDO + fmt.Printf("UNDO: getAttrIn: %+v\n", getAttrIn) // UNDO + UNDOlockRequest := newLockRequest() + UNDOlockRequest.inodeNumber = uint64(inHeader.NodeID) + UNDOlockRequest.exclusive = false + fmt.Printf("UNDO: before addThisLock(), UNDOlockRequest: %+v\n", UNDOlockRequest) + UNDOlockRequest.addThisLock() + fmt.Printf("UNDO: after addThisLock(), UNDOlockRequest: %+v\n", UNDOlockRequest) + for UNDOinodeNumber, UNDOinodeHeldLock := range UNDOlockRequest.locksHeld { + fmt.Printf("UNDO: ...UNDOinodeNumber: %v UNDOinodeHeldLock: %+v\n", UNDOinodeNumber, UNDOinodeHeldLock) + } + UNDOlockRequest.unlockAllWhileLocked() getAttrOut = nil errno = syscall.ENOSYS return diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 84e4bfbf..4525ed69 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -162,6 +162,7 @@ type globalsStruct struct { swiftStorageURL string // retryRPCClientConfig *retryrpc.ClientConfig // retryRPCClient *retryrpc.Client // + mountID string // fissionErrChan chan error // inodeLeaseTable map[uint64]*inodeLeaseStruct // inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index e2aaca5d..cc8e5c65 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -5,6 +5,8 @@ package iclientpkg import ( "container/list" + + "github.com/NVIDIA/proxyfs/imgr/imgrpkg" ) func startLeaseHandler() (err error) { @@ -44,8 +46,11 @@ func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { // func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { var ( + err error inodeHeldLock *inodeHeldLockStruct inodeLease *inodeLeaseStruct + leaseRequest *imgrpkg.LeaseRequestStruct + leaseResponse *imgrpkg.LeaseResponseStruct ok bool ) @@ -122,10 +127,43 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { if inodeLockRequest.exclusive { switch inodeLease.state { case inodeLeaseStateNone: + inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateExclusiveRequested - - // TODO - this is where we issue imgrpkg.LeaseRequestTypeExclusive - + globals.Unlock() + RetryLeaseRequestTypeExclusive: + leaseRequest = &imgrpkg.LeaseRequestStruct{ + MountID: globals.mountID, + InodeNumber: inodeLockRequest.inodeNumber, + LeaseRequestType: imgrpkg.LeaseRequestTypeExclusive, + } + leaseResponse = &imgrpkg.LeaseResponseStruct{} + err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) + if nil != err { + err = renewRPCHandler() + if nil != err { + logFatal(err) + } + goto RetryLeaseRequestTypeExclusive + } + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeExclusive { + logFatalf("TODO: for now, we don't handle a Lease Request actually failing") + } + globals.Lock() + inodeLease.state = inodeLeaseStateExclusiveGranted + if inodeLease.requestList.Front() != inodeLockRequest.listElement { + logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") + } + _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) + inodeLockRequest.listElement = nil + // We can immediately grant the exclusive inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ + inodeLease: inodeLease, + inodeLockRequest: inodeLockRequest, + exclusive: true, + } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + globals.Unlock() case inodeLeaseStateSharedRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) @@ -133,10 +171,43 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateSharedGranted: + inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateSharedPromoting - - // TODO - this is where we issue imgrpkg.LeaseRequestTypePromote - + globals.Unlock() + RetryLeaseRequestTypePromote: + leaseRequest = &imgrpkg.LeaseRequestStruct{ + MountID: globals.mountID, + InodeNumber: inodeLockRequest.inodeNumber, + LeaseRequestType: imgrpkg.LeaseRequestTypePromote, + } + leaseResponse = &imgrpkg.LeaseResponseStruct{} + err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) + if nil != err { + err = renewRPCHandler() + if nil != err { + logFatal(err) + } + goto RetryLeaseRequestTypePromote + } + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypePromoted { + logFatalf("TODO: for now, we don't handle a Lease Request actually failing") + } + globals.Lock() + inodeLease.state = inodeLeaseStateExclusiveGranted + if inodeLease.requestList.Front() != inodeLockRequest.listElement { + logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") + } + _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) + inodeLockRequest.listElement = nil + // We can immediately grant the exclusive inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ + inodeLease: inodeLease, + inodeLockRequest: inodeLockRequest, + exclusive: true, + } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + globals.Unlock() case inodeLeaseStateSharedPromoting: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) @@ -191,10 +262,46 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } else { switch inodeLease.state { case inodeLeaseStateNone: + inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateSharedRequested - - // TODO - this is where we issue imgrpkg.LeaseRequestTypeShared - + globals.Unlock() + RetryLeaseRequestTypeShared: + leaseRequest = &imgrpkg.LeaseRequestStruct{ + MountID: globals.mountID, + InodeNumber: inodeLockRequest.inodeNumber, + LeaseRequestType: imgrpkg.LeaseRequestTypeShared, + } + leaseResponse = &imgrpkg.LeaseResponseStruct{} + err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) + if nil != err { + err = renewRPCHandler() + if nil != err { + logFatal(err) + } + goto RetryLeaseRequestTypeShared + } + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeShared { + logFatalf("TODO: for now, we don't handle a Lease Request actually failing") + } + globals.Lock() + inodeLease.state = inodeLeaseStateSharedGranted + if inodeLease.requestList.Front() != inodeLockRequest.listElement { + logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") + } + _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) + inodeLockRequest.listElement = nil + // We can immediately grant the shared inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ + inodeLease: inodeLease, + inodeLockRequest: inodeLockRequest, + exclusive: false, + } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + if inodeLease.requestList.Front() != nil { + logFatalf("TODO: for now, we don't handle multiple shared lock requests queued up") + } + globals.Unlock() case inodeLeaseStateSharedRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) @@ -276,5 +383,8 @@ func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { // unlockAllWhileLocked is what unlockAll calls after obtaining globals.Lock(). // func (inodeLockRequest *inodeLockRequestStruct) unlockAllWhileLocked() { - // TODO + logInfof("TODO: (*inodeLockRequestStruct)unlockAllWhileLocked() called to release %d lock(s)", len(inodeLockRequest.locksHeld)) + for inodeNumber := range inodeLockRequest.locksHeld { + logInfof("TODO: ...inodeNumber %v", inodeNumber) + } } diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index 69ed3dac..d2ea4a8a 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/NVIDIA/proxyfs/iauth" + "github.com/NVIDIA/proxyfs/imgr/imgrpkg" "github.com/NVIDIA/proxyfs/retryrpc" ) @@ -17,6 +18,8 @@ func startRPCHandler() (err error) { var ( customTransport *http.Transport defaultTransport *http.Transport + mountRequest *imgrpkg.MountRequestStruct + mountResponse *imgrpkg.MountResponseStruct ok bool ) @@ -69,11 +72,62 @@ func startRPCHandler() (err error) { } globals.retryRPCClient, err = retryrpc.NewClient(globals.retryRPCClientConfig) + if nil != err { + return + } + + mountRequest = &imgrpkg.MountRequestStruct{ + VolumeName: globals.config.VolumeName, + AuthToken: fetchSwiftAuthToken(), + } + mountResponse = &imgrpkg.MountResponseStruct{} + + err = globals.retryRPCClient.Send("Mount", mountRequest, mountResponse) + if nil != err { + return + } + + globals.mountID = mountResponse.MountID + + err = nil + return +} + +func renewRPCHandler() (err error) { + var ( + renewMountRequest *imgrpkg.RenewMountRequestStruct + renewMountResponse *imgrpkg.RenewMountResponseStruct + ) + + updateSwithAuthTokenAndSwiftStorageURL() - return // err as set by call to retryrpc.NewClient() is sufficient + renewMountRequest = &imgrpkg.RenewMountRequestStruct{ + MountID: globals.mountID, + AuthToken: fetchSwiftAuthToken(), + } + renewMountResponse = &imgrpkg.RenewMountResponseStruct{} + + err = globals.retryRPCClient.Send("RenewMount", renewMountRequest, renewMountResponse) + + return // err, as set by globals.retryRPCClient.Send("RenewMount", renewMountRequest, renewMountResponse) is sufficient } func stopRPCHandler() (err error) { + var ( + unmountRequest *imgrpkg.UnmountRequestStruct + unmountResponse *imgrpkg.UnmountResponseStruct + ) + + unmountRequest = &imgrpkg.UnmountRequestStruct{ + MountID: globals.mountID, + } + unmountResponse = &imgrpkg.UnmountResponseStruct{} + + err = globals.retryRPCClient.Send("Unmount", unmountRequest, unmountResponse) + if nil != err { + logWarn(err) + } + globals.retryRPCClient.Close() globals.httpClient = nil From 8b2fc895bc4f9896b38128271367fd7433e33b2c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 1 Oct 2021 10:58:43 -0700 Subject: [PATCH 105/258] Non-blocking locks, honoring leases, are now implemented in iclient/iclientpkg ...logFatalf("TODO:...")'s will be called if a currently unsupported lock state (e.g. unblocking another lock request) is encountered --- iclient/iclientpkg/globals.go | 1 + iclient/iclientpkg/lease.go | 55 +++++++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 4525ed69..e7c50763 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -64,6 +64,7 @@ const ( ) type inodeLeaseStruct struct { + inodeNumber uint64 // state inodeLeaseStateType // heldList *list.List // List of granted inodeHeldLockStruct's requestList *list.List // List of pending inodeLockRequestStruct's diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index cc8e5c65..43dd8c08 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -67,6 +67,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLease, ok = globals.inodeLeaseTable[inodeLockRequest.inodeNumber] if !ok { inodeLease = &inodeLeaseStruct{ + inodeNumber: inodeLockRequest.inodeNumber, state: inodeLeaseStateNone, heldList: list.New(), requestList: list.New(), @@ -129,14 +130,18 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateNone: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateExclusiveRequested + globals.Unlock() + RetryLeaseRequestTypeExclusive: + leaseRequest = &imgrpkg.LeaseRequestStruct{ MountID: globals.mountID, InodeNumber: inodeLockRequest.inodeNumber, LeaseRequestType: imgrpkg.LeaseRequestTypeExclusive, } leaseResponse = &imgrpkg.LeaseResponseStruct{} + err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) if nil != err { err = renewRPCHandler() @@ -145,24 +150,31 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } goto RetryLeaseRequestTypeExclusive } + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeExclusive { logFatalf("TODO: for now, we don't handle a Lease Request actually failing") } + globals.Lock() + inodeLease.state = inodeLeaseStateExclusiveGranted + if inodeLease.requestList.Front() != inodeLockRequest.listElement { logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") } + _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) inodeLockRequest.listElement = nil - // We can immediately grant the exclusive inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ inodeLease: inodeLease, inodeLockRequest: inodeLockRequest, exclusive: true, } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + globals.Unlock() case inodeLeaseStateSharedRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct @@ -173,14 +185,18 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateSharedGranted: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateSharedPromoting + globals.Unlock() + RetryLeaseRequestTypePromote: + leaseRequest = &imgrpkg.LeaseRequestStruct{ MountID: globals.mountID, InodeNumber: inodeLockRequest.inodeNumber, LeaseRequestType: imgrpkg.LeaseRequestTypePromote, } leaseResponse = &imgrpkg.LeaseResponseStruct{} + err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) if nil != err { err = renewRPCHandler() @@ -189,24 +205,31 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } goto RetryLeaseRequestTypePromote } + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypePromoted { logFatalf("TODO: for now, we don't handle a Lease Request actually failing") } + globals.Lock() + inodeLease.state = inodeLeaseStateExclusiveGranted + if inodeLease.requestList.Front() != inodeLockRequest.listElement { logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") } + _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) inodeLockRequest.listElement = nil - // We can immediately grant the exclusive inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ inodeLease: inodeLease, inodeLockRequest: inodeLockRequest, exclusive: true, } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + globals.Unlock() case inodeLeaseStateSharedPromoting: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct @@ -264,14 +287,18 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateNone: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateSharedRequested + globals.Unlock() + RetryLeaseRequestTypeShared: + leaseRequest = &imgrpkg.LeaseRequestStruct{ MountID: globals.mountID, InodeNumber: inodeLockRequest.inodeNumber, LeaseRequestType: imgrpkg.LeaseRequestTypeShared, } leaseResponse = &imgrpkg.LeaseResponseStruct{} + err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) if nil != err { err = renewRPCHandler() @@ -280,27 +307,35 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } goto RetryLeaseRequestTypeShared } + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeShared { logFatalf("TODO: for now, we don't handle a Lease Request actually failing") } + globals.Lock() + inodeLease.state = inodeLeaseStateSharedGranted + if inodeLease.requestList.Front() != inodeLockRequest.listElement { logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") } + _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) inodeLockRequest.listElement = nil - // We can immediately grant the shared inodeLockRequestStruct + inodeHeldLock = &inodeHeldLockStruct{ inodeLease: inodeLease, inodeLockRequest: inodeLockRequest, exclusive: false, } + inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + if inodeLease.requestList.Front() != nil { logFatalf("TODO: for now, we don't handle multiple shared lock requests queued up") } + globals.Unlock() case inodeLeaseStateSharedRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct @@ -383,8 +418,16 @@ func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { // unlockAllWhileLocked is what unlockAll calls after obtaining globals.Lock(). // func (inodeLockRequest *inodeLockRequestStruct) unlockAllWhileLocked() { - logInfof("TODO: (*inodeLockRequestStruct)unlockAllWhileLocked() called to release %d lock(s)", len(inodeLockRequest.locksHeld)) - for inodeNumber := range inodeLockRequest.locksHeld { - logInfof("TODO: ...inodeNumber %v", inodeNumber) + var ( + inodeHeldLock *inodeHeldLockStruct + ) + + for _, inodeHeldLock = range inodeLockRequest.locksHeld { + _ = inodeHeldLock.inodeLease.heldList.Remove(inodeHeldLock.listElement) + if inodeHeldLock.inodeLease.requestList.Len() != 0 { + logFatalf("TODO: for now, we don't handle blocked inodeLockRequestStruct's") + } } + + inodeLockRequest.locksHeld = make(map[uint64]*inodeHeldLockStruct) } From d4f5ee7a14a08eb23146d3649526ee82943e793d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 1 Oct 2021 11:49:32 -0700 Subject: [PATCH 106/258] Cleaned up some debugging (UNDO-adorned) code Now... LeaseTable display is actually correct on iclient web page --- iclient/iclientpkg/fission.go | 21 +++++++------------ iclient/iclientpkg/http-server.go | 34 ------------------------------- 2 files changed, 7 insertions(+), 48 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index f369d9f6..288310e4 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -4,7 +4,6 @@ package iclientpkg import ( - "fmt" "syscall" "time" @@ -94,7 +93,8 @@ func (dummy *globalsStruct) DoForget(inHeader *fission.InHeader, forgetIn *fissi func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fission.GetAttrIn) (getAttrOut *fission.GetAttrOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + inodeLockRequest *inodeLockRequestStruct + startTime time.Time = time.Now() ) logTracef("==> DoGetAttr(inHeader: %+v, getAttrIn: %+v)", inHeader, getAttrIn) @@ -107,18 +107,11 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis }() // TODO - fmt.Printf("UNDO: inHeader: %+v\n", inHeader) // UNDO - fmt.Printf("UNDO: getAttrIn: %+v\n", getAttrIn) // UNDO - UNDOlockRequest := newLockRequest() - UNDOlockRequest.inodeNumber = uint64(inHeader.NodeID) - UNDOlockRequest.exclusive = false - fmt.Printf("UNDO: before addThisLock(), UNDOlockRequest: %+v\n", UNDOlockRequest) - UNDOlockRequest.addThisLock() - fmt.Printf("UNDO: after addThisLock(), UNDOlockRequest: %+v\n", UNDOlockRequest) - for UNDOinodeNumber, UNDOinodeHeldLock := range UNDOlockRequest.locksHeld { - fmt.Printf("UNDO: ...UNDOinodeNumber: %v UNDOinodeHeldLock: %+v\n", UNDOinodeNumber, UNDOinodeHeldLock) - } - UNDOlockRequest.unlockAllWhileLocked() + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) + inodeLockRequest.exclusive = false + inodeLockRequest.addThisLock() + inodeLockRequest.unlockAllWhileLocked() getAttrOut = nil errno = syscall.ENOSYS return diff --git a/iclient/iclientpkg/http-server.go b/iclient/iclientpkg/http-server.go index c537fe23..bfc00f89 100644 --- a/iclient/iclientpkg/http-server.go +++ b/iclient/iclientpkg/http-server.go @@ -205,8 +205,6 @@ func (s inodeLeaseTableByInodeNumberSlice) Less(i, j int) bool { return s[i].InodeNumber < s[j].InodeNumber } -var numUNDOs = uint64(1) - func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Request) { var ( err error @@ -224,20 +222,6 @@ func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Requ globals.Lock() - for i := uint64(0); i < numUNDOs; i++ { - globals.inodeLeaseTable[(13*i)+1] = &inodeLeaseStruct{state: inodeLeaseStateNone} // UNDO - globals.inodeLeaseTable[(13*i)+2] = &inodeLeaseStruct{state: inodeLeaseStateSharedRequested} // UNDO - globals.inodeLeaseTable[(13*i)+3] = &inodeLeaseStruct{state: inodeLeaseStateSharedGranted} // UNDO - globals.inodeLeaseTable[(13*i)+4] = &inodeLeaseStruct{state: inodeLeaseStateSharedPromoting} // UNDO - globals.inodeLeaseTable[(13*i)+5] = &inodeLeaseStruct{state: inodeLeaseStateSharedReleasing} // UNDO - globals.inodeLeaseTable[(13*i)+6] = &inodeLeaseStruct{state: inodeLeaseStateSharedExpired} // UNDO - globals.inodeLeaseTable[(13*i)+7] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveRequested} // UNDO - globals.inodeLeaseTable[(13*i)+8] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveGranted} // UNDO - globals.inodeLeaseTable[(13*i)+9] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveDemoting} // UNDO - globals.inodeLeaseTable[(13*i)+10] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveReleasing} // UNDO - globals.inodeLeaseTable[(13*i)+11] = &inodeLeaseStruct{state: inodeLeaseStateExclusiveExpired} // UNDO - } // UNDO - inodeLeaseTable = make(inodeLeaseTableByInodeNumberSlice, len(globals.inodeLeaseTable)) inodeLeaseTableIndex = 0 for inodeNumber, inodeLease = range globals.inodeLeaseTable { @@ -271,20 +255,6 @@ func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Requ inodeLeaseTableIndex++ } - for i := uint64(0); i < numUNDOs; i++ { - delete(globals.inodeLeaseTable, (13*i)+1) // UNDO - delete(globals.inodeLeaseTable, (13*i)+2) // UNDO - delete(globals.inodeLeaseTable, (13*i)+3) // UNDO - delete(globals.inodeLeaseTable, (13*i)+4) // UNDO - delete(globals.inodeLeaseTable, (13*i)+5) // UNDO - delete(globals.inodeLeaseTable, (13*i)+6) // UNDO - delete(globals.inodeLeaseTable, (13*i)+7) // UNDO - delete(globals.inodeLeaseTable, (13*i)+8) // UNDO - delete(globals.inodeLeaseTable, (13*i)+9) // UNDO - delete(globals.inodeLeaseTable, (13*i)+10) // UNDO - delete(globals.inodeLeaseTable, (13*i)+11) // UNDO - } // UNDO - globals.Unlock() sort.Sort(inodeLeaseTable) @@ -378,7 +348,6 @@ func serveHTTPPostOfLeasesDemote(responseWriter http.ResponseWriter, request *ht }() logWarnf("serveHTTPPostOfLeasesDemote() TODO") - numUNDOs++ responseWriter.WriteHeader(http.StatusOK) } @@ -393,9 +362,6 @@ func serveHTTPPostOfLeasesRelease(responseWriter http.ResponseWriter, request *h }() logWarnf("serveHTTPPostOfLeasesRelease() TODO") - if numUNDOs > 0 { - numUNDOs-- - } // UNDO responseWriter.WriteHeader(http.StatusOK) } From 72daefbe7a67bfe9ddc2166fc03802cbca8ebffb Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 6 Oct 2021 09:19:58 -0700 Subject: [PATCH 107/258] Added iclient .conf settings for capping {Shared|Exclusive}Leases and {Read|Write}Cache With the supplied example settings, the {Read|Write}Cache are each effectively limited to 1GiB --- iclient/dev.conf | 6 ++++++ iclient/iclient.conf | 6 ++++++ iclient/iclientpkg/api.go | 6 ++++++ iclient/iclientpkg/globals.go | 36 +++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/iclient/dev.conf b/iclient/dev.conf index 563a383b..13602810 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -21,6 +21,12 @@ RetryRPCPort: 32356 RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s RetryRPCCACertFilePath: # Defaults to /dev/null +MaxSharedLeases: 500 +MaxExclusiveLeases: 100 +ReadCacheLineSize: 1048576 +ReadCacheLineCountMax: 1024 +FileFlushTriggerSize: 10485760 +FileFlushTriggerDuration: 10s LogFilePath: # Defaults to "" LogToConsole: true TraceEnabled: false diff --git a/iclient/iclient.conf b/iclient/iclient.conf index 08fad368..e03eb1df 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -21,6 +21,12 @@ RetryRPCPort: 32356 RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s RetryRPCCACertFilePath: caCert.pem +MaxSharedLeases: 500 +MaxExclusiveLeases: 100 +ReadCacheLineSize: 1048576 +ReadCacheLineCountMax: 1024 +FileFlushTriggerSize: 10485760 +FileFlushTriggerDuration: 10s LogFilePath: # Defaults to "" LogToConsole: true TraceEnabled: false diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index 0e5d6ea9..f5e242b5 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -27,6 +27,12 @@ // RetryRPCDeadlineIO: 60s // RetryRPCKeepAlivePeriod: 60s // RetryRPCCACertFilePath: # Defaults to /dev/null +// MaxSharedLeases: 500 +// MaxExclusiveLeases: 100 # Caps pending FileFlush data at 1GiB (with FileFlushTriggerSize of 10MiB) +// ReadCacheLineSize: 1048576 # 1MiB +// ReadCacheLineCountMax: 1024 # 1GiB (with ReadCacheLineSize of 1MiB) +// FileFlushTriggerSize: 10485760 # [10MiB] Amount of written data before metadata is appended and flush is triggered +// FileFlushTriggerDuration: 10s # Amount of time before unwritten data and its metadata flush is triggered // LogFilePath: iclient.log // LogToConsole: true // TraceEnabled: false diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index e7c50763..926371f3 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -39,6 +39,12 @@ type configStruct struct { RetryRPCDeadlineIO time.Duration RetryRPCKeepAlivePeriod time.Duration RetryRPCCACertFilePath string // Defaults to /dev/null + MaxSharedLeases uint64 + MaxExclusiveLeases uint64 + ReadCacheLineSize uint64 + ReadCacheLineCountMax uint64 + FileFlushTriggerSize uint64 + FileFlushTriggerDuration time.Duration LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled LogToConsole bool TraceEnabled bool @@ -290,6 +296,30 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err if nil != err { globals.config.RetryRPCCACertFilePath = "" } + globals.config.MaxSharedLeases, err = confMap.FetchOptionValueUint64("ICLIENT", "MaxSharedLeases") + if nil != err { + logFatal(err) + } + globals.config.MaxExclusiveLeases, err = confMap.FetchOptionValueUint64("ICLIENT", "MaxExclusiveLeases") + if nil != err { + logFatal(err) + } + globals.config.ReadCacheLineSize, err = confMap.FetchOptionValueUint64("ICLIENT", "ReadCacheLineSize") + if nil != err { + logFatal(err) + } + globals.config.ReadCacheLineCountMax, err = confMap.FetchOptionValueUint64("ICLIENT", "ReadCacheLineCountMax") + if nil != err { + logFatal(err) + } + globals.config.FileFlushTriggerSize, err = confMap.FetchOptionValueUint64("ICLIENT", "FileFlushTriggerSize") + if nil != err { + logFatal(err) + } + globals.config.FileFlushTriggerDuration, err = confMap.FetchOptionValueDuration("ICLIENT", "FileFlushTriggerDuration") + if nil != err { + logFatal(err) + } globals.config.LogFilePath, err = confMap.FetchOptionValueString("ICLIENT", "LogFilePath") if nil != err { err = confMap.VerifyOptionValueIsEmpty("ICLIENT", "LogFilePath") @@ -365,6 +395,12 @@ func uninitializeGlobals() (err error) { globals.config.RetryRPCDeadlineIO = time.Duration(0) globals.config.RetryRPCKeepAlivePeriod = time.Duration(0) globals.config.RetryRPCCACertFilePath = "" + globals.config.MaxSharedLeases = 0 + globals.config.MaxExclusiveLeases = 0 + globals.config.ReadCacheLineSize = 0 + globals.config.ReadCacheLineCountMax = 0 + globals.config.FileFlushTriggerSize = 0 + globals.config.FileFlushTriggerDuration = time.Duration(0) globals.config.LogFilePath = "" globals.config.LogToConsole = false globals.config.TraceEnabled = false From 828d59604ef126174685d20fd04943373c745181 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 6 Oct 2021 11:48:14 -0700 Subject: [PATCH 108/258] Beginning to flush out DoGetAttr()... had to build lots of infrastructure first ...including adding LRU framework for managing leases... --- iclient/iclientpkg/fission.go | 43 ++++++++- iclient/iclientpkg/globals.go | 17 +++- iclient/iclientpkg/lease.go | 80 +++++++++++----- iclient/iclientpkg/rpc.go | 176 +++++++++++++++++++++++++++++++++- 4 files changed, 279 insertions(+), 37 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 288310e4..c6bfdf88 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -4,10 +4,14 @@ package iclientpkg import ( + "fmt" "syscall" "time" "github.com/NVIDIA/fission" + + "github.com/NVIDIA/proxyfs/ilayout" + "github.com/NVIDIA/proxyfs/imgr/imgrpkg" ) const ( @@ -93,8 +97,12 @@ func (dummy *globalsStruct) DoForget(inHeader *fission.InHeader, forgetIn *fissi func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fission.GetAttrIn) (getAttrOut *fission.GetAttrOut, errno syscall.Errno) { var ( - inodeLockRequest *inodeLockRequestStruct - startTime time.Time = time.Now() + err error + getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct + getInodeTableEntryResponse *imgrpkg.GetInodeTableEntryResponseStruct + inodeLease *inodeLeaseStruct + inodeLockRequest *inodeLockRequestStruct + startTime time.Time = time.Now() ) logTracef("==> DoGetAttr(inHeader: %+v, getAttrIn: %+v)", inHeader, getAttrIn) @@ -106,12 +114,39 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis globals.stats.DoGetAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO inodeLockRequest = newLockRequest() inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) inodeLockRequest.exclusive = false inodeLockRequest.addThisLock() - inodeLockRequest.unlockAllWhileLocked() + + inodeLease = lookupInodeLease(uint64(inHeader.NodeID)) + if nil == inodeLease { + inodeLockRequest.unlockAll() + getAttrOut = nil + errno = syscall.ENOENT + return + } + + if nil == inodeLease.inodeHeadV1 { + getInodeTableEntryRequest = &imgrpkg.GetInodeTableEntryRequestStruct{ + MountID: globals.mountID, + InodeNumber: uint64(inHeader.NodeID), + } + getInodeTableEntryResponse = &imgrpkg.GetInodeTableEntryResponseStruct{} + + err = rpcGetInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) + if nil != err { + inodeLockRequest.unlockAll() + getAttrOut = nil + errno = syscall.ENOENT + return + } + + fmt.Printf("TODO: need to fetch .inodeHeadV1 via getInodeTableEntryResponse: %+v\n", getInodeTableEntryResponse) + inodeLease.inodeHeadV1 = &ilayout.InodeHeadV1Struct{} + } + + inodeLockRequest.unlockAll() getAttrOut = nil errno = syscall.ENOSYS return diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 926371f3..20a2924b 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -15,6 +15,7 @@ import ( "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/retryrpc" "github.com/NVIDIA/proxyfs/utils" ) @@ -70,10 +71,12 @@ const ( ) type inodeLeaseStruct struct { - inodeNumber uint64 // - state inodeLeaseStateType // - heldList *list.List // List of granted inodeHeldLockStruct's - requestList *list.List // List of pending inodeLockRequestStruct's + inodeNumber uint64 // + state inodeLeaseStateType // + listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU + heldList *list.List // List of granted inodeHeldLockStruct's + requestList *list.List // List of pending inodeLockRequestStruct's + inodeHeadV1 *ilayout.InodeHeadV1Struct // } type inodeHeldLockStruct struct { @@ -174,6 +177,8 @@ type globalsStruct struct { inodeLeaseTable map[uint64]*inodeLeaseStruct // inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits inodeLockWG sync.WaitGroup // Signaled as each active inodeLockRequestStruct has signaled service complete with .locksHeld empty + sharedLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted + exclusiveLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted httpServer *http.Server // httpServerWG sync.WaitGroup // stats *statsStruct // @@ -366,6 +371,8 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err globals.fissionErrChan = fissionErrChan globals.inodeLeaseTable = make(map[uint64]*inodeLeaseStruct) + globals.sharedLeaseLRU = list.New() + globals.exclusiveLeaseLRU = list.New() globals.stats = &statsStruct{} @@ -411,6 +418,8 @@ func uninitializeGlobals() (err error) { globals.retryRPCCACertPEM = nil globals.inodeLeaseTable = nil + globals.sharedLeaseLRU = nil + globals.exclusiveLeaseLRU = nil globals.fissionErrChan = nil diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 43dd8c08..d458f59d 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -65,12 +65,36 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } inodeLease, ok = globals.inodeLeaseTable[inodeLockRequest.inodeNumber] - if !ok { + if ok { + switch inodeLease.state { + case inodeLeaseStateNone: + case inodeLeaseStateSharedRequested: + globals.sharedLeaseLRU.MoveToBack(inodeLease.listElement) + case inodeLeaseStateSharedGranted: + globals.sharedLeaseLRU.MoveToBack(inodeLease.listElement) + case inodeLeaseStateSharedPromoting: + globals.exclusiveLeaseLRU.MoveToBack(inodeLease.listElement) + case inodeLeaseStateSharedReleasing: + case inodeLeaseStateSharedExpired: + case inodeLeaseStateExclusiveRequested: + globals.exclusiveLeaseLRU.MoveToBack(inodeLease.listElement) + case inodeLeaseStateExclusiveGranted: + globals.exclusiveLeaseLRU.MoveToBack(inodeLease.listElement) + case inodeLeaseStateExclusiveDemoting: + globals.sharedLeaseLRU.MoveToBack(inodeLease.listElement) + case inodeLeaseStateExclusiveReleasing: + case inodeLeaseStateExclusiveExpired: + default: + logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) + } + } else { inodeLease = &inodeLeaseStruct{ inodeNumber: inodeLockRequest.inodeNumber, state: inodeLeaseStateNone, + listElement: nil, heldList: list.New(), requestList: list.New(), + inodeHeadV1: nil, } globals.inodeLeaseTable[inodeLockRequest.inodeNumber] = inodeLease @@ -130,11 +154,10 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateNone: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateExclusiveRequested + inodeLease.listElement = globals.exclusiveLeaseLRU.PushBack(inodeLease) globals.Unlock() - RetryLeaseRequestTypeExclusive: - leaseRequest = &imgrpkg.LeaseRequestStruct{ MountID: globals.mountID, InodeNumber: inodeLockRequest.inodeNumber, @@ -142,13 +165,9 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } leaseResponse = &imgrpkg.LeaseResponseStruct{} - err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) + err = rpcLease(leaseRequest, leaseResponse) if nil != err { - err = renewRPCHandler() - if nil != err { - logFatal(err) - } - goto RetryLeaseRequestTypeExclusive + logFatal(err) } if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeExclusive { @@ -185,11 +204,11 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateSharedGranted: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateSharedPromoting + _ = globals.sharedLeaseLRU.Remove(inodeLease.listElement) + inodeLease.listElement = globals.exclusiveLeaseLRU.PushBack(inodeLease) globals.Unlock() - RetryLeaseRequestTypePromote: - leaseRequest = &imgrpkg.LeaseRequestStruct{ MountID: globals.mountID, InodeNumber: inodeLockRequest.inodeNumber, @@ -197,13 +216,9 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } leaseResponse = &imgrpkg.LeaseResponseStruct{} - err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) + err = rpcLease(leaseRequest, leaseResponse) if nil != err { - err = renewRPCHandler() - if nil != err { - logFatal(err) - } - goto RetryLeaseRequestTypePromote + logFatal(err) } if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypePromoted { @@ -287,11 +302,10 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateNone: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) inodeLease.state = inodeLeaseStateSharedRequested + inodeLease.listElement = globals.sharedLeaseLRU.PushBack(inodeLease) globals.Unlock() - RetryLeaseRequestTypeShared: - leaseRequest = &imgrpkg.LeaseRequestStruct{ MountID: globals.mountID, InodeNumber: inodeLockRequest.inodeNumber, @@ -299,13 +313,9 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } leaseResponse = &imgrpkg.LeaseResponseStruct{} - err = globals.retryRPCClient.Send("Lease", leaseRequest, leaseResponse) + err = rpcLease(leaseRequest, leaseResponse) if nil != err { - err = renewRPCHandler() - if nil != err { - logFatal(err) - } - goto RetryLeaseRequestTypeShared + logFatal(err) } if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeShared { @@ -431,3 +441,23 @@ func (inodeLockRequest *inodeLockRequestStruct) unlockAllWhileLocked() { inodeLockRequest.locksHeld = make(map[uint64]*inodeHeldLockStruct) } + +func lookupInodeLease(inodeNumber uint64) (inodeLease *inodeLeaseStruct) { + globals.Lock() + inodeLease = lookupInodeLeaseWhileLocked(inodeNumber) + globals.Unlock() + return +} + +func lookupInodeLeaseWhileLocked(inodeNumber uint64) (inodeLease *inodeLeaseStruct) { + var ( + ok bool + ) + + inodeLease, ok = globals.inodeLeaseTable[inodeNumber] + if !ok { + inodeLease = nil + } + + return +} diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index d2ea4a8a..b90bc1d8 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "sync" + "time" "github.com/NVIDIA/proxyfs/iauth" "github.com/NVIDIA/proxyfs/imgr/imgrpkg" @@ -82,7 +83,7 @@ func startRPCHandler() (err error) { } mountResponse = &imgrpkg.MountResponseStruct{} - err = globals.retryRPCClient.Send("Mount", mountRequest, mountResponse) + err = rpcMount(mountRequest, mountResponse) if nil != err { return } @@ -107,9 +108,9 @@ func renewRPCHandler() (err error) { } renewMountResponse = &imgrpkg.RenewMountResponseStruct{} - err = globals.retryRPCClient.Send("RenewMount", renewMountRequest, renewMountResponse) + err = rpcRenewMount(renewMountRequest, renewMountResponse) - return // err, as set by globals.retryRPCClient.Send("RenewMount", renewMountRequest, renewMountResponse) is sufficient + return // err, as set by rpcRenewMount(renewMountRequest, renewMountResponse) is sufficient } func stopRPCHandler() (err error) { @@ -123,7 +124,7 @@ func stopRPCHandler() (err error) { } unmountResponse = &imgrpkg.UnmountResponseStruct{} - err = globals.retryRPCClient.Send("Unmount", unmountRequest, unmountResponse) + err = rpcUnmount(unmountRequest, unmountResponse) if nil != err { logWarn(err) } @@ -214,3 +215,170 @@ func updateSwithAuthTokenAndSwiftStorageURL() { globals.swiftAuthWaitGroup = nil globals.Unlock() } + +func performRenewableRPC(method string, request interface{}, reply interface{}) (err error) { +Retry: + + err = globals.retryRPCClient.Send(method, request, reply) + if nil != err { + err = renewRPCHandler() + if nil != err { + logFatal(err) + } + goto Retry + } + + return +} + +func rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performRenewableRPC("AdjustInodeTableEntryOpenCount", adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + + return +} + +func rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest *imgrpkg.DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *imgrpkg.DeleteInodeTableEntryResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performRenewableRPC("DeleteInodeTableEntry", deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) + + return +} + +func rpcFetchNonceRange(fetchNonceRangeRequest *imgrpkg.FetchNonceRangeRequestStruct, fetchNonceRangeResponse *imgrpkg.FetchNonceRangeResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.FetchNonceRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performRenewableRPC("FetchNonceRange", fetchNonceRangeRequest, fetchNonceRangeResponse) + + return +} + +func rpcFlush(flushRequest *imgrpkg.FlushRequestStruct, flushResponse *imgrpkg.FlushResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.FlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performRenewableRPC("Flush", flushRequest, flushResponse) + + return +} + +func rpcGetInodeTableEntry(getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *imgrpkg.GetInodeTableEntryResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performRenewableRPC("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) + + return +} + +func rpcLease(leaseRequest *imgrpkg.LeaseRequestStruct, leaseResponse *imgrpkg.LeaseResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.LeaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performRenewableRPC("Lease", leaseRequest, leaseResponse) + + return +} + +func rpcMount(mountRequest *imgrpkg.MountRequestStruct, mountResponse *imgrpkg.MountResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.MountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = globals.retryRPCClient.Send("Mount", mountRequest, mountResponse) + + return +} + +func rpcPutInodeTableEntries(putInodeTableEntriesRequest *imgrpkg.PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *imgrpkg.PutInodeTableEntriesResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.PutInodeTableEntriesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performRenewableRPC("PutInodeTableEntries", putInodeTableEntriesRequest, putInodeTableEntriesResponse) + + return +} + +func rpcRenewMount(renewMountRequest *imgrpkg.RenewMountRequestStruct, renewMountResponse *imgrpkg.RenewMountResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.DoGetAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = globals.retryRPCClient.Send("RenewMount", renewMountRequest, renewMountResponse) + + return +} + +func rpcUnmount(unmountRequest *imgrpkg.UnmountRequestStruct, unmountResponse *imgrpkg.UnmountResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.UnmountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performRenewableRPC("Unmount", unmountRequest, unmountResponse) + + return +} + +func performObjectGETRange(objectNumber uint64, offset uint64, length uint64) (buf []byte, err error) { + return nil, nil // TODO +} + +func performObjectTAIL(objectNumber uint64, length uint64) (buf []byte, err error) { + return nil, nil // TODO +} + +func performObjectGETWithRangeHeader(objectNumber uint64, rangeHeader string) (buf []byte, err error) { + return nil, nil // TODO +} From d206e62abf3202a6c84fd792f3db94b60bdf1119 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 6 Oct 2021 17:27:19 -0700 Subject: [PATCH 109/258] Got "stat /mnt" to complete... the first FUSE upcall to do everything necessary... --- iclient/dev.conf | 63 ++++++------- iclient/iclient.conf | 63 ++++++------- iclient/iclientpkg/api.go | 65 +++++++------- iclient/iclientpkg/fission.go | 102 +++++++++++++++++---- iclient/iclientpkg/globals.go | 163 +++++++++++++++++++++------------- iclient/iclientpkg/rpc.go | 107 +++++++++++++++++++--- 6 files changed, 386 insertions(+), 177 deletions(-) diff --git a/iclient/dev.conf b/iclient/dev.conf index 13602810..726ab260 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -2,34 +2,37 @@ # SPDX-License-Identifier: Apache-2.0 [ICLIENT] -VolumeName: testvol -MountPointDirPath: /mnt -FUSEAllowOther: true -FUSEMaxBackground: 1000 -FUSECongestionThreshhold: 0 -FUSEMaxWrite: 131076 -AuthPlugInPath: iauth/iauth-swift/iauth-swift.so +VolumeName: testvol +MountPointDirPath: /mnt +FUSEAllowOther: true +FUSEMaxBackground: 1000 +FUSECongestionThreshhold: 0 +FUSEMaxWrite: 131076 +FUSEEntryValidDuration: 0s +FUSEAttrValidDuration: 0s +AuthPlugInPath: iauth/iauth-swift/iauth-swift.so AuthPlugInEnvName: -AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} -SwiftRetryDelay: 100ms -SwiftRetryExpBackoff: 2 -SwiftRetryLimit: 4 -SwiftTimeout: 10m -SwiftConnectionPoolSize: 128 -RetryRPCPublicIPAddr: dev -RetryRPCPort: 32356 -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCACertFilePath: # Defaults to /dev/null -MaxSharedLeases: 500 -MaxExclusiveLeases: 100 -ReadCacheLineSize: 1048576 -ReadCacheLineCountMax: 1024 -FileFlushTriggerSize: 10485760 -FileFlushTriggerDuration: 10s -LogFilePath: # Defaults to "" -LogToConsole: true -TraceEnabled: false -FUSELogEnabled: false -HTTPServerIPAddr: dev -HTTPServerPort: 15347 +AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +SwiftTimeout: 10m +SwiftRetryLimit: 4 +SwiftRetryDelay: 100ms +SwiftRetryDelayVariance: 25 +SwiftRetryExponentialBackoff: 1.4 +SwiftConnectionPoolSize: 128 +RetryRPCPublicIPAddr: dev +RetryRPCPort: 32356 +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s +RetryRPCCACertFilePath: +MaxSharedLeases: 500 +MaxExclusiveLeases: 100 +ReadCacheLineSize: 1048576 +ReadCacheLineCountMax: 1024 +FileFlushTriggerSize: 10485760 +FileFlushTriggerDuration: 10s +LogFilePath: +LogToConsole: true +TraceEnabled: false +FUSELogEnabled: false +HTTPServerIPAddr: dev +HTTPServerPort: 15347 diff --git a/iclient/iclient.conf b/iclient/iclient.conf index e03eb1df..42c46a87 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -2,34 +2,37 @@ # SPDX-License-Identifier: Apache-2.0 [ICLIENT] -VolumeName: testvol -MountPointDirPath: /mnt -FUSEAllowOther: true -FUSEMaxBackground: 1000 -FUSECongestionThreshhold: 0 -FUSEMaxWrite: 131076 -AuthPlugInPath: iauth-swift.so +VolumeName: testvol +MountPointDirPath: /mnt +FUSEAllowOther: true +FUSEMaxBackground: 1000 +FUSECongestionThreshhold: 0 +FUSEMaxWrite: 131076 +FUSEEntryValidDuration: 0s +FUSEAttrValidDuration: 0s +AuthPlugInPath: iauth-swift.so AuthPlugInEnvName: -AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} -SwiftRetryDelay: 100ms -SwiftRetryExpBackoff: 2 -SwiftRetryLimit: 4 -SwiftTimeout: 10m -SwiftConnectionPoolSize: 128 -RetryRPCPublicIPAddr: imgr -RetryRPCPort: 32356 -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCACertFilePath: caCert.pem -MaxSharedLeases: 500 -MaxExclusiveLeases: 100 -ReadCacheLineSize: 1048576 -ReadCacheLineCountMax: 1024 -FileFlushTriggerSize: 10485760 -FileFlushTriggerDuration: 10s -LogFilePath: # Defaults to "" -LogToConsole: true -TraceEnabled: false -FUSELogEnabled: false -HTTPServerIPAddr: iclient -HTTPServerPort: 15347 +AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +SwiftTimeout: 10m +SwiftRetryLimit: 4 +SwiftRetryDelay: 100ms +SwiftRetryDelayVariance: 25 +SwiftRetryExponentialBackoff: 1.4 +SwiftConnectionPoolSize: 128 +RetryRPCPublicIPAddr: imgr +RetryRPCPort: 32356 +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s +RetryRPCCACertFilePath: caCert.pem +MaxSharedLeases: 500 +MaxExclusiveLeases: 100 +ReadCacheLineSize: 1048576 +ReadCacheLineCountMax: 1024 +FileFlushTriggerSize: 10485760 +FileFlushTriggerDuration: 10s +LogFilePath: +LogToConsole: true +TraceEnabled: false +FUSELogEnabled: false +HTTPServerIPAddr: iclient +HTTPServerPort: 15347 diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index f5e242b5..035c8389 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -8,37 +8,40 @@ // argument, a package conf ConfMap. Here is a sample .conf file: // // [ICLIENT] -// VolumeName: testvol -// MountPointDirPath: /mnt -// FUSEAllowOther: true -// FUSEMaxBackground: 1000 -// FUSECongestionThreshhold: 0 -// FUSEMaxWrite: 131076 -// AuthPlugInPath: iauth-swift.so -// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here -// AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} -// SwiftRetryDelay: 100ms -// SwiftRetryExpBackoff: 2 -// SwiftRetryLimit: 4 -// SwiftTimeout: 10m -// SwiftConnectionPoolSize: 128 -// RetryRPCPublicIPAddr: imgr -// RetryRPCPort: 32356 -// RetryRPCDeadlineIO: 60s -// RetryRPCKeepAlivePeriod: 60s -// RetryRPCCACertFilePath: # Defaults to /dev/null -// MaxSharedLeases: 500 -// MaxExclusiveLeases: 100 # Caps pending FileFlush data at 1GiB (with FileFlushTriggerSize of 10MiB) -// ReadCacheLineSize: 1048576 # 1MiB -// ReadCacheLineCountMax: 1024 # 1GiB (with ReadCacheLineSize of 1MiB) -// FileFlushTriggerSize: 10485760 # [10MiB] Amount of written data before metadata is appended and flush is triggered -// FileFlushTriggerDuration: 10s # Amount of time before unwritten data and its metadata flush is triggered -// LogFilePath: iclient.log -// LogToConsole: true -// TraceEnabled: false -// FUSELogEnabled: false -// HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) -// HTTPServerPort: # Defaults to disabling the embedded HTTP Server +// VolumeName: testvol +// MountPointDirPath: /mnt +// FUSEAllowOther: true +// FUSEMaxBackground: 1000 +// FUSECongestionThreshhold: 0 +// FUSEMaxWrite: 131076 +// FUSEEntryValidDuration: 0s +// FUSEAttrValidDuration: 0s +// AuthPlugInPath: iauth-swift.so +// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here +// AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +// SwiftTimeout: 10m +// SwiftRetryLimit: 4 +// SwiftRetryDelay: 100ms +// SwiftRetryDelayVariance: 25 # Percentage (1-100) of SwitRetryDelay +// SwiftRetryExponentialBackoff: 1.4 +// SwiftConnectionPoolSize: 128 +// RetryRPCPublicIPAddr: imgr +// RetryRPCPort: 32356 +// RetryRPCDeadlineIO: 60s +// RetryRPCKeepAlivePeriod: 60s +// RetryRPCCACertFilePath: # Defaults to /dev/null +// MaxSharedLeases: 500 +// MaxExclusiveLeases: 100 # Caps pending FileFlush data at 1GiB (with FileFlushTriggerSize of 10MiB) +// ReadCacheLineSize: 1048576 # 1MiB +// ReadCacheLineCountMax: 1024 # 1GiB (with ReadCacheLineSize of 1MiB) +// FileFlushTriggerSize: 10485760 # [10MiB] Amount of written data before metadata is appended and flush is triggered +// FileFlushTriggerDuration: 10s # Amount of time before unwritten data and its metadata flush is triggered +// LogFilePath: iclient.log +// LogToConsole: true +// TraceEnabled: false +// FUSELogEnabled: false +// HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) +// HTTPServerPort: # Defaults to disabling the embedded HTTP Server // // Most of the config keys are required and must have values. One set of exceptions // are the HTTPServer{IPAddr|Port} keys that, if not present (or HTTPServerPort is diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index c6bfdf88..76309db2 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -4,7 +4,6 @@ package iclientpkg import ( - "fmt" "syscall" "time" @@ -16,6 +15,7 @@ import ( const ( attrBlockSize = uint32(512) + attrRDev = uint32(0) fuseSubtype = "ProxyFS" @@ -44,6 +44,9 @@ func performMountFUSE() (err error) { globals.fissionErrChan, ) + globals.fuseEntryValidDurationSec, globals.fuseEntryValidDurationNSec = nsToUnixTime(uint64(globals.config.FUSEEntryValidDuration)) + globals.fuseAttrValidDurationSec, globals.fuseAttrValidDurationNSec = nsToUnixTime(uint64(globals.config.FUSEAttrValidDuration)) + err = globals.fissionVolume.DoMount() return @@ -54,6 +57,9 @@ func performUnmountFUSE() (err error) { globals.fissionVolume = nil + globals.fuseEntryValidDurationSec, globals.fuseEntryValidDurationNSec = 0, 0 + globals.fuseAttrValidDurationSec, globals.fuseAttrValidDurationNSec = 0, 0 + return } @@ -100,9 +106,14 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis err error getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct getInodeTableEntryResponse *imgrpkg.GetInodeTableEntryResponseStruct + inodeHeadV1Buf []byte inodeLease *inodeLeaseStruct inodeLockRequest *inodeLockRequestStruct + modificationTimeNSec uint32 + modificationTimeSec uint64 startTime time.Time = time.Now() + statusChangeTimeNSec uint32 + statusChangeTimeSec uint64 ) logTracef("==> DoGetAttr(inHeader: %+v, getAttrIn: %+v)", inHeader, getAttrIn) @@ -142,13 +153,49 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis return } - fmt.Printf("TODO: need to fetch .inodeHeadV1 via getInodeTableEntryResponse: %+v\n", getInodeTableEntryResponse) - inodeLease.inodeHeadV1 = &ilayout.InodeHeadV1Struct{} + inodeHeadV1Buf, err = objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength) + if nil != err { + logFatalf("objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber: %v, getInodeTableEntryResponse.InodeHeadLength: %v) failed: %v", getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength, err) + } + + inodeLease.inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) + if nil != err { + logFatalf("ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) failed: %v", err) + } + } + + modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inodeLease.inodeHeadV1.ModificationTime.UnixNano())) + statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(inodeLease.inodeHeadV1.StatusChangeTime.UnixNano())) + + getAttrOut = &fission.GetAttrOut{ + AttrValidSec: globals.fuseAttrValidDurationSec, + AttrValidNSec: globals.fuseAttrValidDurationNSec, + Dummy: 0, + Attr: fission.Attr{ + Ino: inodeLease.inodeHeadV1.InodeNumber, + Size: inodeLease.inodeHeadV1.Size, // Possibly overwritten by fixAttrSizes() + Blocks: 0, // Computed by fixAttrSizes() + ATimeSec: modificationTimeSec, + MTimeSec: modificationTimeSec, + CTimeSec: statusChangeTimeSec, + ATimeNSec: modificationTimeNSec, + MTimeNSec: modificationTimeNSec, + CTimeNSec: statusChangeTimeNSec, + Mode: computeAttrMode(inodeLease.inodeHeadV1.InodeType, inodeLease.inodeHeadV1.Mode), + NLink: uint32(len(inodeLease.inodeHeadV1.LinkTable)), + UID: uint32(inodeLease.inodeHeadV1.UserID), + GID: uint32(inodeLease.inodeHeadV1.GroupID), + RDev: attrRDev, + BlkSize: attrBlockSize, // Possibly overwritten by fixAttrSizes() + Padding: 0, + }, } inodeLockRequest.unlockAll() - getAttrOut = nil - errno = syscall.ENOSYS + + fixAttrSizes(&getAttrOut.Attr) + + errno = 0 return } @@ -921,15 +968,40 @@ func (dummy *globalsStruct) DoLSeek(inHeader *fission.InHeader, lSeekIn *fission return } +func nsToUnixTime(ns uint64) (sec uint64, nsec uint32) { + sec = ns / 1e9 + nsec = uint32(ns - (sec * 1e9)) + return +} + +func unixTimeToNs(sec uint64, nsec uint32) (ns uint64) { + ns = (sec * 1e9) + uint64(nsec) + return +} + +func computeAttrMode(iLayoutInodeType uint8, iLayoutMode uint16) (attrMode uint32) { + attrMode = uint32(iLayoutMode) + switch iLayoutInodeType { + case ilayout.InodeTypeDir: + attrMode |= syscall.S_IFDIR + case ilayout.InodeTypeFile: + attrMode |= syscall.S_IFREG + case ilayout.InodeTypeSymLink: + attrMode |= syscall.S_IFLNK + default: + logFatalf("iLayoutInodeType (%v) unknown", iLayoutInodeType) + } + return +} + func fixAttrSizes(attr *fission.Attr) { - // TODO - // if syscall.S_IFREG == (attr.Mode & syscall.S_IFMT) { - // attr.Blocks = attr.Size + (uint64(attrBlkSize) - 1) - // attr.Blocks /= uint64(attrBlkSize) - // attr.BlkSize = attrBlkSize - // } else { - // attr.Size = 0 - // attr.Blocks = 0 - // attr.BlkSize = 0 - // } + if syscall.S_IFREG == (attr.Mode & syscall.S_IFMT) { + attr.Blocks = attr.Size + (uint64(attrBlockSize) - 1) + attr.Blocks /= uint64(attrBlockSize) + attr.BlkSize = attrBlockSize + } else { + attr.Size = 0 + attr.Blocks = 0 + attr.BlkSize = 0 + } } diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 20a2924b..0e56c4a1 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -21,37 +21,45 @@ import ( ) type configStruct struct { - VolumeName string - MountPointDirPath string - FUSEAllowOther bool - FUSEMaxBackground uint16 - FUSECongestionThreshhold uint16 - FUSEMaxWrite uint32 - AuthPlugInPath string - AuthPlugInEnvName string - AuthPlugInEnvValue string - SwiftRetryDelay time.Duration - SwiftRetryExpBackoff float64 - SwiftRetryLimit uint32 - SwiftTimeout time.Duration - SwiftConnectionPoolSize uint32 - RetryRPCPublicIPAddr string - RetryRPCPort uint16 - RetryRPCDeadlineIO time.Duration - RetryRPCKeepAlivePeriod time.Duration - RetryRPCCACertFilePath string // Defaults to /dev/null - MaxSharedLeases uint64 - MaxExclusiveLeases uint64 - ReadCacheLineSize uint64 - ReadCacheLineCountMax uint64 - FileFlushTriggerSize uint64 - FileFlushTriggerDuration time.Duration - LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled - LogToConsole bool - TraceEnabled bool - FUSELogEnabled bool - HTTPServerIPAddr string - HTTPServerPort uint16 // To be served on HTTPServerIPAddr via TCP + VolumeName string + MountPointDirPath string + FUSEAllowOther bool + FUSEMaxBackground uint16 + FUSECongestionThreshhold uint16 + FUSEMaxWrite uint32 + FUSEEntryValidDuration time.Duration + FUSEAttrValidDuration time.Duration + AuthPlugInPath string + AuthPlugInEnvName string + AuthPlugInEnvValue string + SwiftTimeout time.Duration + SwiftRetryLimit uint64 + SwiftRetryDelay time.Duration + SwiftRetryDelayVariance uint8 + SwiftRetryExponentialBackoff float64 + SwiftConnectionPoolSize uint32 + RetryRPCPublicIPAddr string + RetryRPCPort uint16 + RetryRPCDeadlineIO time.Duration + RetryRPCKeepAlivePeriod time.Duration + RetryRPCCACertFilePath string // Defaults to /dev/null + MaxSharedLeases uint64 + MaxExclusiveLeases uint64 + ReadCacheLineSize uint64 + ReadCacheLineCountMax uint64 + FileFlushTriggerSize uint64 + FileFlushTriggerDuration time.Duration + LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled + LogToConsole bool + TraceEnabled bool + FUSELogEnabled bool + HTTPServerIPAddr string + HTTPServerPort uint16 // To be served on HTTPServerIPAddr via TCP +} + +type swiftRetryDelayElementStruct struct { + nominal time.Duration + variance time.Duration } type inodeLeaseStateType uint32 @@ -161,28 +169,33 @@ type statsStruct struct { } type globalsStruct struct { - sync.Mutex // Serializes access to inodeLeaseTable - config configStruct // - logFile *os.File // == nil if config.LogFilePath == "" - retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" - httpClient *http.Client // - swiftAuthInString string // - swiftAuthWaitGroup *sync.WaitGroup // != nil if updateAuthTokenAndStorageURL() is active - swiftAuthToken string // - swiftStorageURL string // - retryRPCClientConfig *retryrpc.ClientConfig // - retryRPCClient *retryrpc.Client // - mountID string // - fissionErrChan chan error // - inodeLeaseTable map[uint64]*inodeLeaseStruct // - inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits - inodeLockWG sync.WaitGroup // Signaled as each active inodeLockRequestStruct has signaled service complete with .locksHeld empty - sharedLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted - exclusiveLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted - httpServer *http.Server // - httpServerWG sync.WaitGroup // - stats *statsStruct // - fissionVolume fission.Volume // + sync.Mutex // Serializes access to inodeLeaseTable + config configStruct // + fuseEntryValidDurationSec uint64 // + fuseEntryValidDurationNSec uint32 // + fuseAttrValidDurationSec uint64 // + fuseAttrValidDurationNSec uint32 // + logFile *os.File // == nil if config.LogFilePath == "" + retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" + httpClient *http.Client // + swiftRetryDelay []swiftRetryDelayElementStruct // + swiftAuthInString string // + swiftAuthWaitGroup *sync.WaitGroup // != nil if updateAuthTokenAndStorageURL() is active + swiftAuthToken string // + swiftStorageURL string // + retryRPCClientConfig *retryrpc.ClientConfig // + retryRPCClient *retryrpc.Client // + mountID string // + fissionErrChan chan error // + inodeLeaseTable map[uint64]*inodeLeaseStruct // + inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits + inodeLockWG sync.WaitGroup // Signaled as each active inodeLockRequestStruct has signaled service complete with .locksHeld empty + sharedLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted + exclusiveLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted + httpServer *http.Server // + httpServerWG sync.WaitGroup // + stats *statsStruct // + fissionVolume fission.Volume // } var globals globalsStruct @@ -224,6 +237,14 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err if nil != err { logFatal(err) } + globals.config.FUSEEntryValidDuration, err = confMap.FetchOptionValueDuration("ICLIENT", "FUSEEntryValidDuration") + if nil != err { + logFatal(err) + } + globals.config.FUSEAttrValidDuration, err = confMap.FetchOptionValueDuration("ICLIENT", "FUSEAttrValidDuration") + if nil != err { + logFatal(err) + } globals.config.AuthPlugInPath, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInPath") if nil != err { logFatal(err) @@ -261,19 +282,30 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err } } } - globals.config.SwiftRetryDelay, err = confMap.FetchOptionValueDuration("ICLIENT", "SwiftRetryDelay") + globals.config.SwiftTimeout, err = confMap.FetchOptionValueDuration("ICLIENT", "SwiftTimeout") if nil != err { logFatal(err) } - globals.config.SwiftRetryExpBackoff, err = confMap.FetchOptionValueFloat64("ICLIENT", "SwiftRetryExpBackoff") + globals.config.SwiftRetryLimit, err = confMap.FetchOptionValueUint64("ICLIENT", "SwiftRetryLimit") if nil != err { logFatal(err) } - globals.config.SwiftRetryLimit, err = confMap.FetchOptionValueUint32("ICLIENT", "SwiftRetryLimit") + globals.config.SwiftRetryDelay, err = confMap.FetchOptionValueDuration("ICLIENT", "SwiftRetryDelay") if nil != err { logFatal(err) } - globals.config.SwiftTimeout, err = confMap.FetchOptionValueDuration("ICLIENT", "SwiftTimeout") + globals.config.SwiftRetryDelayVariance, err = confMap.FetchOptionValueUint8("ICLIENT", "SwiftRetryDelayVariance") + if nil == err { + if 0 == globals.config.SwiftRetryDelayVariance { + logFatalf("[Agent]SwiftRetryDelayVariance must be > 0") + } + if 100 < globals.config.SwiftRetryDelayVariance { + logFatalf("[Agent]SwiftRetryDelayVariance (%v) must be <= 100", globals.config.SwiftRetryDelayVariance) + } + } else { + logFatal(err) + } + globals.config.SwiftRetryExponentialBackoff, err = confMap.FetchOptionValueFloat64("ICLIENT", "SwiftRetryExponentialBackoff") if nil != err { logFatal(err) } @@ -359,6 +391,9 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err logInfof("globals.config:\n%s", configJSONified) + globals.fuseEntryValidDurationSec, globals.fuseEntryValidDurationNSec = nsToUnixTime(uint64(globals.config.FUSEEntryValidDuration)) + globals.fuseAttrValidDurationSec, globals.fuseAttrValidDurationNSec = nsToUnixTime(uint64(globals.config.FUSEAttrValidDuration)) + if "" == globals.config.RetryRPCCACertFilePath { globals.retryRPCCACertPEM = nil } else { @@ -389,13 +424,16 @@ func uninitializeGlobals() (err error) { globals.config.FUSEMaxBackground = 0 globals.config.FUSECongestionThreshhold = 0 globals.config.FUSEMaxWrite = 0 + globals.config.FUSEEntryValidDuration = time.Duration(0) + globals.config.FUSEAttrValidDuration = time.Duration(0) globals.config.AuthPlugInPath = "" globals.config.AuthPlugInEnvName = "" globals.config.AuthPlugInEnvValue = "" - globals.config.SwiftRetryDelay = time.Duration(0) - globals.config.SwiftRetryExpBackoff = 0.0 - globals.config.SwiftRetryLimit = 0 globals.config.SwiftTimeout = time.Duration(0) + globals.config.SwiftRetryLimit = 0 + globals.config.SwiftRetryDelay = time.Duration(0) + globals.config.SwiftRetryDelayVariance = 0 + globals.config.SwiftRetryExponentialBackoff = 0.0 globals.config.SwiftConnectionPoolSize = 0 globals.config.RetryRPCPublicIPAddr = "" globals.config.RetryRPCPort = 0 @@ -415,6 +453,11 @@ func uninitializeGlobals() (err error) { globals.config.HTTPServerIPAddr = "" globals.config.HTTPServerPort = 0 + globals.fuseEntryValidDurationSec = 0 + globals.fuseEntryValidDurationNSec = 0 + globals.fuseAttrValidDurationSec = 0 + globals.fuseAttrValidDurationNSec = 0 + globals.retryRPCCACertPEM = nil globals.inodeLeaseTable = nil diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index b90bc1d8..226803dd 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -5,23 +5,33 @@ package iclientpkg import ( "fmt" + "io/ioutil" + "math/rand" "net/http" "os" + "strconv" "sync" "time" "github.com/NVIDIA/proxyfs/iauth" + "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/imgr/imgrpkg" "github.com/NVIDIA/proxyfs/retryrpc" ) +const ( + HTTPUserAgent = "iclient" +) + func startRPCHandler() (err error) { var ( - customTransport *http.Transport - defaultTransport *http.Transport - mountRequest *imgrpkg.MountRequestStruct - mountResponse *imgrpkg.MountResponseStruct - ok bool + customTransport *http.Transport + defaultTransport *http.Transport + mountRequest *imgrpkg.MountRequestStruct + mountResponse *imgrpkg.MountResponseStruct + nextSwiftRetryDelayNominal time.Duration + ok bool + swiftRetryDelayIndex uint64 ) defaultTransport, ok = http.DefaultTransport.(*http.Transport) @@ -55,6 +65,15 @@ func startRPCHandler() (err error) { Timeout: globals.config.SwiftTimeout, } + globals.swiftRetryDelay = make([]swiftRetryDelayElementStruct, globals.config.SwiftRetryLimit) + + nextSwiftRetryDelayNominal = globals.config.SwiftRetryDelay + + for swiftRetryDelayIndex = 0; swiftRetryDelayIndex < globals.config.SwiftRetryLimit; swiftRetryDelayIndex++ { + globals.swiftRetryDelay[swiftRetryDelayIndex].nominal = nextSwiftRetryDelayNominal + globals.swiftRetryDelay[swiftRetryDelayIndex].variance = nextSwiftRetryDelayNominal * time.Duration(globals.config.SwiftRetryDelayVariance) / time.Duration(100) + } + if globals.config.AuthPlugInEnvName == "" { globals.swiftAuthInString = globals.config.AuthPlugInEnvValue } else { @@ -133,6 +152,8 @@ func stopRPCHandler() (err error) { globals.httpClient = nil + globals.swiftRetryDelay = nil + globals.retryRPCClientConfig = nil globals.retryRPCClient = nil @@ -371,14 +392,78 @@ func rpcUnmount(unmountRequest *imgrpkg.UnmountRequestStruct, unmountResponse *i return } -func performObjectGETRange(objectNumber uint64, offset uint64, length uint64) (buf []byte, err error) { - return nil, nil // TODO +func objectGETRange(objectNumber uint64, offset uint64, length uint64) (buf []byte, err error) { + var ( + rangeHeader string + ) + + rangeHeader = "bytes=" + strconv.FormatUint(offset, 10) + "-" + strconv.FormatUint((offset+length-1), 10) + + buf, err = objectGETWithRangeHeader(objectNumber, rangeHeader) + + return } -func performObjectTAIL(objectNumber uint64, length uint64) (buf []byte, err error) { - return nil, nil // TODO +func objectGETTail(objectNumber uint64, length uint64) (buf []byte, err error) { + var ( + rangeHeader string + ) + + rangeHeader = "bytes=-" + strconv.FormatUint(length, 10) + + buf, err = objectGETWithRangeHeader(objectNumber, rangeHeader) + + return } -func performObjectGETWithRangeHeader(objectNumber uint64, rangeHeader string) (buf []byte, err error) { - return nil, nil // TODO +func objectGETWithRangeHeader(objectNumber uint64, rangeHeader string) (buf []byte, err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + retryDelay time.Duration + retryIndex uint64 + ) + + retryIndex = 0 + + for { + httpRequest, err = http.NewRequest("GET", fetchSwiftStorageURL()+"/"+ilayout.GetObjectNameAsString(objectNumber), nil) + if nil != err { + return + } + + httpRequest.Header["User-Agent"] = []string{HTTPUserAgent} + httpRequest.Header["X-Auth-Token"] = []string{fetchSwiftAuthToken()} + httpRequest.Header["Range"] = []string{rangeHeader} + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + logFatalf("globals.httpClient.Do(httpRequest) failed: %v", err) + } + + buf, err = ioutil.ReadAll(httpResponse.Body) + _ = httpResponse.Body.Close() + if nil != err { + logFatalf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + err = nil + return + } + + if retryIndex >= globals.config.SwiftRetryLimit { + err = fmt.Errorf("objectGETWithRangeHeader(objectNumber: %v, rangeHeader: \"%s\") reached SwiftRetryLimit", objectNumber, rangeHeader) + logWarn(err) + return + } + + if http.StatusUnauthorized == httpResponse.StatusCode { + updateSwithAuthTokenAndSwiftStorageURL() + } + + retryDelay = globals.swiftRetryDelay[retryIndex].nominal - time.Duration(rand.Int63n(int64(globals.swiftRetryDelay[retryIndex].variance))) + time.Sleep(retryDelay) + retryIndex++ + } } From 083c3ccf6a7f745bc03f04b4d2fa4b8a036d8548 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 6 Oct 2021 17:42:16 -0700 Subject: [PATCH 110/258] Workaround hang during iclient shutdown due to RPC Unmount returning an ETODO --- iclient/iclientpkg/rpc.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index 226803dd..2455d942 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -387,7 +387,9 @@ func rpcUnmount(unmountRequest *imgrpkg.UnmountRequestStruct, unmountResponse *i globals.stats.UnmountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - err = performRenewableRPC("Unmount", unmountRequest, unmountResponse) + // TODO: Since the Unmount RPC fails with an ETODO, we would mistakenly try to reauth + // err = performRenewableRPC("Unmount", unmountRequest, unmountResponse) + err = globals.retryRPCClient.Send("Unmount", unmountRequest, unmountResponse) return } From 19bf77fe875cbc8d3574b7b8a0731968c114699c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 7 Oct 2021 09:46:01 -0700 Subject: [PATCH 111/258] Fixed check for when to attempt a re-auth/RenewMount instead of just on any error ...also added framework and initial code for the rest of inodeLeaseStruct operations --- iclient/iclientpkg/globals.go | 18 +++- iclient/iclientpkg/inode.go | 172 ++++++++++++++++++++++++++++++++++ iclient/iclientpkg/lease.go | 16 ++-- iclient/iclientpkg/rpc.go | 58 +++++------- 4 files changed, 223 insertions(+), 41 deletions(-) create mode 100644 iclient/iclientpkg/inode.go diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 0e56c4a1..f84dfced 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -13,6 +13,8 @@ import ( "github.com/NVIDIA/fission" + "github.com/NVIDIA/sortedmap" + "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/conf" "github.com/NVIDIA/proxyfs/ilayout" @@ -78,13 +80,23 @@ const ( inodeLeaseStateExclusiveExpired ) +type layoutMapEntryStruct struct { + objectSize uint64 + bytesReferenced uint64 +} + type inodeLeaseStruct struct { inodeNumber uint64 // state inodeLeaseStateType // - listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU - heldList *list.List // List of granted inodeHeldLockStruct's - requestList *list.List // List of pending inodeLockRequestStruct's + listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU + heldList *list.List // List of granted inodeHeldLockStruct's + requestList *list.List // List of pending inodeLockRequestStruct's inodeHeadV1 *ilayout.InodeHeadV1Struct // + payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} + // For FileInode: ExtentMap B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} + layoutMap map[uint64]layoutMapEntryStruct // For DirInode & FileInode: Map form of .inodeHeadV1.Layout + putObjectNumber uint64 // For DirInode & FileInode: + putObjectBuffer []byte // ObjectNumber and buffer to PUT during next flush } type inodeHeldLockStruct struct { diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go new file mode 100644 index 00000000..786ddc7a --- /dev/null +++ b/iclient/iclientpkg/inode.go @@ -0,0 +1,172 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +import ( + "encoding/json" + "fmt" + + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/ilayout" +) + +func (inodeLease *inodeLeaseStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + var ( + fileOffset uint64 + ok bool + ) + + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + keyAsString, ok = key.(string) + if ok { + err = nil + } else { + err = fmt.Errorf("key.(string) returned !ok") + } + case ilayout.InodeTypeFile: + fileOffset, ok = key.(uint64) + if ok { + keyAsString = fmt.Sprintf("%016X", fileOffset) + err = nil + } else { + err = fmt.Errorf("key.(uint64) returned !ok") + } + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + return +} + +func (inodeLease *inodeLeaseStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + var ( + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1JSON []byte + extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct + extentMapEntryValueV1JSON []byte + ok bool + ) + + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + directoryEntryValueV1, ok = value.(*ilayout.DirectoryEntryValueV1Struct) + if ok { + directoryEntryValueV1JSON, err = json.Marshal(directoryEntryValueV1) + if nil == err { + valueAsString = string(directoryEntryValueV1JSON[:]) + err = nil + } else { + err = fmt.Errorf("json.Marshal(directoryEntryValueV1) failed: %v", err) + } + } else { + err = fmt.Errorf("value.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + case ilayout.InodeTypeFile: + extentMapEntryValueV1, ok = value.(*ilayout.ExtentMapEntryValueV1Struct) + if ok { + extentMapEntryValueV1JSON, err = json.Marshal(extentMapEntryValueV1) + if nil == err { + valueAsString = string(extentMapEntryValueV1JSON[:]) + err = nil + } else { + err = fmt.Errorf("json.Marshal(extentMapEntryValueV1) failed: %v", err) + } + } else { + err = fmt.Errorf("value.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") + } + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + return +} + +func (inodeLease *inodeLeaseStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + case ilayout.InodeTypeFile: + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + if objectNumber == inodeLease.putObjectNumber { + if (objectOffset + objectLength) <= uint64(len(inodeLease.putObjectBuffer)) { + nodeByteSlice = make([]byte, objectLength) + _ = copy(nodeByteSlice, inodeLease.putObjectBuffer[objectOffset:(objectOffset+objectLength)]) + err = nil + } else { + err = fmt.Errorf("(objectOffset + objectLength) > uint64(len(inodeLease.putObjectBuffer))") + } + } else { + nodeByteSlice, err = objectGETRange(objectNumber, objectOffset, objectLength) + + if (nil == err) && (uint64(len(nodeByteSlice)) < objectLength) { + err = fmt.Errorf("uint64(len(nodeByteSlice)) < objectLength") + } + } + + return +} + +func (inodeLease *inodeLeaseStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + case ilayout.InodeTypeFile: + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + return // TODO +} + +func (inodeLease *inodeLeaseStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + case ilayout.InodeTypeFile: + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + return // TODO +} + +func (inodeLease *inodeLeaseStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + case ilayout.InodeTypeFile: + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + return // TODO +} + +func (inodeLease *inodeLeaseStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + case ilayout.InodeTypeFile: + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + return // TODO +} + +func (inodeLease *inodeLeaseStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + case ilayout.InodeTypeFile: + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + return // TODO +} + +func (inodeLease *inodeLeaseStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + case ilayout.InodeTypeFile: + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + return // TODO +} diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index d458f59d..3eaab1a7 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -89,12 +89,16 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } } else { inodeLease = &inodeLeaseStruct{ - inodeNumber: inodeLockRequest.inodeNumber, - state: inodeLeaseStateNone, - listElement: nil, - heldList: list.New(), - requestList: list.New(), - inodeHeadV1: nil, + inodeNumber: inodeLockRequest.inodeNumber, + state: inodeLeaseStateNone, + listElement: nil, + heldList: list.New(), + requestList: list.New(), + inodeHeadV1: nil, + payload: nil, + layoutMap: nil, + putObjectNumber: 0, + putObjectBuffer: nil, } globals.inodeLeaseTable[inodeLockRequest.inodeNumber] = inodeLease diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index 2455d942..7e810180 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "strconv" + "strings" "sync" "time" @@ -113,25 +114,6 @@ func startRPCHandler() (err error) { return } -func renewRPCHandler() (err error) { - var ( - renewMountRequest *imgrpkg.RenewMountRequestStruct - renewMountResponse *imgrpkg.RenewMountResponseStruct - ) - - updateSwithAuthTokenAndSwiftStorageURL() - - renewMountRequest = &imgrpkg.RenewMountRequestStruct{ - MountID: globals.mountID, - AuthToken: fetchSwiftAuthToken(), - } - renewMountResponse = &imgrpkg.RenewMountResponseStruct{} - - err = rpcRenewMount(renewMountRequest, renewMountResponse) - - return // err, as set by rpcRenewMount(renewMountRequest, renewMountResponse) is sufficient -} - func stopRPCHandler() (err error) { var ( unmountRequest *imgrpkg.UnmountRequestStruct @@ -237,15 +219,29 @@ func updateSwithAuthTokenAndSwiftStorageURL() { globals.Unlock() } -func performRenewableRPC(method string, request interface{}, reply interface{}) (err error) { +func performMountRenewableRPC(method string, request interface{}, reply interface{}) (err error) { + var ( + renewMountRequest *imgrpkg.RenewMountRequestStruct + renewMountResponse *imgrpkg.RenewMountResponseStruct + ) + Retry: err = globals.retryRPCClient.Send(method, request, reply) - if nil != err { - err = renewRPCHandler() + if (nil != err) && (strings.HasPrefix(err.Error(), imgrpkg.EAuthTokenRejected)) { + updateSwithAuthTokenAndSwiftStorageURL() + + renewMountRequest = &imgrpkg.RenewMountRequestStruct{ + MountID: globals.mountID, + AuthToken: fetchSwiftAuthToken(), + } + renewMountResponse = &imgrpkg.RenewMountResponseStruct{} + + err = rpcRenewMount(renewMountRequest, renewMountResponse) if nil != err { logFatal(err) } + goto Retry } @@ -261,7 +257,7 @@ func rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *im globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - err = performRenewableRPC("AdjustInodeTableEntryOpenCount", adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + err = performMountRenewableRPC("AdjustInodeTableEntryOpenCount", adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) return } @@ -275,7 +271,7 @@ func rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest *imgrpkg.DeleteInodeT globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - err = performRenewableRPC("DeleteInodeTableEntry", deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) + err = performMountRenewableRPC("DeleteInodeTableEntry", deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) return } @@ -289,7 +285,7 @@ func rpcFetchNonceRange(fetchNonceRangeRequest *imgrpkg.FetchNonceRangeRequestSt globals.stats.FetchNonceRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - err = performRenewableRPC("FetchNonceRange", fetchNonceRangeRequest, fetchNonceRangeResponse) + err = performMountRenewableRPC("FetchNonceRange", fetchNonceRangeRequest, fetchNonceRangeResponse) return } @@ -303,7 +299,7 @@ func rpcFlush(flushRequest *imgrpkg.FlushRequestStruct, flushResponse *imgrpkg.F globals.stats.FlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - err = performRenewableRPC("Flush", flushRequest, flushResponse) + err = performMountRenewableRPC("Flush", flushRequest, flushResponse) return } @@ -317,7 +313,7 @@ func rpcGetInodeTableEntry(getInodeTableEntryRequest *imgrpkg.GetInodeTableEntry globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - err = performRenewableRPC("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) + err = performMountRenewableRPC("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) return } @@ -331,7 +327,7 @@ func rpcLease(leaseRequest *imgrpkg.LeaseRequestStruct, leaseResponse *imgrpkg.L globals.stats.LeaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - err = performRenewableRPC("Lease", leaseRequest, leaseResponse) + err = performMountRenewableRPC("Lease", leaseRequest, leaseResponse) return } @@ -359,7 +355,7 @@ func rpcPutInodeTableEntries(putInodeTableEntriesRequest *imgrpkg.PutInodeTableE globals.stats.PutInodeTableEntriesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - err = performRenewableRPC("PutInodeTableEntries", putInodeTableEntriesRequest, putInodeTableEntriesResponse) + err = performMountRenewableRPC("PutInodeTableEntries", putInodeTableEntriesRequest, putInodeTableEntriesResponse) return } @@ -387,9 +383,7 @@ func rpcUnmount(unmountRequest *imgrpkg.UnmountRequestStruct, unmountResponse *i globals.stats.UnmountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO: Since the Unmount RPC fails with an ETODO, we would mistakenly try to reauth - // err = performRenewableRPC("Unmount", unmountRequest, unmountResponse) - err = globals.retryRPCClient.Send("Unmount", unmountRequest, unmountResponse) + err = performMountRenewableRPC("Unmount", unmountRequest, unmountResponse) return } From 082519bff5949b34a4c8e1ce8c43284e2c224ad1 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 7 Oct 2021 15:11:57 -0700 Subject: [PATCH 112/258] Added iclient .conf settings for B+Tree tunables --- iclient/dev.conf | 66 ++++++++++--------- iclient/iclient.conf | 68 ++++++++++--------- iclient/iclientpkg/api.go | 72 ++++++++++---------- iclient/iclientpkg/globals.go | 119 +++++++++++++++++++++------------- iclient/iclientpkg/inode.go | 86 ++++++++++++++++++++++++ iclient/iclientpkg/lease.go | 24 ++++--- 6 files changed, 283 insertions(+), 152 deletions(-) diff --git a/iclient/dev.conf b/iclient/dev.conf index 726ab260..ae3d641f 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -2,37 +2,41 @@ # SPDX-License-Identifier: Apache-2.0 [ICLIENT] -VolumeName: testvol -MountPointDirPath: /mnt -FUSEAllowOther: true -FUSEMaxBackground: 1000 -FUSECongestionThreshhold: 0 -FUSEMaxWrite: 131076 -FUSEEntryValidDuration: 0s -FUSEAttrValidDuration: 0s -AuthPlugInPath: iauth/iauth-swift/iauth-swift.so +VolumeName: testvol +MountPointDirPath: /mnt +FUSEAllowOther: true +FUSEMaxBackground: 1000 +FUSECongestionThreshhold: 0 +FUSEMaxWrite: 131076 +FUSEEntryValidDuration: 0s +FUSEAttrValidDuration: 0s +AuthPlugInPath: iauth/iauth-swift/iauth-swift.so AuthPlugInEnvName: -AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} -SwiftTimeout: 10m -SwiftRetryLimit: 4 -SwiftRetryDelay: 100ms -SwiftRetryDelayVariance: 25 -SwiftRetryExponentialBackoff: 1.4 -SwiftConnectionPoolSize: 128 -RetryRPCPublicIPAddr: dev -RetryRPCPort: 32356 -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s +AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +SwiftTimeout: 10m +SwiftRetryLimit: 4 +SwiftRetryDelay: 100ms +SwiftRetryDelayVariance: 25 +SwiftRetryExponentialBackoff: 1.4 +SwiftConnectionPoolSize: 128 +RetryRPCPublicIPAddr: dev +RetryRPCPort: 32356 +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s RetryRPCCACertFilePath: -MaxSharedLeases: 500 -MaxExclusiveLeases: 100 -ReadCacheLineSize: 1048576 -ReadCacheLineCountMax: 1024 -FileFlushTriggerSize: 10485760 -FileFlushTriggerDuration: 10s +MaxSharedLeases: 500 +MaxExclusiveLeases: 100 +InodePayloadEvictLowLimit: 100000 +InodePayloadEvictHighLimit: 100010 +DirInodeMaxKeysPerBPlusTreePage: 1024 +FileInodeMaxKeysPerBPlusTreePage: 2048 +ReadCacheLineSize: 1048576 +ReadCacheLineCountMax: 1024 +FileFlushTriggerSize: 10485760 +FileFlushTriggerDuration: 10s LogFilePath: -LogToConsole: true -TraceEnabled: false -FUSELogEnabled: false -HTTPServerIPAddr: dev -HTTPServerPort: 15347 +LogToConsole: true +TraceEnabled: false +FUSELogEnabled: false +HTTPServerIPAddr: dev +HTTPServerPort: 15347 diff --git a/iclient/iclient.conf b/iclient/iclient.conf index 42c46a87..96aa1d30 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -2,37 +2,41 @@ # SPDX-License-Identifier: Apache-2.0 [ICLIENT] -VolumeName: testvol -MountPointDirPath: /mnt -FUSEAllowOther: true -FUSEMaxBackground: 1000 -FUSECongestionThreshhold: 0 -FUSEMaxWrite: 131076 -FUSEEntryValidDuration: 0s -FUSEAttrValidDuration: 0s -AuthPlugInPath: iauth-swift.so +VolumeName: testvol +MountPointDirPath: /mnt +FUSEAllowOther: true +FUSEMaxBackground: 1000 +FUSECongestionThreshhold: 0 +FUSEMaxWrite: 131076 +FUSEEntryValidDuration: 0s +FUSEAttrValidDuration: 0s +AuthPlugInPath: iauth-swift.so AuthPlugInEnvName: -AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} -SwiftTimeout: 10m -SwiftRetryLimit: 4 -SwiftRetryDelay: 100ms -SwiftRetryDelayVariance: 25 -SwiftRetryExponentialBackoff: 1.4 -SwiftConnectionPoolSize: 128 -RetryRPCPublicIPAddr: imgr -RetryRPCPort: 32356 -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCACertFilePath: caCert.pem -MaxSharedLeases: 500 -MaxExclusiveLeases: 100 -ReadCacheLineSize: 1048576 -ReadCacheLineCountMax: 1024 -FileFlushTriggerSize: 10485760 -FileFlushTriggerDuration: 10s +AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +SwiftTimeout: 10m +SwiftRetryLimit: 4 +SwiftRetryDelay: 100ms +SwiftRetryDelayVariance: 25 +SwiftRetryExponentialBackoff: 1.4 +SwiftConnectionPoolSize: 128 +RetryRPCPublicIPAddr: imgr +RetryRPCPort: 32356 +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s +RetryRPCCACertFilePath: caCert.pem +MaxSharedLeases: 500 +MaxExclusiveLeases: 100 +InodePayloadEvictLowLimit: 100000 +InodePayloadEvictHighLimit: 100010 +DirInodeMaxKeysPerBPlusTreePage: 1024 +FileInodeMaxKeysPerBPlusTreePage: 2048 +ReadCacheLineSize: 1048576 +ReadCacheLineCountMax: 1024 +FileFlushTriggerSize: 10485760 +FileFlushTriggerDuration: 10s LogFilePath: -LogToConsole: true -TraceEnabled: false -FUSELogEnabled: false -HTTPServerIPAddr: iclient -HTTPServerPort: 15347 +LogToConsole: true +TraceEnabled: false +FUSELogEnabled: false +HTTPServerIPAddr: iclient +HTTPServerPort: 15347 diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index 035c8389..74a635e3 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -8,40 +8,44 @@ // argument, a package conf ConfMap. Here is a sample .conf file: // // [ICLIENT] -// VolumeName: testvol -// MountPointDirPath: /mnt -// FUSEAllowOther: true -// FUSEMaxBackground: 1000 -// FUSECongestionThreshhold: 0 -// FUSEMaxWrite: 131076 -// FUSEEntryValidDuration: 0s -// FUSEAttrValidDuration: 0s -// AuthPlugInPath: iauth-swift.so -// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here -// AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} -// SwiftTimeout: 10m -// SwiftRetryLimit: 4 -// SwiftRetryDelay: 100ms -// SwiftRetryDelayVariance: 25 # Percentage (1-100) of SwitRetryDelay -// SwiftRetryExponentialBackoff: 1.4 -// SwiftConnectionPoolSize: 128 -// RetryRPCPublicIPAddr: imgr -// RetryRPCPort: 32356 -// RetryRPCDeadlineIO: 60s -// RetryRPCKeepAlivePeriod: 60s -// RetryRPCCACertFilePath: # Defaults to /dev/null -// MaxSharedLeases: 500 -// MaxExclusiveLeases: 100 # Caps pending FileFlush data at 1GiB (with FileFlushTriggerSize of 10MiB) -// ReadCacheLineSize: 1048576 # 1MiB -// ReadCacheLineCountMax: 1024 # 1GiB (with ReadCacheLineSize of 1MiB) -// FileFlushTriggerSize: 10485760 # [10MiB] Amount of written data before metadata is appended and flush is triggered -// FileFlushTriggerDuration: 10s # Amount of time before unwritten data and its metadata flush is triggered -// LogFilePath: iclient.log -// LogToConsole: true -// TraceEnabled: false -// FUSELogEnabled: false -// HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) -// HTTPServerPort: # Defaults to disabling the embedded HTTP Server +// VolumeName: testvol +// MountPointDirPath: /mnt +// FUSEAllowOther: true +// FUSEMaxBackground: 1000 +// FUSECongestionThreshhold: 0 +// FUSEMaxWrite: 131076 +// FUSEEntryValidDuration: 0s +// FUSEAttrValidDuration: 0s +// AuthPlugInPath: iauth-swift.so +// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here +// AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} +// SwiftTimeout: 10m +// SwiftRetryLimit: 4 +// SwiftRetryDelay: 100ms +// SwiftRetryDelayVariance: 25 # Percentage (1-100) of SwitRetryDelay +// SwiftRetryExponentialBackoff: 1.4 +// SwiftConnectionPoolSize: 128 +// RetryRPCPublicIPAddr: imgr +// RetryRPCPort: 32356 +// RetryRPCDeadlineIO: 60s +// RetryRPCKeepAlivePeriod: 60s +// RetryRPCCACertFilePath: # Defaults to /dev/null +// MaxSharedLeases: 500 +// MaxExclusiveLeases: 100 # Caps pending FileFlush data at 1GiB (with FileFlushTriggerSize of 10MiB) +// InodePayloadEvictLowLimit: 100000 +// InodePayloadEvictHighLimit: 100010 +// DirInodeMaxKeysPerBPlusTreePage: 1024 +// FileInodeMaxKeysPerBPlusTreePage: 2048 +// ReadCacheLineSize: 1048576 # 1MiB +// ReadCacheLineCountMax: 1024 # 1GiB (with ReadCacheLineSize of 1MiB) +// FileFlushTriggerSize: 10485760 # [10MiB] Amount of written data before metadata is appended and flush is triggered +// FileFlushTriggerDuration: 10s # Amount of time before unwritten data and its metadata flush is triggered +// LogFilePath: iclient.log +// LogToConsole: true +// TraceEnabled: false +// FUSELogEnabled: false +// HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) +// HTTPServerPort: # Defaults to disabling the embedded HTTP Server // // Most of the config keys are required and must have values. One set of exceptions // are the HTTPServer{IPAddr|Port} keys that, if not present (or HTTPServerPort is diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index f84dfced..14a6d43a 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -23,40 +23,44 @@ import ( ) type configStruct struct { - VolumeName string - MountPointDirPath string - FUSEAllowOther bool - FUSEMaxBackground uint16 - FUSECongestionThreshhold uint16 - FUSEMaxWrite uint32 - FUSEEntryValidDuration time.Duration - FUSEAttrValidDuration time.Duration - AuthPlugInPath string - AuthPlugInEnvName string - AuthPlugInEnvValue string - SwiftTimeout time.Duration - SwiftRetryLimit uint64 - SwiftRetryDelay time.Duration - SwiftRetryDelayVariance uint8 - SwiftRetryExponentialBackoff float64 - SwiftConnectionPoolSize uint32 - RetryRPCPublicIPAddr string - RetryRPCPort uint16 - RetryRPCDeadlineIO time.Duration - RetryRPCKeepAlivePeriod time.Duration - RetryRPCCACertFilePath string // Defaults to /dev/null - MaxSharedLeases uint64 - MaxExclusiveLeases uint64 - ReadCacheLineSize uint64 - ReadCacheLineCountMax uint64 - FileFlushTriggerSize uint64 - FileFlushTriggerDuration time.Duration - LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled - LogToConsole bool - TraceEnabled bool - FUSELogEnabled bool - HTTPServerIPAddr string - HTTPServerPort uint16 // To be served on HTTPServerIPAddr via TCP + VolumeName string + MountPointDirPath string + FUSEAllowOther bool + FUSEMaxBackground uint16 + FUSECongestionThreshhold uint16 + FUSEMaxWrite uint32 + FUSEEntryValidDuration time.Duration + FUSEAttrValidDuration time.Duration + AuthPlugInPath string + AuthPlugInEnvName string + AuthPlugInEnvValue string + SwiftTimeout time.Duration + SwiftRetryLimit uint64 + SwiftRetryDelay time.Duration + SwiftRetryDelayVariance uint8 + SwiftRetryExponentialBackoff float64 + SwiftConnectionPoolSize uint32 + RetryRPCPublicIPAddr string + RetryRPCPort uint16 + RetryRPCDeadlineIO time.Duration + RetryRPCKeepAlivePeriod time.Duration + RetryRPCCACertFilePath string // Defaults to /dev/null + MaxSharedLeases uint64 + MaxExclusiveLeases uint64 + InodePayloadEvictLowLimit uint64 + InodePayloadEvictHighLimit uint64 + DirInodeMaxKeysPerBPlusTreePage uint64 + FileInodeMaxKeysPerBPlusTreePage uint64 + ReadCacheLineSize uint64 + ReadCacheLineCountMax uint64 + FileFlushTriggerSize uint64 + FileFlushTriggerDuration time.Duration + LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled + LogToConsole bool + TraceEnabled bool + FUSELogEnabled bool + HTTPServerIPAddr string + HTTPServerPort uint16 // To be served on HTTPServerIPAddr via TCP } type swiftRetryDelayElementStruct struct { @@ -88,15 +92,19 @@ type layoutMapEntryStruct struct { type inodeLeaseStruct struct { inodeNumber uint64 // state inodeLeaseStateType // - listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU - heldList *list.List // List of granted inodeHeldLockStruct's - requestList *list.List // List of pending inodeLockRequestStruct's + listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU + heldList *list.List // List of granted inodeHeldLockStruct's + requestList *list.List // List of pending inodeLockRequestStruct's inodeHeadV1 *ilayout.InodeHeadV1Struct // - payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} - // For FileInode: ExtentMap B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} - layoutMap map[uint64]layoutMapEntryStruct // For DirInode & FileInode: Map form of .inodeHeadV1.Layout - putObjectNumber uint64 // For DirInode & FileInode: - putObjectBuffer []byte // ObjectNumber and buffer to PUT during next flush + payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} + // For FileInode: ExtentMap B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} + layoutMap map[uint64]layoutMapEntryStruct // For DirInode & FileInode: Map form of .inodeHeadV1.Layout + superBlockInodeObjectCountAdjustment int64 // + superBlockInodeObjectSizeAdjustment int64 // + superBlockInodeBytesReferencedAdjustment int64 // + dereferencedObjectNumberArray []uint64 // + putObjectNumber uint64 // For DirInode & FileInode: + putObjectBuffer []byte // ObjectNumber and buffer to PUT during next flush } type inodeHeldLockStruct struct { @@ -181,7 +189,7 @@ type statsStruct struct { } type globalsStruct struct { - sync.Mutex // Serializes access to inodeLeaseTable + sync.Mutex // config configStruct // fuseEntryValidDurationSec uint64 // fuseEntryValidDurationNSec uint32 // @@ -200,8 +208,7 @@ type globalsStruct struct { mountID string // fissionErrChan chan error // inodeLeaseTable map[uint64]*inodeLeaseStruct // - inodeLeaseWG sync.WaitGroup // Signaled as each (*inodeLeaseStruct).goroutine() exits - inodeLockWG sync.WaitGroup // Signaled as each active inodeLockRequestStruct has signaled service complete with .locksHeld empty + inodeLeasePayloadCache sortedmap.BPlusTreeCache // sharedLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted exclusiveLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted httpServer *http.Server // @@ -353,6 +360,22 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err if nil != err { logFatal(err) } + globals.config.InodePayloadEvictLowLimit, err = confMap.FetchOptionValueUint64("ICLIENT", "InodePayloadEvictLowLimit") + if nil != err { + logFatal(err) + } + globals.config.InodePayloadEvictHighLimit, err = confMap.FetchOptionValueUint64("ICLIENT", "InodePayloadEvictHighLimit") + if nil != err { + logFatal(err) + } + globals.config.DirInodeMaxKeysPerBPlusTreePage, err = confMap.FetchOptionValueUint64("ICLIENT", "DirInodeMaxKeysPerBPlusTreePage") + if nil != err { + logFatal(err) + } + globals.config.FileInodeMaxKeysPerBPlusTreePage, err = confMap.FetchOptionValueUint64("ICLIENT", "FileInodeMaxKeysPerBPlusTreePage") + if nil != err { + logFatal(err) + } globals.config.ReadCacheLineSize, err = confMap.FetchOptionValueUint64("ICLIENT", "ReadCacheLineSize") if nil != err { logFatal(err) @@ -418,6 +441,7 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err globals.fissionErrChan = fissionErrChan globals.inodeLeaseTable = make(map[uint64]*inodeLeaseStruct) + globals.inodeLeasePayloadCache = sortedmap.NewBPlusTreeCache(globals.config.InodePayloadEvictLowLimit, globals.config.InodePayloadEvictHighLimit) globals.sharedLeaseLRU = list.New() globals.exclusiveLeaseLRU = list.New() @@ -454,6 +478,10 @@ func uninitializeGlobals() (err error) { globals.config.RetryRPCCACertFilePath = "" globals.config.MaxSharedLeases = 0 globals.config.MaxExclusiveLeases = 0 + globals.config.InodePayloadEvictLowLimit = 0 + globals.config.InodePayloadEvictHighLimit = 0 + globals.config.DirInodeMaxKeysPerBPlusTreePage = 0 + globals.config.FileInodeMaxKeysPerBPlusTreePage = 0 globals.config.ReadCacheLineSize = 0 globals.config.ReadCacheLineCountMax = 0 globals.config.FileFlushTriggerSize = 0 @@ -473,6 +501,7 @@ func uninitializeGlobals() (err error) { globals.retryRPCCACertPEM = nil globals.inodeLeaseTable = nil + globals.inodeLeasePayloadCache = nil globals.sharedLeaseLRU = nil globals.exclusiveLeaseLRU = nil diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 786ddc7a..99a8d6db 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -170,3 +170,89 @@ func (inodeLease *inodeLeaseStruct) UnpackValue(payloadData []byte) (value sorte } return // TODO } + +func (inodeLease *inodeLeaseStruct) newPayload() (err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + inodeLease.payload = sortedmap.NewBPlusTree( + globals.config.DirInodeMaxKeysPerBPlusTreePage, + sortedmap.CompareString, + inodeLease, + globals.inodeLeasePayloadCache) + err = nil + case ilayout.InodeTypeFile: + inodeLease.payload = sortedmap.NewBPlusTree( + globals.config.FileInodeMaxKeysPerBPlusTreePage, + sortedmap.CompareUint64, + inodeLease, + globals.inodeLeasePayloadCache) + err = nil + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + return +} + +func (inodeLease *inodeLeaseStruct) oldPayload() (err error) { + switch inodeLease.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + inodeLease.payload, err = sortedmap.OldBPlusTree( + inodeLease.inodeHeadV1.PayloadObjectNumber, + inodeLease.inodeHeadV1.PayloadObjectOffset, + inodeLease.inodeHeadV1.PayloadObjectLength, + sortedmap.CompareString, + inodeLease, + globals.inodeLeasePayloadCache) + if nil != err { + inodeLease.payload = nil + } + case ilayout.InodeTypeFile: + inodeLease.payload, err = sortedmap.OldBPlusTree( + inodeLease.inodeHeadV1.PayloadObjectNumber, + inodeLease.inodeHeadV1.PayloadObjectOffset, + inodeLease.inodeHeadV1.PayloadObjectLength, + sortedmap.CompareUint64, + inodeLease, + globals.inodeLeasePayloadCache) + if nil != err { + inodeLease.payload = nil + } + default: + err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + return +} + +func (inodeLease *inodeLeaseStruct) convertInodeHeadV1LayoutToLayoutMap() { + var ( + ilayoutInodeHeadLayoutEntryV1 ilayout.InodeHeadLayoutEntryV1Struct + ) + + inodeLease.layoutMap = make(map[uint64]layoutMapEntryStruct) + + for _, ilayoutInodeHeadLayoutEntryV1 = range inodeLease.inodeHeadV1.Layout { + inodeLease.layoutMap[ilayoutInodeHeadLayoutEntryV1.ObjectNumber] = layoutMapEntryStruct{ + objectSize: ilayoutInodeHeadLayoutEntryV1.ObjectSize, + bytesReferenced: ilayoutInodeHeadLayoutEntryV1.BytesReferenced, + } + } +} + +func (inodeLease *inodeLeaseStruct) convertLayoutMapToInodeHeadV1Layout() { + var ( + ilayoutInodeHeadV1LayoutIndex uint64 = 0 + layoutMapEntry layoutMapEntryStruct + objectNumber uint64 + ) + + // TODO + inodeLease.inodeHeadV1.Layout = make([]ilayout.InodeHeadLayoutEntryV1Struct, len(inodeLease.layoutMap)) + + for objectNumber, layoutMapEntry = range inodeLease.layoutMap { + inodeLease.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectNumber = objectNumber + inodeLease.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectSize = layoutMapEntry.objectSize + inodeLease.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced + } +} diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 3eaab1a7..0114a3b4 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -89,16 +89,20 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } } else { inodeLease = &inodeLeaseStruct{ - inodeNumber: inodeLockRequest.inodeNumber, - state: inodeLeaseStateNone, - listElement: nil, - heldList: list.New(), - requestList: list.New(), - inodeHeadV1: nil, - payload: nil, - layoutMap: nil, - putObjectNumber: 0, - putObjectBuffer: nil, + inodeNumber: inodeLockRequest.inodeNumber, + state: inodeLeaseStateNone, + listElement: nil, + heldList: list.New(), + requestList: list.New(), + inodeHeadV1: nil, + payload: nil, + layoutMap: nil, + superBlockInodeObjectCountAdjustment: 0, + superBlockInodeObjectSizeAdjustment: 0, + superBlockInodeBytesReferencedAdjustment: 0, + dereferencedObjectNumberArray: make([]uint64, 0), + putObjectNumber: 0, + putObjectBuffer: nil, } globals.inodeLeaseTable[inodeLockRequest.inodeNumber] = inodeLease From 390edb8cb9c52dabda3250827d512fc17c80ad02 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 7 Oct 2021 15:17:15 -0700 Subject: [PATCH 113/258] Switch inodeLeaseStruct.state to .leaseState in prep for inodeLeaseStruct -> inodeStruct --- iclient/iclientpkg/globals.go | 2 +- iclient/iclientpkg/http-server.go | 4 ++-- iclient/iclientpkg/lease.go | 26 +++++++++++++------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 14a6d43a..aa56aa2f 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -91,7 +91,7 @@ type layoutMapEntryStruct struct { type inodeLeaseStruct struct { inodeNumber uint64 // - state inodeLeaseStateType // + leaseState inodeLeaseStateType // listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU heldList *list.List // List of granted inodeHeldLockStruct's requestList *list.List // List of pending inodeLockRequestStruct's diff --git a/iclient/iclientpkg/http-server.go b/iclient/iclientpkg/http-server.go index bfc00f89..0fd4d377 100644 --- a/iclient/iclientpkg/http-server.go +++ b/iclient/iclientpkg/http-server.go @@ -226,7 +226,7 @@ func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Requ inodeLeaseTableIndex = 0 for inodeNumber, inodeLease = range globals.inodeLeaseTable { inodeLeaseTable[inodeLeaseTableIndex].InodeNumber = inodeNumber - switch inodeLease.state { + switch inodeLease.leaseState { case inodeLeaseStateNone: inodeLeaseTable[inodeLeaseTableIndex].State = "None" case inodeLeaseStateSharedRequested: @@ -250,7 +250,7 @@ func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Requ case inodeLeaseStateExclusiveExpired: inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveExpired" default: - logFatalf("globals.inodeLeaseTable[inudeNumber:0x%016X].state (%v) unrecognized", inodeNumber, inodeLease.state) + logFatalf("globals.inodeLeaseTable[inudeNumber:0x%016X].leaseState (%v) unrecognized", inodeNumber, inodeLease.leaseState) } inodeLeaseTableIndex++ } diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 0114a3b4..c3454bbc 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -66,7 +66,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLease, ok = globals.inodeLeaseTable[inodeLockRequest.inodeNumber] if ok { - switch inodeLease.state { + switch inodeLease.leaseState { case inodeLeaseStateNone: case inodeLeaseStateSharedRequested: globals.sharedLeaseLRU.MoveToBack(inodeLease.listElement) @@ -85,12 +85,12 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateExclusiveReleasing: case inodeLeaseStateExclusiveExpired: default: - logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) + logFatalf("switch inodeLease.leaseState unexpected: %v", inodeLease.leaseState) } } else { inodeLease = &inodeLeaseStruct{ inodeNumber: inodeLockRequest.inodeNumber, - state: inodeLeaseStateNone, + leaseState: inodeLeaseStateNone, listElement: nil, heldList: list.New(), requestList: list.New(), @@ -158,10 +158,10 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } if inodeLockRequest.exclusive { - switch inodeLease.state { + switch inodeLease.leaseState { case inodeLeaseStateNone: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) - inodeLease.state = inodeLeaseStateExclusiveRequested + inodeLease.leaseState = inodeLeaseStateExclusiveRequested inodeLease.listElement = globals.exclusiveLeaseLRU.PushBack(inodeLease) globals.Unlock() @@ -184,7 +184,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { globals.Lock() - inodeLease.state = inodeLeaseStateExclusiveGranted + inodeLease.leaseState = inodeLeaseStateExclusiveGranted if inodeLease.requestList.Front() != inodeLockRequest.listElement { logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") @@ -211,7 +211,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.Wait() case inodeLeaseStateSharedGranted: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) - inodeLease.state = inodeLeaseStateSharedPromoting + inodeLease.leaseState = inodeLeaseStateSharedPromoting _ = globals.sharedLeaseLRU.Remove(inodeLease.listElement) inodeLease.listElement = globals.exclusiveLeaseLRU.PushBack(inodeLease) @@ -235,7 +235,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { globals.Lock() - inodeLease.state = inodeLeaseStateExclusiveGranted + inodeLease.leaseState = inodeLeaseStateExclusiveGranted if inodeLease.requestList.Front() != inodeLockRequest.listElement { logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") @@ -303,13 +303,13 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.unlockAllWhileLocked() globals.Unlock() default: - logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) + logFatalf("switch inodeLease.leaseState unexpected: %v", inodeLease.leaseState) } } else { - switch inodeLease.state { + switch inodeLease.leaseState { case inodeLeaseStateNone: inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) - inodeLease.state = inodeLeaseStateSharedRequested + inodeLease.leaseState = inodeLeaseStateSharedRequested inodeLease.listElement = globals.sharedLeaseLRU.PushBack(inodeLease) globals.Unlock() @@ -332,7 +332,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { globals.Lock() - inodeLease.state = inodeLeaseStateSharedGranted + inodeLease.leaseState = inodeLeaseStateSharedGranted if inodeLease.requestList.Front() != inodeLockRequest.listElement { logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") @@ -420,7 +420,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.unlockAllWhileLocked() globals.Unlock() default: - logFatalf("switch inodeLease.state unexpected: %v", inodeLease.state) + logFatalf("switch inodeLease.leaseState unexpected: %v", inodeLease.leaseState) } } } From 8aff5b40eac329a43526c745672d9f7de4617cfd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 7 Oct 2021 15:45:21 -0700 Subject: [PATCH 114/258] Renamed inodeLeaseStruct to inodeStruct since it is really everything about an Inode... not just Leases --- iclient/iclientpkg/fission.go | 28 +++--- iclient/iclientpkg/globals.go | 26 ++--- iclient/iclientpkg/http-server.go | 76 +++++++-------- iclient/iclientpkg/inode.go | 134 +++++++++++++------------- iclient/iclientpkg/lease.go | 154 +++++++++++++++--------------- 5 files changed, 209 insertions(+), 209 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 76309db2..95278dbe 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -106,8 +106,8 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis err error getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct getInodeTableEntryResponse *imgrpkg.GetInodeTableEntryResponseStruct + inode *inodeStruct inodeHeadV1Buf []byte - inodeLease *inodeLeaseStruct inodeLockRequest *inodeLockRequestStruct modificationTimeNSec uint32 modificationTimeSec uint64 @@ -130,15 +130,15 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis inodeLockRequest.exclusive = false inodeLockRequest.addThisLock() - inodeLease = lookupInodeLease(uint64(inHeader.NodeID)) - if nil == inodeLease { + inode = lookupInode(uint64(inHeader.NodeID)) + if nil == inode { inodeLockRequest.unlockAll() getAttrOut = nil errno = syscall.ENOENT return } - if nil == inodeLease.inodeHeadV1 { + if nil == inode.inodeHeadV1 { getInodeTableEntryRequest = &imgrpkg.GetInodeTableEntryRequestStruct{ MountID: globals.mountID, InodeNumber: uint64(inHeader.NodeID), @@ -158,33 +158,33 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis logFatalf("objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber: %v, getInodeTableEntryResponse.InodeHeadLength: %v) failed: %v", getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength, err) } - inodeLease.inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) + inode.inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) if nil != err { logFatalf("ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) failed: %v", err) } } - modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inodeLease.inodeHeadV1.ModificationTime.UnixNano())) - statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(inodeLease.inodeHeadV1.StatusChangeTime.UnixNano())) + modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.ModificationTime.UnixNano())) + statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.StatusChangeTime.UnixNano())) getAttrOut = &fission.GetAttrOut{ AttrValidSec: globals.fuseAttrValidDurationSec, AttrValidNSec: globals.fuseAttrValidDurationNSec, Dummy: 0, Attr: fission.Attr{ - Ino: inodeLease.inodeHeadV1.InodeNumber, - Size: inodeLease.inodeHeadV1.Size, // Possibly overwritten by fixAttrSizes() - Blocks: 0, // Computed by fixAttrSizes() + Ino: inode.inodeHeadV1.InodeNumber, + Size: inode.inodeHeadV1.Size, // Possibly overwritten by fixAttrSizes() + Blocks: 0, // Computed by fixAttrSizes() ATimeSec: modificationTimeSec, MTimeSec: modificationTimeSec, CTimeSec: statusChangeTimeSec, ATimeNSec: modificationTimeNSec, MTimeNSec: modificationTimeNSec, CTimeNSec: statusChangeTimeNSec, - Mode: computeAttrMode(inodeLease.inodeHeadV1.InodeType, inodeLease.inodeHeadV1.Mode), - NLink: uint32(len(inodeLease.inodeHeadV1.LinkTable)), - UID: uint32(inodeLease.inodeHeadV1.UserID), - GID: uint32(inodeLease.inodeHeadV1.GroupID), + Mode: computeAttrMode(inode.inodeHeadV1.InodeType, inode.inodeHeadV1.Mode), + NLink: uint32(len(inode.inodeHeadV1.LinkTable)), + UID: uint32(inode.inodeHeadV1.UserID), + GID: uint32(inode.inodeHeadV1.GroupID), RDev: attrRDev, BlkSize: attrBlockSize, // Possibly overwritten by fixAttrSizes() Padding: 0, diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index aa56aa2f..6b75a795 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -71,7 +71,7 @@ type swiftRetryDelayElementStruct struct { type inodeLeaseStateType uint32 const ( - inodeLeaseStateNone inodeLeaseStateType = iota // Only used if (*inodeLeaseStruct).requestList is non-empty + inodeLeaseStateNone inodeLeaseStateType = iota // Only used if (*inodeStruct).requestList is non-empty inodeLeaseStateSharedRequested inodeLeaseStateSharedGranted inodeLeaseStateSharedPromoting @@ -89,7 +89,7 @@ type layoutMapEntryStruct struct { bytesReferenced uint64 } -type inodeLeaseStruct struct { +type inodeStruct struct { inodeNumber uint64 // leaseState inodeLeaseStateType // listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU @@ -108,17 +108,17 @@ type inodeLeaseStruct struct { } type inodeHeldLockStruct struct { - inodeLease *inodeLeaseStruct + inode *inodeStruct inodeLockRequest *inodeLockRequestStruct exclusive bool - listElement *list.Element //Maintains position in .inodeNumber's indicated inodeLeaseStruct.heldList + listElement *list.Element //Maintains position in .inodeNumber's indicated inodeStruct.heldList } type inodeLockRequestStruct struct { sync.WaitGroup // Signaled when the state of the lock request has been served inodeNumber uint64 // The inodeNumber for which the latest lock request is being made exclusive bool // Indicates if the latest lock request is exclusive - listElement *list.Element // Maintains position in .inodeNumber's indicated inodeLeaseStruct.requestList + listElement *list.Element // Maintains position in .inodeNumber's indicated inodeStruct.requestList locksHeld map[uint64]*inodeHeldLockStruct // At entry, contains the list of inodeLock's already held (shared or exclusively) // At exit, either contains the earlier list appended with the granted inodeLock // or is empty indicating the caller should restart lock request sequence @@ -207,10 +207,10 @@ type globalsStruct struct { retryRPCClient *retryrpc.Client // mountID string // fissionErrChan chan error // - inodeLeaseTable map[uint64]*inodeLeaseStruct // - inodeLeasePayloadCache sortedmap.BPlusTreeCache // - sharedLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted - exclusiveLeaseLRU *list.List // LRU-ordered list of inodeLeaseStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted + inodeTable map[uint64]*inodeStruct // + inodePayloadCache sortedmap.BPlusTreeCache // + sharedLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted + exclusiveLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted httpServer *http.Server // httpServerWG sync.WaitGroup // stats *statsStruct // @@ -440,8 +440,8 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err globals.fissionErrChan = fissionErrChan - globals.inodeLeaseTable = make(map[uint64]*inodeLeaseStruct) - globals.inodeLeasePayloadCache = sortedmap.NewBPlusTreeCache(globals.config.InodePayloadEvictLowLimit, globals.config.InodePayloadEvictHighLimit) + globals.inodeTable = make(map[uint64]*inodeStruct) + globals.inodePayloadCache = sortedmap.NewBPlusTreeCache(globals.config.InodePayloadEvictLowLimit, globals.config.InodePayloadEvictHighLimit) globals.sharedLeaseLRU = list.New() globals.exclusiveLeaseLRU = list.New() @@ -500,8 +500,8 @@ func uninitializeGlobals() (err error) { globals.retryRPCCACertPEM = nil - globals.inodeLeaseTable = nil - globals.inodeLeasePayloadCache = nil + globals.inodeTable = nil + globals.inodePayloadCache = nil globals.sharedLeaseLRU = nil globals.exclusiveLeaseLRU = nil diff --git a/iclient/iclientpkg/http-server.go b/iclient/iclientpkg/http-server.go index 0fd4d377..d9728cb8 100644 --- a/iclient/iclientpkg/http-server.go +++ b/iclient/iclientpkg/http-server.go @@ -185,35 +185,35 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ } } -type inodeLeaseTableByInodeNumberElement struct { +type inodeTableByInodeNumberElement struct { InodeNumber uint64 State string } -type inodeLeaseTableByInodeNumberSlice []inodeLeaseTableByInodeNumberElement +type inodeTableByInodeNumberSlice []inodeTableByInodeNumberElement -func (s inodeLeaseTableByInodeNumberSlice) Len() int { +func (s inodeTableByInodeNumberSlice) Len() int { return len(s) } -func (s inodeLeaseTableByInodeNumberSlice) Swap(i, j int) { +func (s inodeTableByInodeNumberSlice) Swap(i, j int) { s[i].InodeNumber, s[j].InodeNumber = s[j].InodeNumber, s[i].InodeNumber s[i].State, s[j].State = s[j].State, s[i].State } -func (s inodeLeaseTableByInodeNumberSlice) Less(i, j int) bool { +func (s inodeTableByInodeNumberSlice) Less(i, j int) bool { return s[i].InodeNumber < s[j].InodeNumber } func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Request) { var ( - err error - inodeLease *inodeLeaseStruct - inodeLeaseTable inodeLeaseTableByInodeNumberSlice - inodeLeaseTableJSON []byte - inodeLeaseTableIndex int - inodeNumber uint64 - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeTable inodeTableByInodeNumberSlice + inodeTableJSON []byte + inodeTableIndex int + inodeNumber uint64 + startTime time.Time = time.Now() ) defer func() { @@ -222,64 +222,64 @@ func serveHTTPGetOfLeases(responseWriter http.ResponseWriter, request *http.Requ globals.Lock() - inodeLeaseTable = make(inodeLeaseTableByInodeNumberSlice, len(globals.inodeLeaseTable)) - inodeLeaseTableIndex = 0 - for inodeNumber, inodeLease = range globals.inodeLeaseTable { - inodeLeaseTable[inodeLeaseTableIndex].InodeNumber = inodeNumber - switch inodeLease.leaseState { + inodeTable = make(inodeTableByInodeNumberSlice, len(globals.inodeTable)) + inodeTableIndex = 0 + for inodeNumber, inode = range globals.inodeTable { + inodeTable[inodeTableIndex].InodeNumber = inodeNumber + switch inode.leaseState { case inodeLeaseStateNone: - inodeLeaseTable[inodeLeaseTableIndex].State = "None" + inodeTable[inodeTableIndex].State = "None" case inodeLeaseStateSharedRequested: - inodeLeaseTable[inodeLeaseTableIndex].State = "SharedRequested" + inodeTable[inodeTableIndex].State = "SharedRequested" case inodeLeaseStateSharedGranted: - inodeLeaseTable[inodeLeaseTableIndex].State = "SharedGranted" + inodeTable[inodeTableIndex].State = "SharedGranted" case inodeLeaseStateSharedPromoting: - inodeLeaseTable[inodeLeaseTableIndex].State = "SharedPromoting" + inodeTable[inodeTableIndex].State = "SharedPromoting" case inodeLeaseStateSharedReleasing: - inodeLeaseTable[inodeLeaseTableIndex].State = "SharedReleasing" + inodeTable[inodeTableIndex].State = "SharedReleasing" case inodeLeaseStateSharedExpired: - inodeLeaseTable[inodeLeaseTableIndex].State = "SharedExpired" + inodeTable[inodeTableIndex].State = "SharedExpired" case inodeLeaseStateExclusiveRequested: - inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveRequested" + inodeTable[inodeTableIndex].State = "ExclusiveRequested" case inodeLeaseStateExclusiveGranted: - inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveGranted" + inodeTable[inodeTableIndex].State = "ExclusiveGranted" case inodeLeaseStateExclusiveDemoting: - inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveDemoting" + inodeTable[inodeTableIndex].State = "ExclusiveDemoting" case inodeLeaseStateExclusiveReleasing: - inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveReleasing" + inodeTable[inodeTableIndex].State = "ExclusiveReleasing" case inodeLeaseStateExclusiveExpired: - inodeLeaseTable[inodeLeaseTableIndex].State = "ExclusiveExpired" + inodeTable[inodeTableIndex].State = "ExclusiveExpired" default: - logFatalf("globals.inodeLeaseTable[inudeNumber:0x%016X].leaseState (%v) unrecognized", inodeNumber, inodeLease.leaseState) + logFatalf("globals.inodeTable[inudeNumber:0x%016X].leaseState (%v) unrecognized", inodeNumber, inode.leaseState) } - inodeLeaseTableIndex++ + inodeTableIndex++ } globals.Unlock() - sort.Sort(inodeLeaseTable) + sort.Sort(inodeTable) - inodeLeaseTableJSON, err = json.Marshal(inodeLeaseTable) + inodeTableJSON, err = json.Marshal(inodeTable) if nil != err { - logFatalf("json.Marshal(inodeLeaseTable) failed: %v", err) + logFatalf("json.Marshal(inodeTable) failed: %v", err) } if strings.Contains(request.Header.Get("Accept"), "text/html") { responseWriter.Header().Set("Content-Type", "text/html") responseWriter.WriteHeader(http.StatusOK) - _, err = responseWriter.Write([]byte(fmt.Sprintf(leasesTemplate, version.ProxyFSVersion, string(inodeLeaseTableJSON[:])))) + _, err = responseWriter.Write([]byte(fmt.Sprintf(leasesTemplate, version.ProxyFSVersion, string(inodeTableJSON[:])))) if nil != err { - logWarnf("responseWriter.Write([]byte(fmt.Sprintf(leasesTemplate, version.ProxyFSVersion, string(inodeLeaseTableJSON[:])))) failed: %v", err) + logWarnf("responseWriter.Write([]byte(fmt.Sprintf(leasesTemplate, version.ProxyFSVersion, string(inodeTableJSON[:])))) failed: %v", err) } } else { - responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(inodeLeaseTableJSON))) + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(inodeTableJSON))) responseWriter.Header().Set("Content-Type", "application/json") responseWriter.WriteHeader(http.StatusOK) - _, err = responseWriter.Write(inodeLeaseTableJSON) + _, err = responseWriter.Write(inodeTableJSON) if nil != err { - logWarnf("responseWriter.Write(inodeLeaseTableJSON) failed: %v", err) + logWarnf("responseWriter.Write(inodeTableJSON) failed: %v", err) } } } diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 99a8d6db..d0bee2d0 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -12,13 +12,13 @@ import ( "github.com/NVIDIA/proxyfs/ilayout" ) -func (inodeLease *inodeLeaseStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { +func (inode *inodeStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { var ( fileOffset uint64 ok bool ) - switch inodeLease.inodeHeadV1.InodeType { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: keyAsString, ok = key.(string) if ok { @@ -35,13 +35,13 @@ func (inodeLease *inodeLeaseStruct) DumpKey(key sortedmap.Key) (keyAsString stri err = fmt.Errorf("key.(uint64) returned !ok") } default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return } -func (inodeLease *inodeLeaseStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { +func (inode *inodeStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { var ( directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct directoryEntryValueV1JSON []byte @@ -50,7 +50,7 @@ func (inodeLease *inodeLeaseStruct) DumpValue(value sortedmap.Value) (valueAsStr ok bool ) - switch inodeLease.inodeHeadV1.InodeType { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: directoryEntryValueV1, ok = value.(*ilayout.DirectoryEntryValueV1Struct) if ok { @@ -78,27 +78,27 @@ func (inodeLease *inodeLeaseStruct) DumpValue(value sortedmap.Value) (valueAsStr err = fmt.Errorf("value.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") } default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return } -func (inodeLease *inodeLeaseStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: case ilayout.InodeTypeFile: default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - if objectNumber == inodeLease.putObjectNumber { - if (objectOffset + objectLength) <= uint64(len(inodeLease.putObjectBuffer)) { + if objectNumber == inode.putObjectNumber { + if (objectOffset + objectLength) <= uint64(len(inode.putObjectBuffer)) { nodeByteSlice = make([]byte, objectLength) - _ = copy(nodeByteSlice, inodeLease.putObjectBuffer[objectOffset:(objectOffset+objectLength)]) + _ = copy(nodeByteSlice, inode.putObjectBuffer[objectOffset:(objectOffset+objectLength)]) err = nil } else { - err = fmt.Errorf("(objectOffset + objectLength) > uint64(len(inodeLease.putObjectBuffer))") + err = fmt.Errorf("(objectOffset + objectLength) > uint64(len(inode.putObjectBuffer))") } } else { nodeByteSlice, err = objectGETRange(objectNumber, objectOffset, objectLength) @@ -111,136 +111,136 @@ func (inodeLease *inodeLeaseStruct) GetNode(objectNumber uint64, objectOffset ui return } -func (inodeLease *inodeLeaseStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: case ilayout.InodeTypeFile: default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return // TODO } -func (inodeLease *inodeLeaseStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: case ilayout.InodeTypeFile: default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return // TODO } -func (inodeLease *inodeLeaseStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: case ilayout.InodeTypeFile: default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return // TODO } -func (inodeLease *inodeLeaseStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: case ilayout.InodeTypeFile: default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return // TODO } -func (inodeLease *inodeLeaseStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: case ilayout.InodeTypeFile: default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return // TODO } -func (inodeLease *inodeLeaseStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: case ilayout.InodeTypeFile: default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return // TODO } -func (inodeLease *inodeLeaseStruct) newPayload() (err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) newPayload() (err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: - inodeLease.payload = sortedmap.NewBPlusTree( + inode.payload = sortedmap.NewBPlusTree( globals.config.DirInodeMaxKeysPerBPlusTreePage, sortedmap.CompareString, - inodeLease, - globals.inodeLeasePayloadCache) + inode, + globals.inodePayloadCache) err = nil case ilayout.InodeTypeFile: - inodeLease.payload = sortedmap.NewBPlusTree( + inode.payload = sortedmap.NewBPlusTree( globals.config.FileInodeMaxKeysPerBPlusTreePage, sortedmap.CompareUint64, - inodeLease, - globals.inodeLeasePayloadCache) + inode, + globals.inodePayloadCache) err = nil default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return } -func (inodeLease *inodeLeaseStruct) oldPayload() (err error) { - switch inodeLease.inodeHeadV1.InodeType { +func (inode *inodeStruct) oldPayload() (err error) { + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: - inodeLease.payload, err = sortedmap.OldBPlusTree( - inodeLease.inodeHeadV1.PayloadObjectNumber, - inodeLease.inodeHeadV1.PayloadObjectOffset, - inodeLease.inodeHeadV1.PayloadObjectLength, + inode.payload, err = sortedmap.OldBPlusTree( + inode.inodeHeadV1.PayloadObjectNumber, + inode.inodeHeadV1.PayloadObjectOffset, + inode.inodeHeadV1.PayloadObjectLength, sortedmap.CompareString, - inodeLease, - globals.inodeLeasePayloadCache) + inode, + globals.inodePayloadCache) if nil != err { - inodeLease.payload = nil + inode.payload = nil } case ilayout.InodeTypeFile: - inodeLease.payload, err = sortedmap.OldBPlusTree( - inodeLease.inodeHeadV1.PayloadObjectNumber, - inodeLease.inodeHeadV1.PayloadObjectOffset, - inodeLease.inodeHeadV1.PayloadObjectLength, + inode.payload, err = sortedmap.OldBPlusTree( + inode.inodeHeadV1.PayloadObjectNumber, + inode.inodeHeadV1.PayloadObjectOffset, + inode.inodeHeadV1.PayloadObjectLength, sortedmap.CompareUint64, - inodeLease, - globals.inodeLeasePayloadCache) + inode, + globals.inodePayloadCache) if nil != err { - inodeLease.payload = nil + inode.payload = nil } default: - err = fmt.Errorf("inodeLease.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inodeLease.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } return } -func (inodeLease *inodeLeaseStruct) convertInodeHeadV1LayoutToLayoutMap() { +func (inode *inodeStruct) convertInodeHeadV1LayoutToLayoutMap() { var ( ilayoutInodeHeadLayoutEntryV1 ilayout.InodeHeadLayoutEntryV1Struct ) - inodeLease.layoutMap = make(map[uint64]layoutMapEntryStruct) + inode.layoutMap = make(map[uint64]layoutMapEntryStruct) - for _, ilayoutInodeHeadLayoutEntryV1 = range inodeLease.inodeHeadV1.Layout { - inodeLease.layoutMap[ilayoutInodeHeadLayoutEntryV1.ObjectNumber] = layoutMapEntryStruct{ + for _, ilayoutInodeHeadLayoutEntryV1 = range inode.inodeHeadV1.Layout { + inode.layoutMap[ilayoutInodeHeadLayoutEntryV1.ObjectNumber] = layoutMapEntryStruct{ objectSize: ilayoutInodeHeadLayoutEntryV1.ObjectSize, bytesReferenced: ilayoutInodeHeadLayoutEntryV1.BytesReferenced, } } } -func (inodeLease *inodeLeaseStruct) convertLayoutMapToInodeHeadV1Layout() { +func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { var ( ilayoutInodeHeadV1LayoutIndex uint64 = 0 layoutMapEntry layoutMapEntryStruct @@ -248,11 +248,11 @@ func (inodeLease *inodeLeaseStruct) convertLayoutMapToInodeHeadV1Layout() { ) // TODO - inodeLease.inodeHeadV1.Layout = make([]ilayout.InodeHeadLayoutEntryV1Struct, len(inodeLease.layoutMap)) + inode.inodeHeadV1.Layout = make([]ilayout.InodeHeadLayoutEntryV1Struct, len(inode.layoutMap)) - for objectNumber, layoutMapEntry = range inodeLease.layoutMap { - inodeLease.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectNumber = objectNumber - inodeLease.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectSize = layoutMapEntry.objectSize - inodeLease.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced + for objectNumber, layoutMapEntry = range inode.layoutMap { + inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectNumber = objectNumber + inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectSize = layoutMapEntry.objectSize + inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced } } diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index c3454bbc..5627f395 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -47,8 +47,8 @@ func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { var ( err error + inode *inodeStruct inodeHeldLock *inodeHeldLockStruct - inodeLease *inodeLeaseStruct leaseRequest *imgrpkg.LeaseRequestStruct leaseResponse *imgrpkg.LeaseResponseStruct ok bool @@ -64,31 +64,31 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { logFatalf("*inodeLockRequestStruct)addThisLock() called with .inodeNumber already present in .locksHeld map") } - inodeLease, ok = globals.inodeLeaseTable[inodeLockRequest.inodeNumber] + inode, ok = globals.inodeTable[inodeLockRequest.inodeNumber] if ok { - switch inodeLease.leaseState { + switch inode.leaseState { case inodeLeaseStateNone: case inodeLeaseStateSharedRequested: - globals.sharedLeaseLRU.MoveToBack(inodeLease.listElement) + globals.sharedLeaseLRU.MoveToBack(inode.listElement) case inodeLeaseStateSharedGranted: - globals.sharedLeaseLRU.MoveToBack(inodeLease.listElement) + globals.sharedLeaseLRU.MoveToBack(inode.listElement) case inodeLeaseStateSharedPromoting: - globals.exclusiveLeaseLRU.MoveToBack(inodeLease.listElement) + globals.exclusiveLeaseLRU.MoveToBack(inode.listElement) case inodeLeaseStateSharedReleasing: case inodeLeaseStateSharedExpired: case inodeLeaseStateExclusiveRequested: - globals.exclusiveLeaseLRU.MoveToBack(inodeLease.listElement) + globals.exclusiveLeaseLRU.MoveToBack(inode.listElement) case inodeLeaseStateExclusiveGranted: - globals.exclusiveLeaseLRU.MoveToBack(inodeLease.listElement) + globals.exclusiveLeaseLRU.MoveToBack(inode.listElement) case inodeLeaseStateExclusiveDemoting: - globals.sharedLeaseLRU.MoveToBack(inodeLease.listElement) + globals.sharedLeaseLRU.MoveToBack(inode.listElement) case inodeLeaseStateExclusiveReleasing: case inodeLeaseStateExclusiveExpired: default: - logFatalf("switch inodeLease.leaseState unexpected: %v", inodeLease.leaseState) + logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } } else { - inodeLease = &inodeLeaseStruct{ + inode = &inodeStruct{ inodeNumber: inodeLockRequest.inodeNumber, leaseState: inodeLeaseStateNone, listElement: nil, @@ -105,15 +105,15 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { putObjectBuffer: nil, } - globals.inodeLeaseTable[inodeLockRequest.inodeNumber] = inodeLease + globals.inodeTable[inodeLockRequest.inodeNumber] = inode } - if inodeLease.requestList.Len() != 0 { + if inode.requestList.Len() != 0 { // At lease one other inodeLockRequestStruct is blocked, so this one must block inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() @@ -122,13 +122,13 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { return } - if inodeLease.heldList.Len() != 0 { + if inode.heldList.Len() != 0 { if inodeLockRequest.exclusive { // Lock is held, so this exclusive inodeLockRequestStruct must block inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() @@ -137,9 +137,9 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { return } - inodeHeldLock, ok = inodeLease.heldList.Front().Value.(*inodeHeldLockStruct) + inodeHeldLock, ok = inode.heldList.Front().Value.(*inodeHeldLockStruct) if !ok { - logFatalf("inodeLease.heldList.Front().Value.(*inodeHeldLockStruct) returned !ok") + logFatalf("inode.heldList.Front().Value.(*inodeHeldLockStruct) returned !ok") } if inodeHeldLock.exclusive { @@ -147,7 +147,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() @@ -158,11 +158,11 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } if inodeLockRequest.exclusive { - switch inodeLease.leaseState { + switch inode.leaseState { case inodeLeaseStateNone: - inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) - inodeLease.leaseState = inodeLeaseStateExclusiveRequested - inodeLease.listElement = globals.exclusiveLeaseLRU.PushBack(inodeLease) + inodeLockRequest.listElement = inode.requestList.PushFront(inodeLockRequest) + inode.leaseState = inodeLeaseStateExclusiveRequested + inode.listElement = globals.exclusiveLeaseLRU.PushBack(inode) globals.Unlock() @@ -184,36 +184,36 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { globals.Lock() - inodeLease.leaseState = inodeLeaseStateExclusiveGranted + inode.leaseState = inodeLeaseStateExclusiveGranted - if inodeLease.requestList.Front() != inodeLockRequest.listElement { - logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") + if inode.requestList.Front() != inodeLockRequest.listElement { + logFatalf("inode.requestList.Front() != inodeLockRequest.listElement") } - _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) + _ = inode.requestList.Remove(inodeLockRequest.listElement) inodeLockRequest.listElement = nil inodeHeldLock = &inodeHeldLockStruct{ - inodeLease: inodeLease, + inode: inode, inodeLockRequest: inodeLockRequest, exclusive: true, } - inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock globals.Unlock() case inodeLeaseStateSharedRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateSharedGranted: - inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) - inodeLease.leaseState = inodeLeaseStateSharedPromoting - _ = globals.sharedLeaseLRU.Remove(inodeLease.listElement) - inodeLease.listElement = globals.exclusiveLeaseLRU.PushBack(inodeLease) + inodeLockRequest.listElement = inode.requestList.PushFront(inodeLockRequest) + inode.leaseState = inodeLeaseStateSharedPromoting + _ = globals.sharedLeaseLRU.Remove(inode.listElement) + inode.listElement = globals.exclusiveLeaseLRU.PushBack(inode) globals.Unlock() @@ -235,35 +235,35 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { globals.Lock() - inodeLease.leaseState = inodeLeaseStateExclusiveGranted + inode.leaseState = inodeLeaseStateExclusiveGranted - if inodeLease.requestList.Front() != inodeLockRequest.listElement { - logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") + if inode.requestList.Front() != inodeLockRequest.listElement { + logFatalf("inode.requestList.Front() != inodeLockRequest.listElement") } - _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) + _ = inode.requestList.Remove(inodeLockRequest.listElement) inodeLockRequest.listElement = nil inodeHeldLock = &inodeHeldLockStruct{ - inodeLease: inodeLease, + inode: inode, inodeLockRequest: inodeLockRequest, exclusive: true, } - inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock globals.Unlock() case inodeLeaseStateSharedPromoting: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateSharedReleasing: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateSharedExpired: @@ -273,29 +273,29 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateExclusiveRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateExclusiveGranted: // We can immediately grant the exclusive inodeLockRequestStruct inodeHeldLock = &inodeHeldLockStruct{ - inodeLease: inodeLease, + inode: inode, inodeLockRequest: inodeLockRequest, exclusive: true, } - inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock globals.Unlock() case inodeLeaseStateExclusiveDemoting: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateExclusiveReleasing: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateExclusiveExpired: @@ -303,14 +303,14 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.unlockAllWhileLocked() globals.Unlock() default: - logFatalf("switch inodeLease.leaseState unexpected: %v", inodeLease.leaseState) + logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } } else { - switch inodeLease.leaseState { + switch inode.leaseState { case inodeLeaseStateNone: - inodeLockRequest.listElement = inodeLease.requestList.PushFront(inodeLockRequest) - inodeLease.leaseState = inodeLeaseStateSharedRequested - inodeLease.listElement = globals.sharedLeaseLRU.PushBack(inodeLease) + inodeLockRequest.listElement = inode.requestList.PushFront(inodeLockRequest) + inode.leaseState = inodeLeaseStateSharedRequested + inode.listElement = globals.sharedLeaseLRU.PushBack(inode) globals.Unlock() @@ -332,25 +332,25 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { globals.Lock() - inodeLease.leaseState = inodeLeaseStateSharedGranted + inode.leaseState = inodeLeaseStateSharedGranted - if inodeLease.requestList.Front() != inodeLockRequest.listElement { - logFatalf("inodeLease.requestList.Front() != inodeLockRequest.listElement") + if inode.requestList.Front() != inodeLockRequest.listElement { + logFatalf("inode.requestList.Front() != inodeLockRequest.listElement") } - _ = inodeLease.requestList.Remove(inodeLockRequest.listElement) + _ = inode.requestList.Remove(inodeLockRequest.listElement) inodeLockRequest.listElement = nil inodeHeldLock = &inodeHeldLockStruct{ - inodeLease: inodeLease, + inode: inode, inodeLockRequest: inodeLockRequest, exclusive: false, } - inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock - if inodeLease.requestList.Front() != nil { + if inode.requestList.Front() != nil { logFatalf("TODO: for now, we don't handle multiple shared lock requests queued up") } @@ -358,29 +358,29 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateSharedRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateSharedGranted: // We can immediately grant the shared inodeLockRequestStruct inodeHeldLock = &inodeHeldLockStruct{ - inodeLease: inodeLease, + inode: inode, inodeLockRequest: inodeLockRequest, exclusive: false, } - inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock globals.Unlock() case inodeLeaseStateSharedPromoting: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateSharedReleasing: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateSharedExpired: @@ -390,29 +390,29 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { case inodeLeaseStateExclusiveRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateExclusiveGranted: // We can immediately grant the shared inodeLockRequestStruct inodeHeldLock = &inodeHeldLockStruct{ - inodeLease: inodeLease, + inode: inode, inodeLockRequest: inodeLockRequest, exclusive: false, } - inodeHeldLock.listElement = inodeLease.heldList.PushBack(inodeHeldLock) + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock globals.Unlock() case inodeLeaseStateExclusiveDemoting: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateExclusiveReleasing: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) - inodeLockRequest.listElement = inodeLease.requestList.PushBack(inodeLockRequest) + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) globals.Unlock() inodeLockRequest.Wait() case inodeLeaseStateExclusiveExpired: @@ -420,7 +420,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.unlockAllWhileLocked() globals.Unlock() default: - logFatalf("switch inodeLease.leaseState unexpected: %v", inodeLease.leaseState) + logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } } } @@ -441,8 +441,8 @@ func (inodeLockRequest *inodeLockRequestStruct) unlockAllWhileLocked() { ) for _, inodeHeldLock = range inodeLockRequest.locksHeld { - _ = inodeHeldLock.inodeLease.heldList.Remove(inodeHeldLock.listElement) - if inodeHeldLock.inodeLease.requestList.Len() != 0 { + _ = inodeHeldLock.inode.heldList.Remove(inodeHeldLock.listElement) + if inodeHeldLock.inode.requestList.Len() != 0 { logFatalf("TODO: for now, we don't handle blocked inodeLockRequestStruct's") } } @@ -450,21 +450,21 @@ func (inodeLockRequest *inodeLockRequestStruct) unlockAllWhileLocked() { inodeLockRequest.locksHeld = make(map[uint64]*inodeHeldLockStruct) } -func lookupInodeLease(inodeNumber uint64) (inodeLease *inodeLeaseStruct) { +func lookupInode(inodeNumber uint64) (inode *inodeStruct) { globals.Lock() - inodeLease = lookupInodeLeaseWhileLocked(inodeNumber) + inode = lookupInodeWhileLocked(inodeNumber) globals.Unlock() return } -func lookupInodeLeaseWhileLocked(inodeNumber uint64) (inodeLease *inodeLeaseStruct) { +func lookupInodeWhileLocked(inodeNumber uint64) (inode *inodeStruct) { var ( ok bool ) - inodeLease, ok = globals.inodeLeaseTable[inodeNumber] + inode, ok = globals.inodeTable[inodeNumber] if !ok { - inodeLease = nil + inode = nil } return From 90fab14dd74fb7c466ce108a760309898e150a3e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 7 Oct 2021 16:31:30 -0700 Subject: [PATCH 115/258] Completed B+Tree loading and Layout Map conversion --- iclient/iclientpkg/inode.go | 51 ++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index d0bee2d0..31fa2fe2 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -114,61 +114,99 @@ func (inode *inodeStruct) GetNode(objectNumber uint64, objectOffset uint64, obje func (inode *inodeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: + err = fmt.Errorf("TODO") case ilayout.InodeTypeFile: + err = fmt.Errorf("TODO") default: err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - return // TODO + + return } func (inode *inodeStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: + err = fmt.Errorf("TODO") case ilayout.InodeTypeFile: + err = fmt.Errorf("TODO") default: err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - return // TODO + + return } func (inode *inodeStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: + err = fmt.Errorf("TODO") case ilayout.InodeTypeFile: + err = fmt.Errorf("TODO") default: err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - return // TODO + + return } func (inode *inodeStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + var ( + nextPos int + ) + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: + key, nextPos, err = ilayout.GetLEStringFromBuf(payloadData, 0) + if nil == err { + bytesConsumed = uint64(nextPos) + } case ilayout.InodeTypeFile: + key, nextPos, err = ilayout.GetLEUint64FromBuf(payloadData, 0) + if nil == err { + bytesConsumed = uint64(nextPos) + } default: err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - return // TODO + + return } func (inode *inodeStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: + err = fmt.Errorf("TODO") case ilayout.InodeTypeFile: + err = fmt.Errorf("TODO") default: err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - return // TODO + + return } func (inode *inodeStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + var ( + bytesConsumedAsInt int + ) + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: + value, bytesConsumedAsInt, err = ilayout.UnmarshalDirectoryEntryValueV1(payloadData) + if nil == err { + bytesConsumed = uint64(bytesConsumedAsInt) + } case ilayout.InodeTypeFile: + value, bytesConsumedAsInt, err = ilayout.UnmarshalExtentMapEntryValueV1(payloadData) + if nil == err { + bytesConsumed = uint64(bytesConsumedAsInt) + } default: err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - return // TODO + + return } func (inode *inodeStruct) newPayload() (err error) { @@ -247,7 +285,6 @@ func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { objectNumber uint64 ) - // TODO inode.inodeHeadV1.Layout = make([]ilayout.InodeHeadLayoutEntryV1Struct, len(inode.layoutMap)) for objectNumber, layoutMapEntry = range inode.layoutMap { From cec057196f98fbd03a0b59f7497342b57f9af44b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 11 Oct 2021 10:30:24 -0700 Subject: [PATCH 116/258] Picked up Golang 1.17.2 & updated package fission Also instantiated iclient's Read Cache --- Dockerfile | 2 +- go.mod | 4 +- go.sum | 14 ++----- iclient/iclientpkg/globals.go | 72 ++++++++++++++++++++++------------- 4 files changed, 53 insertions(+), 39 deletions(-) diff --git a/Dockerfile b/Dockerfile index f37b2778..53b08fa3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,7 +68,7 @@ # --env DISPLAY: tells Docker to set ENV DISPLAY for X apps (e.g. wireshark) FROM alpine:3.14.0 as base -ARG GolangVersion=1.17 +ARG GolangVersion=1.17.2 ARG MakeTarget RUN apk add --no-cache libc6-compat diff --git a/go.mod b/go.mod index 422992a8..d88cbe1c 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 - github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab + github.com/NVIDIA/fission v0.0.0-20211011170442-08d21745ac97 github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 github.com/ansel1/merry v1.6.1 github.com/creachadair/cityhash v0.1.0 @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf + golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac ) require ( diff --git a/go.sum b/go.sum index 65e380c4..316c1960 100644 --- a/go.sum +++ b/go.sum @@ -43,9 +43,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= -github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab h1:/t1tCaj3eWQXCRIme+JzSy+p78OPDjzNRirWAisWuVc= -github.com/NVIDIA/fission v0.0.0-20210818194118-c8072a1555ab/go.mod h1:cDh6DCOixUxCkPkyqNroWeSSSZpryqnmFKIylvc6aSE= -github.com/NVIDIA/sortedmap v0.0.0-20210818193114-64918d535a18/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= +github.com/NVIDIA/fission v0.0.0-20211011170442-08d21745ac97 h1:hKmuJ4MizJlef3SWX/cFEuu9wj09Ml9II0YCk5YEyJg= +github.com/NVIDIA/fission v0.0.0-20211011170442-08d21745ac97/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -295,7 +294,6 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -319,7 +317,6 @@ github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -621,6 +618,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -795,12 +794,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.0 h1:2t0h8NA59dpVQpa5Yh8cIcR6nHAeBIEk0zlLVqfw4N4= -gopkg.in/ini.v1 v1.63.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.1 h1:WlmD2fPTg4maPpRITalGs62TK7VMMtP5E9CHH7aFy6Y= -gopkg.in/ini.v1 v1.63.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 6b75a795..6e136f1f 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -188,33 +188,47 @@ type statsStruct struct { DoLSeekUsecs bucketstats.BucketLog2Round // (*globalsStruct)DoLSeek() } +type readCacheKeyStruct struct { + objectNumber uint64 + lineNumber uint64 +} + +type readCacheLineStruct struct { + sync.WaitGroup // Used by those needing to block while a prior accessor reads in .buf + key readCacheKeyStruct // + listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU + buf []byte // == nil if being read in +} + type globalsStruct struct { - sync.Mutex // - config configStruct // - fuseEntryValidDurationSec uint64 // - fuseEntryValidDurationNSec uint32 // - fuseAttrValidDurationSec uint64 // - fuseAttrValidDurationNSec uint32 // - logFile *os.File // == nil if config.LogFilePath == "" - retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" - httpClient *http.Client // - swiftRetryDelay []swiftRetryDelayElementStruct // - swiftAuthInString string // - swiftAuthWaitGroup *sync.WaitGroup // != nil if updateAuthTokenAndStorageURL() is active - swiftAuthToken string // - swiftStorageURL string // - retryRPCClientConfig *retryrpc.ClientConfig // - retryRPCClient *retryrpc.Client // - mountID string // - fissionErrChan chan error // - inodeTable map[uint64]*inodeStruct // - inodePayloadCache sortedmap.BPlusTreeCache // - sharedLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted - exclusiveLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted - httpServer *http.Server // - httpServerWG sync.WaitGroup // - stats *statsStruct // - fissionVolume fission.Volume // + sync.Mutex // + config configStruct // + fuseEntryValidDurationSec uint64 // + fuseEntryValidDurationNSec uint32 // + fuseAttrValidDurationSec uint64 // + fuseAttrValidDurationNSec uint32 // + logFile *os.File // == nil if config.LogFilePath == "" + retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" + httpClient *http.Client // + swiftRetryDelay []swiftRetryDelayElementStruct // + swiftAuthInString string // + swiftAuthWaitGroup *sync.WaitGroup // != nil if updateAuthTokenAndStorageURL() is active + swiftAuthToken string // + swiftStorageURL string // + retryRPCClientConfig *retryrpc.ClientConfig // + retryRPCClient *retryrpc.Client // + mountID string // + fissionErrChan chan error // + readCacheMap map[readCacheKeyStruct]readCacheLineStruct // + readCacheLRU *list.List // LRU-ordered list of readCacheLineStruct.listElement's + inodeTable map[uint64]*inodeStruct // + inodePayloadCache sortedmap.BPlusTreeCache // + sharedLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted + exclusiveLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted + httpServer *http.Server // + httpServerWG sync.WaitGroup // + stats *statsStruct // + fissionVolume fission.Volume // } var globals globalsStruct @@ -440,6 +454,9 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err globals.fissionErrChan = fissionErrChan + globals.readCacheMap = make(map[readCacheKeyStruct]readCacheLineStruct) + globals.readCacheLRU = list.New() + globals.inodeTable = make(map[uint64]*inodeStruct) globals.inodePayloadCache = sortedmap.NewBPlusTreeCache(globals.config.InodePayloadEvictLowLimit, globals.config.InodePayloadEvictHighLimit) globals.sharedLeaseLRU = list.New() @@ -500,6 +517,9 @@ func uninitializeGlobals() (err error) { globals.retryRPCCACertPEM = nil + globals.readCacheMap = nil + globals.readCacheLRU = nil + globals.inodeTable = nil globals.inodePayloadCache = nil globals.sharedLeaseLRU = nil From 44f31e664dd51d6ced83afe19928e44647467dcc Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 11 Oct 2021 18:25:01 -0700 Subject: [PATCH 117/258] Structural work on iclientpkg's DoRead() & DoWrite() FH (file handle) work in place... and enforced in DoRead() & DoWrite() Read and Write(back) Caches are almost done ExtentMap modifications in DoWrite() are a TODO LayoutMap needs to be instrumented in DoWrite() Flush (time, size, explicit) needs to be implemented [WIP]Implementing DoRead() in iclientpkg --- iclient/iclientpkg/fission.go | 280 +++++++++++++++++++++++++++++++++- iclient/iclientpkg/globals.go | 65 +++++--- iclient/iclientpkg/inode.go | 108 +++++++++++++ iclient/iclientpkg/lease.go | 84 ++++++---- 4 files changed, 482 insertions(+), 55 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 95278dbe..4b628120 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -8,6 +8,7 @@ import ( "time" "github.com/NVIDIA/fission" + "github.com/NVIDIA/sortedmap" "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/imgr/imgrpkg" @@ -125,10 +126,14 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis globals.stats.DoGetAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() +Retry: inodeLockRequest = newLockRequest() inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) inodeLockRequest.exclusive = false inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } inode = lookupInode(uint64(inHeader.NodeID)) if nil == inode { @@ -397,7 +402,23 @@ func (dummy *globalsStruct) DoOpen(inHeader *fission.InHeader, openIn *fission.O func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.ReadIn) (readOut *fission.ReadOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + curOffset uint64 + err error + extentMapEntryIndexV1 int + extentMapEntryIndexV1Max int // Entry entry at or just after where readIn.Offset+readIn.Size may reside + extentMapEntryIndexV1Min int // First entry at or just before where readIn.Offset may reside + extentMapEntryKeyV1 uint64 + extentMapEntryKeyV1AsKey sortedmap.Key + extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct + extentMapEntryValueV1AsValue sortedmap.Value + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + ok bool + openHandle *openHandleStruct + readPlan []*ilayout.ExtentMapEntryValueV1Struct + readPlanEntry *ilayout.ExtentMapEntryValueV1Struct // If .ObjectNumber == 0, .ObjectOffset is ignored... .Length is the number of zero fill bytes + remainingSize uint64 + startTime time.Time = time.Now() ) logTracef("==> DoRead(inHeader: %+v, readIn: %+v)", inHeader, readIn) @@ -413,15 +434,199 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R globals.stats.DoReadUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - readOut = nil - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + openHandle = lookupOpenHandleByInodeNumber(readIn.FH) + if nil == openHandle { + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.EBADF + return + } + if openHandle.inodeNumber != uint64(inHeader.NodeID) { + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.EBADF + return + } + if !openHandle.fissionFlagsRead { + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.EBADF + return + } + + inode = lookupInode(openHandle.inodeNumber) + if nil == inode { + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.ENOENT + return + } + + switch inode.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.EISDIR + return + case ilayout.InodeTypeFile: + // Fall through + case ilayout.InodeTypeSymLink: + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.EBADF + return + default: + logFatalf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + curOffset = readIn.Offset + if curOffset >= inode.inodeHeadV1.Size { + inodeLockRequest.unlockAll() + readOut = &fission.ReadOut{ + Data: make([]byte, 0, 0), + } + errno = 0 + return + } + + remainingSize = uint64(readIn.Size) + if (curOffset + remainingSize) > inode.inodeHeadV1.Size { + remainingSize = inode.inodeHeadV1.Size - curOffset + } + + readOut = &fission.ReadOut{ + Data: make([]byte, 0, remainingSize), + } + + if nil == inode.payload { + err = inode.oldPayload() + if nil != err { + logFatal(err) + } + } + + extentMapEntryIndexV1Min, _, err = inode.payload.BisectLeft(curOffset) + if nil != err { + logFatal(err) + } + + if extentMapEntryIndexV1Min < 0 { + extentMapEntryIndexV1Min = 0 + } + + extentMapEntryIndexV1Max, _, err = inode.payload.BisectLeft(readIn.Offset + remainingSize) + if nil != err { + logFatal(err) + } + + readPlan = make([]*ilayout.ExtentMapEntryValueV1Struct, 0, 2*(extentMapEntryIndexV1Max-extentMapEntryIndexV1Min)) + + for extentMapEntryIndexV1 = extentMapEntryIndexV1Min; extentMapEntryIndexV1 < extentMapEntryIndexV1Max; extentMapEntryIndexV1++ { + extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, _, err = inode.payload.GetByIndex(extentMapEntryIndexV1) + if nil != err { + logFatal(err) + } + + extentMapEntryKeyV1, ok = extentMapEntryKeyV1AsKey.(uint64) + if !ok { + logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") + } + + if curOffset < extentMapEntryKeyV1 { + if remainingSize < (extentMapEntryKeyV1 - curOffset) { + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: remainingSize, + ObjectNumber: 0, + ObjectOffset: 0, + }) + + curOffset += remainingSize + remainingSize = 0 + + break + } else { + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: extentMapEntryKeyV1 - curOffset, + ObjectNumber: 0, + ObjectOffset: 0, + }) + + remainingSize -= extentMapEntryKeyV1 - curOffset + curOffset = extentMapEntryKeyV1 + } + } + + extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) + if !ok { + logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1) returned !ok") + } + + if remainingSize <= extentMapEntryValueV1.Length { + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: remainingSize, + ObjectNumber: extentMapEntryValueV1.ObjectNumber, + ObjectOffset: extentMapEntryValueV1.ObjectOffset, + }) + + curOffset += remainingSize + remainingSize = 0 + + break + } + + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: extentMapEntryValueV1.Length, + ObjectNumber: extentMapEntryValueV1.ObjectNumber, + ObjectOffset: extentMapEntryValueV1.ObjectOffset, + }) + + curOffset += extentMapEntryValueV1.Length + remainingSize -= extentMapEntryValueV1.Length + } + + if remainingSize > 0 { + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: remainingSize, + ObjectNumber: 0, + ObjectOffset: 0, + }) + } + + for _, readPlanEntry = range readPlan { + switch readPlanEntry.ObjectNumber { + case 0: + readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length, readPlanEntry.Length)...) + case inode.putObjectNumber: + readOut.Data = append(readOut.Data, inode.putObjectBuffer[readPlanEntry.ObjectOffset:(readPlanEntry.ObjectOffset+readPlanEntry.Length)]...) + default: + // TODO - need to actually read from the cache (in a coherent/cooperative way) + // TODO - need to handle case where extent crosses cache line boundary + // UNDO - but for now, we will simply append zeroes + readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length, readPlanEntry.Length)...) // UNDO + } + } + + inodeLockRequest.unlockAll() + + errno = 0 return } func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission.WriteIn) (writeOut *fission.WriteOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoWrite(inHeader: %+v, writeIn: &{FH:%v Offset:%v Size:%v: WriteFlags:%v LockOwner:%v Flags:%v Padding:%v len(Data):%v})", inHeader, writeIn.FH, writeIn.Offset, writeIn.Size, writeIn.WriteFlags, writeIn.LockOwner, writeIn.Flags, writeIn.Padding, len(writeIn.Data)) @@ -433,8 +638,69 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission globals.stats.DoWriteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - writeOut = nil +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + openHandle = lookupOpenHandleByInodeNumber(writeIn.FH) + if nil == openHandle { + inodeLockRequest.unlockAll() + writeOut = nil + errno = syscall.EBADF + return + } + if openHandle.inodeNumber != uint64(inHeader.NodeID) { + inodeLockRequest.unlockAll() + writeOut = nil + errno = syscall.EBADF + return + } + if !openHandle.fissionFlagsWrite { + inodeLockRequest.unlockAll() + writeOut = nil + errno = syscall.EBADF + return + } + + inode = lookupInode(openHandle.inodeNumber) + if nil == inode { + inodeLockRequest.unlockAll() + writeOut = nil + errno = syscall.ENOENT + return + } + + switch inode.inodeHeadV1.InodeType { + case ilayout.InodeTypeDir: + inodeLockRequest.unlockAll() + writeOut = nil + errno = syscall.EISDIR + return + case ilayout.InodeTypeFile: + // Fall through + case ilayout.InodeTypeSymLink: + inodeLockRequest.unlockAll() + writeOut = nil + errno = syscall.EBADF + return + default: + logFatalf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + // TODO - need to patch ExtentMap + // TODO - need to append to inode.putObjectBuffer + // TODO - need to detect combinable ExtentMap entries + // TODO - need to trigger a flush if len(payload.putObjectBuffer) >= globals.config.FileFlushTriggerSize + // TODO - need to (possibly) trigger new timer after globals.config.FileFlushTriggerDuration + + inodeLockRequest.unlockAll() + + writeOut = nil // UNDO errno = syscall.ENOSYS return } diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 6e136f1f..09cd5a6b 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -89,14 +89,24 @@ type layoutMapEntryStruct struct { bytesReferenced uint64 } +type openHandleStruct struct { + inodeNumber uint64 + fissionFH uint64 // As returned in fission.{CreateOut|OpenOut|OpenDirOut}.FH + fissionFlagsAppend bool // As supplied in fission.{CreateIn|OpenIn}.Flags [true if (.Flags & syscall.O_APPEND) == syscall.O_APPEND] + fissionFlagsRead bool // As supplied in fission.{CreateIn|OpenIn|OpenDirIn}.Flags [true if (.Flags & (syscall.O_RDONLY|syscall.O_RDWR|syscall.O_WRONLY) is one of syscall.{O_RDONLY|O_RDWR}] + fissionFlagsWrite bool // As supplied in fission.{CreateIn|OpenIn|OpenDirIn}.Flags [true if (.Flags & (syscall.O_RDONLY|syscall.O_RDWR|syscall.O_WRONLY) is one of syscall.{O_RDWR|O_WRONLY}] +} + type inodeStruct struct { - inodeNumber uint64 // - leaseState inodeLeaseStateType // - listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU - heldList *list.List // List of granted inodeHeldLockStruct's - requestList *list.List // List of pending inodeLockRequestStruct's - inodeHeadV1 *ilayout.InodeHeadV1Struct // - payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} + sync.WaitGroup // Signaled to indicate .markedForDelete == true triggered removal has completed + inodeNumber uint64 // + markedForDelete bool // If true, remove from globalsStruct.inodeTable upon last dereference + leaseState inodeLeaseStateType // + listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU + heldList *list.List // List of granted inodeHeldLockStruct's + requestList *list.List // List of pending inodeLockRequestStruct's + inodeHeadV1 *ilayout.InodeHeadV1Struct // + payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} // For FileInode: ExtentMap B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} layoutMap map[uint64]layoutMapEntryStruct // For DirInode & FileInode: Map form of .inodeHeadV1.Layout superBlockInodeObjectCountAdjustment int64 // @@ -124,6 +134,18 @@ type inodeLockRequestStruct struct { // or is empty indicating the caller should restart lock request sequence } +type readCacheKeyStruct struct { + objectNumber uint64 + lineNumber uint64 +} + +type readCacheLineStruct struct { + sync.WaitGroup // Used by those needing to block while a prior accessor reads in .buf + key readCacheKeyStruct // + listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU + buf []byte // == nil if being read in +} + type statsStruct struct { GetConfigUsecs bucketstats.BucketLog2Round // GET /config GetLeasesUsecs bucketstats.BucketLog2Round // GET /leases @@ -188,18 +210,6 @@ type statsStruct struct { DoLSeekUsecs bucketstats.BucketLog2Round // (*globalsStruct)DoLSeek() } -type readCacheKeyStruct struct { - objectNumber uint64 - lineNumber uint64 -} - -type readCacheLineStruct struct { - sync.WaitGroup // Used by those needing to block while a prior accessor reads in .buf - key readCacheKeyStruct // - listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU - buf []byte // == nil if being read in -} - type globalsStruct struct { sync.Mutex // config configStruct // @@ -219,10 +229,15 @@ type globalsStruct struct { retryRPCClient *retryrpc.Client // mountID string // fissionErrChan chan error // + nonceWaitGroup *sync.WaitGroup // != nil if rpcFetchNonceRange() already underway + nextNonce uint64 // + noncesRemaining uint64 // readCacheMap map[readCacheKeyStruct]readCacheLineStruct // readCacheLRU *list.List // LRU-ordered list of readCacheLineStruct.listElement's inodeTable map[uint64]*inodeStruct // inodePayloadCache sortedmap.BPlusTreeCache // + openHandleMapByInodeNumber map[uint64]*openHandleStruct // Key == openHandleStruct.inodeNumber + openHandleMapByFissionFH map[uint64]*openHandleStruct // Key == openHandleStruct.fissionFH sharedLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted exclusiveLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted httpServer *http.Server // @@ -454,11 +469,17 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err globals.fissionErrChan = fissionErrChan + globals.nonceWaitGroup = nil + globals.nextNonce = 0 + globals.noncesRemaining = 0 + globals.readCacheMap = make(map[readCacheKeyStruct]readCacheLineStruct) globals.readCacheLRU = list.New() globals.inodeTable = make(map[uint64]*inodeStruct) globals.inodePayloadCache = sortedmap.NewBPlusTreeCache(globals.config.InodePayloadEvictLowLimit, globals.config.InodePayloadEvictHighLimit) + globals.openHandleMapByInodeNumber = make(map[uint64]*openHandleStruct) + globals.openHandleMapByFissionFH = make(map[uint64]*openHandleStruct) globals.sharedLeaseLRU = list.New() globals.exclusiveLeaseLRU = list.New() @@ -522,11 +543,17 @@ func uninitializeGlobals() (err error) { globals.inodeTable = nil globals.inodePayloadCache = nil + globals.openHandleMapByInodeNumber = nil + globals.openHandleMapByFissionFH = nil globals.sharedLeaseLRU = nil globals.exclusiveLeaseLRU = nil globals.fissionErrChan = nil + globals.nonceWaitGroup = nil + globals.nextNonce = 0 + globals.noncesRemaining = 0 + bucketstats.UnRegister("ICLIENT", "") err = nil diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 31fa2fe2..411544f2 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -6,10 +6,12 @@ package iclientpkg import ( "encoding/json" "fmt" + "sync" "github.com/NVIDIA/sortedmap" "github.com/NVIDIA/proxyfs/ilayout" + "github.com/NVIDIA/proxyfs/imgr/imgrpkg" ) func (inode *inodeStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { @@ -209,6 +211,112 @@ func (inode *inodeStruct) UnpackValue(payloadData []byte) (value sortedmap.Value return } +func lookupInode(inodeNumber uint64) (inode *inodeStruct) { + var ( + ok bool + ) + + globals.Lock() + + inode, ok = globals.inodeTable[inodeNumber] + if !ok { + inode = nil + } + + globals.Unlock() + + return +} + +func lookupOpenHandleByInodeNumber(inodeNumber uint64) (openHandle *openHandleStruct) { + var ( + ok bool + ) + + globals.Lock() + + openHandle, ok = globals.openHandleMapByInodeNumber[inodeNumber] + if !ok { + openHandle = nil + } + + globals.Unlock() + + return +} + +func lookupOpenHandleByFissionFH(fissionFH uint64) (openHandle *openHandleStruct) { + var ( + ok bool + ) + + globals.Lock() + + openHandle, ok = globals.openHandleMapByFissionFH[fissionFH] + if !ok { + openHandle = nil + } + + globals.Unlock() + + return +} + +func fetchNonce() (nonceToReturn uint64) { + var ( + err error + fetchNonceRangeRequest *imgrpkg.FetchNonceRangeRequestStruct + fetchNonceRangeResponse *imgrpkg.FetchNonceRangeResponseStruct + nonceWaitGroup *sync.WaitGroup + ) + +Retry: + + globals.Lock() + + if globals.noncesRemaining > 0 { + nonceToReturn = globals.nextNonce + globals.nextNonce++ + globals.noncesRemaining-- + globals.Unlock() + return + } + + nonceWaitGroup = globals.nonceWaitGroup + if nil != nonceWaitGroup { + globals.Unlock() + nonceWaitGroup.Wait() + goto Retry + } + + globals.nonceWaitGroup = &sync.WaitGroup{} + globals.nonceWaitGroup.Add(1) + + globals.Unlock() + + fetchNonceRangeRequest = &imgrpkg.FetchNonceRangeRequestStruct{ + MountID: globals.mountID, + } + fetchNonceRangeResponse = &imgrpkg.FetchNonceRangeResponseStruct{} + + err = rpcFetchNonceRange(fetchNonceRangeRequest, fetchNonceRangeResponse) + if nil != err { + logFatal(err) + } + + globals.Lock() + + globals.nextNonce = fetchNonceRangeResponse.NextNonce + globals.noncesRemaining = fetchNonceRangeResponse.NumNoncesFetched + + globals.nonceWaitGroup.Done() + globals.nonceWaitGroup = nil + + globals.Unlock() + + goto Retry +} + func (inode *inodeStruct) newPayload() (err error) { switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 5627f395..425ec8ce 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -54,6 +54,8 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { ok bool ) +Retry: + globals.Lock() if inodeLockRequest.inodeNumber == 0 { @@ -66,6 +68,12 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inode, ok = globals.inodeTable[inodeLockRequest.inodeNumber] if ok { + if inode.markedForDelete { + globals.Unlock() + inode.Wait() + goto Retry + } + switch inode.leaseState { case inodeLeaseStateNone: case inodeLeaseStateSharedRequested: @@ -90,6 +98,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } else { inode = &inodeStruct{ inodeNumber: inodeLockRequest.inodeNumber, + markedForDelete: false, leaseState: inodeLeaseStateNone, listElement: nil, heldList: list.New(), @@ -268,8 +277,8 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.Wait() case inodeLeaseStateSharedExpired: // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up - inodeLockRequest.unlockAllWhileLocked() globals.Unlock() + inodeLockRequest.unlockAll() case inodeLeaseStateExclusiveRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) @@ -300,8 +309,8 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.Wait() case inodeLeaseStateExclusiveExpired: // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up - inodeLockRequest.unlockAllWhileLocked() globals.Unlock() + inodeLockRequest.unlockAll() default: logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } @@ -385,8 +394,8 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.Wait() case inodeLeaseStateSharedExpired: // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up - inodeLockRequest.unlockAllWhileLocked() globals.Unlock() + inodeLockRequest.unlockAll() case inodeLeaseStateExclusiveRequested: // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct inodeLockRequest.Add(1) @@ -417,8 +426,8 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeLockRequest.Wait() case inodeLeaseStateExclusiveExpired: // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up - inodeLockRequest.unlockAllWhileLocked() globals.Unlock() + inodeLockRequest.unlockAll() default: logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } @@ -428,44 +437,61 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { // unlockAll is called to explicitly release all locks listed in the locksHeld map. // func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { - globals.Lock() - inodeLockRequest.unlockAllWhileLocked() - globals.Unlock() -} - -// unlockAllWhileLocked is what unlockAll calls after obtaining globals.Lock(). -// -func (inodeLockRequest *inodeLockRequestStruct) unlockAllWhileLocked() { var ( + err error + inode *inodeStruct inodeHeldLock *inodeHeldLockStruct + leaseRequest *imgrpkg.LeaseRequestStruct + leaseResponse *imgrpkg.LeaseResponseStruct + releaseList []*inodeStruct = make([]*inodeStruct, 0) ) + globals.Lock() + for _, inodeHeldLock = range inodeLockRequest.locksHeld { _ = inodeHeldLock.inode.heldList.Remove(inodeHeldLock.listElement) - if inodeHeldLock.inode.requestList.Len() != 0 { + if inodeHeldLock.inode.requestList.Len() == 0 { + if inodeHeldLock.inode.heldList.Len() == 0 { + if inodeHeldLock.inode.markedForDelete { + releaseList = append(releaseList, inodeHeldLock.inode) + delete(globals.inodeTable, inodeHeldLock.inode.inodeNumber) + } + } + } else { logFatalf("TODO: for now, we don't handle blocked inodeLockRequestStruct's") } } - inodeLockRequest.locksHeld = make(map[uint64]*inodeHeldLockStruct) -} - -func lookupInode(inodeNumber uint64) (inode *inodeStruct) { - globals.Lock() - inode = lookupInodeWhileLocked(inodeNumber) globals.Unlock() - return -} -func lookupInodeWhileLocked(inodeNumber uint64) (inode *inodeStruct) { - var ( - ok bool - ) + for _, inode = range releaseList { + switch inode.leaseState { + case inodeLeaseStateSharedGranted: + inode.leaseState = inodeLeaseStateSharedReleasing + case inodeLeaseStateExclusiveGranted: + inode.leaseState = inodeLeaseStateExclusiveReleasing + default: + logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) + } + + leaseRequest = &imgrpkg.LeaseRequestStruct{ + MountID: globals.mountID, + InodeNumber: inodeLockRequest.inodeNumber, + LeaseRequestType: imgrpkg.LeaseRequestTypeRelease, + } + leaseResponse = &imgrpkg.LeaseResponseStruct{} + + err = rpcLease(leaseRequest, leaseResponse) + if nil != err { + logFatal(err) + } - inode, ok = globals.inodeTable[inodeNumber] - if !ok { - inode = nil + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeReleased { + logFatalf("received unexpected leaseResponse.LeaseResponseType: %v", leaseResponse.LeaseResponseType) + } + + inode.Done() } - return + inodeLockRequest.locksHeld = make(map[uint64]*inodeHeldLockStruct) } From 46988f677fd7a07fde80e957c33a36aa76e1da66 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 10:49:10 -0700 Subject: [PATCH 118/258] Optimized logic in iclientpkg/fission.go to ideally only get shared locks if sufficient --- go.mod | 2 +- go.sum | 2 + iclient/iclientpkg/fission.go | 78 +++++++++++++++++++---------------- iclient/iclientpkg/inode.go | 52 +++++++++++++++++++++++ 4 files changed, 97 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index d88cbe1c..cb6dae23 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac + golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 ) require ( diff --git a/go.sum b/go.sum index 316c1960..afd55354 100644 --- a/go.sum +++ b/go.sum @@ -620,6 +620,8 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0NHHxO3Z0g7Gus2O1kk= +golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 4b628120..da57c69f 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -11,7 +11,6 @@ import ( "github.com/NVIDIA/sortedmap" "github.com/NVIDIA/proxyfs/ilayout" - "github.com/NVIDIA/proxyfs/imgr/imgrpkg" ) const ( @@ -104,17 +103,14 @@ func (dummy *globalsStruct) DoForget(inHeader *fission.InHeader, forgetIn *fissi func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fission.GetAttrIn) (getAttrOut *fission.GetAttrOut, errno syscall.Errno) { var ( - err error - getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct - getInodeTableEntryResponse *imgrpkg.GetInodeTableEntryResponseStruct - inode *inodeStruct - inodeHeadV1Buf []byte - inodeLockRequest *inodeLockRequestStruct - modificationTimeNSec uint32 - modificationTimeSec uint64 - startTime time.Time = time.Now() - statusChangeTimeNSec uint32 - statusChangeTimeSec uint64 + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + modificationTimeNSec uint32 + modificationTimeSec uint64 + obtainExclusiveLock bool + startTime time.Time = time.Now() + statusChangeTimeNSec uint32 + statusChangeTimeSec uint64 ) logTracef("==> DoGetAttr(inHeader: %+v, getAttrIn: %+v)", inHeader, getAttrIn) @@ -126,10 +122,12 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis globals.stats.DoGetAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + obtainExclusiveLock = false + Retry: inodeLockRequest = newLockRequest() inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) - inodeLockRequest.exclusive = false + inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { goto Retry @@ -143,30 +141,16 @@ Retry: return } - if nil == inode.inodeHeadV1 { - getInodeTableEntryRequest = &imgrpkg.GetInodeTableEntryRequestStruct{ - MountID: globals.mountID, - InodeNumber: uint64(inHeader.NodeID), - } - getInodeTableEntryResponse = &imgrpkg.GetInodeTableEntryResponseStruct{} - - err = rpcGetInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) - if nil != err { + obtainExclusiveLock, errno = inode.ensureInodeHeadV1NonNil(obtainExclusiveLock) + if errno == 0 { + if obtainExclusiveLock { inodeLockRequest.unlockAll() - getAttrOut = nil - errno = syscall.ENOENT - return - } - - inodeHeadV1Buf, err = objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength) - if nil != err { - logFatalf("objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber: %v, getInodeTableEntryResponse.InodeHeadLength: %v) failed: %v", getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength, err) - } - - inode.inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) - if nil != err { - logFatalf("ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) failed: %v", err) + goto Retry } + } else { + inodeLockRequest.unlockAll() + getAttrOut = nil + return } modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.ModificationTime.UnixNano())) @@ -413,6 +397,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R extentMapEntryValueV1AsValue sortedmap.Value inode *inodeStruct inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool ok bool openHandle *openHandleStruct readPlan []*ilayout.ExtentMapEntryValueV1Struct @@ -434,10 +419,12 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R globals.stats.DoReadUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + obtainExclusiveLock = false + Retry: inodeLockRequest = newLockRequest() inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) - inodeLockRequest.exclusive = true + inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { goto Retry @@ -471,6 +458,18 @@ Retry: return } + obtainExclusiveLock, errno = inode.ensureInodeHeadV1NonNil(obtainExclusiveLock) + if errno == 0 { + if obtainExclusiveLock { + inodeLockRequest.unlockAll() + goto Retry + } + } else { + inodeLockRequest.unlockAll() + readOut = nil + return + } + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: inodeLockRequest.unlockAll() @@ -675,6 +674,13 @@ Retry: return } + _, errno = inode.ensureInodeHeadV1NonNil(true) + if errno != 0 { + inodeLockRequest.unlockAll() + writeOut = nil + return + } + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: inodeLockRequest.unlockAll() diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 411544f2..eca496a5 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "sync" + "syscall" "github.com/NVIDIA/sortedmap" @@ -401,3 +402,54 @@ func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced } } + +// ensureInodeHeadV1NonNil will ensure inode.inodeHeadV1 is non-nil. As such, if it is +// currently nil at entry, ensureInodeHeadV1NonNil() will need exclusive access to the +// inode. In this case, if the exclusive lock is not held, ensureInodeHeadV1NonNil() +// will return without error but indicate an exclusive lock is needed. Note that if +// inode.inodeHeadV1 was non-nil or an exclusive lock was already held, ensureInodeHeadV1NonNil() +// will return that it (no longer) needs an exclusive lock and without error. +// +func (inode *inodeStruct) ensureInodeHeadV1NonNil(exclusivelyLocked bool) (needExclusiveLock bool, errno syscall.Errno) { + var ( + err error + getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct + getInodeTableEntryResponse *imgrpkg.GetInodeTableEntryResponseStruct + inodeHeadV1Buf []byte + ) + + if nil == inode.inodeHeadV1 { + if exclusivelyLocked { + needExclusiveLock = false + + getInodeTableEntryRequest = &imgrpkg.GetInodeTableEntryRequestStruct{ + MountID: globals.mountID, + InodeNumber: inode.inodeNumber, + } + getInodeTableEntryResponse = &imgrpkg.GetInodeTableEntryResponseStruct{} + + err = rpcGetInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) + if nil != err { + errno = syscall.ENOENT + return + } + + inodeHeadV1Buf, err = objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength) + if nil != err { + logFatalf("objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber: %v, getInodeTableEntryResponse.InodeHeadLength: %v) failed: %v", getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength, err) + } + + inode.inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) + if nil != err { + logFatalf("ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) failed: %v", err) + } + } else { + needExclusiveLock = true + } + } else { + needExclusiveLock = false + } + + errno = 0 + return +} From 2505ea48660db8f999540761375187c54719a6a5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 11:11:05 -0700 Subject: [PATCH 119/258] Added creation/destruction of Open Handles in iclientpkg --- iclient/iclientpkg/inode.go | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index eca496a5..1132bfe1 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -229,6 +229,45 @@ func lookupInode(inodeNumber uint64) (inode *inodeStruct) { return } +func createOpenHandle(inodeNumber uint64) (openHandle *openHandleStruct) { + openHandle = &openHandleStruct{ + inodeNumber: inodeNumber, + fissionFH: fetchNonce(), + } + + globals.Lock() + + globals.openHandleMapByInodeNumber[openHandle.inodeNumber] = openHandle + globals.openHandleMapByFissionFH[openHandle.fissionFH] = openHandle + + globals.Unlock() + + return +} + +func (openHandle *openHandleStruct) destroy() { + var ( + ok bool + ) + + globals.Lock() + + _, ok = globals.openHandleMapByInodeNumber[openHandle.inodeNumber] + if !ok { + logFatalf("globals.openHandleMapByInodeNumber[openHandle.inodeNumber] returned !ok") + } + + _, ok = globals.openHandleMapByFissionFH[openHandle.fissionFH] + if !ok { + logFatalf("globals.openHandleMapByFissionFH[openHandle.fissionFH] returned !ok") + } + + delete(globals.openHandleMapByInodeNumber, openHandle.inodeNumber) + delete(globals.openHandleMapByFissionFH, openHandle.fissionFH) + + globals.Unlock() +} + func lookupOpenHandleByInodeNumber(inodeNumber uint64) (openHandle *openHandleStruct) { var ( ok bool From eb6e942745184f79edc4eaf73e0d9c5cd1feae4c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 14:38:43 -0700 Subject: [PATCH 120/258] Point GitHub "Build-and-unit-test" to standard golang image --- .github/workflows/run-unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 0aa4c36d..a9d486fb 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -9,7 +9,7 @@ jobs: fetch-depth: 0 submodules: false - name: Build, run unit tests, and get coverage - run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs swiftstack/proxyfs_unit_tests + run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 - name: Slack Notification uses: rtCamp/action-slack-notify@v2 env: From 7275eb523c198961e3d3eb52156ebf9a08abb695 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 14:56:19 -0700 Subject: [PATCH 121/258] Experimenting with how to see what is in the golang:1.17.2 container during unit testing --- .github/workflows/run-unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index a9d486fb..c3e2c4f8 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -9,7 +9,7 @@ jobs: fetch-depth: 0 submodules: false - name: Build, run unit tests, and get coverage - run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 + run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "ls -alR" - name: Slack Notification uses: rtCamp/action-slack-notify@v2 env: From b2f61824ead63e243d15df60e24a65f2d92545f9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 14:59:06 -0700 Subject: [PATCH 122/258] Gotta look above /go... --- .github/workflows/run-unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index c3e2c4f8..6fddb1a6 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -9,7 +9,7 @@ jobs: fetch-depth: 0 submodules: false - name: Build, run unit tests, and get coverage - run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "ls -alR" + run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "ls -alR /" - name: Slack Notification uses: rtCamp/action-slack-notify@v2 env: From a49f070dabc73aec4be548e60550399285d36810 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 15:00:46 -0700 Subject: [PATCH 123/258] And again... I think the source tree is now here... --- .github/workflows/run-unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 6fddb1a6..c5905e77 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -9,7 +9,7 @@ jobs: fetch-depth: 0 submodules: false - name: Build, run unit tests, and get coverage - run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "ls -alR /" + run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "cd /gopathroot/src/github.com/NVIDIA/proxyfs ; ls" - name: Slack Notification uses: rtCamp/action-slack-notify@v2 env: From c5d8f2de881877199da4c322757e4baeac3cec40 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 15:02:21 -0700 Subject: [PATCH 124/258] OK.. I think I've got it... --- .github/workflows/run-unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index c5905e77..aa9084e3 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -9,7 +9,7 @@ jobs: fetch-depth: 0 submodules: false - name: Build, run unit tests, and get coverage - run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "cd /gopathroot/src/github.com/NVIDIA/proxyfs ; ls" + run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "cd /gopathroot/src/github.com/NVIDIA/proxyfs ; make ci" - name: Slack Notification uses: rtCamp/action-slack-notify@v2 env: From 3c3bf11a607bacb90266d1b067128b44989476c5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 15:54:54 -0700 Subject: [PATCH 125/258] Pickup support for setting default_permissions on FUSE volume ...this means that VFS/FUSE themselves perform all permission checks... iclientpkg need not --- go.mod | 4 ++-- go.sum | 10 ++++------ iclient/iclientpkg/fission.go | 6 +++++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index cb6dae23..2a444fa7 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 - github.com/NVIDIA/fission v0.0.0-20211011170442-08d21745ac97 + github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066 github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 github.com/ansel1/merry v1.6.1 github.com/creachadair/cityhash v0.1.0 @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 + golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 ) require ( diff --git a/go.sum b/go.sum index afd55354..8d268e72 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= -github.com/NVIDIA/fission v0.0.0-20211011170442-08d21745ac97 h1:hKmuJ4MizJlef3SWX/cFEuu9wj09Ml9II0YCk5YEyJg= -github.com/NVIDIA/fission v0.0.0-20211011170442-08d21745ac97/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= +github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066 h1:K/8Os/A5UFB6mJiQaeGJtXjvb4sOLm6xj7ulQxqUACw= +github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -616,12 +616,10 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0NHHxO3Z0g7Gus2O1kk= golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index da57c69f..8cebf4dd 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -17,6 +17,8 @@ const ( attrBlockSize = uint32(512) attrRDev = uint32(0) + fuseDefaultPermissions = true // Make VFS/FUSE do access checks rather than this driver + fuseSubtype = "ProxyFS" initOutFlags = uint32(0) | @@ -38,6 +40,7 @@ func performMountFUSE() (err error) { globals.config.MountPointDirPath, fuseSubtype, globals.config.FUSEMaxWrite, + fuseDefaultPermissions, globals.config.FUSEAllowOther, &globals, newLogger(), @@ -1043,7 +1046,8 @@ func (dummy *globalsStruct) DoAccess(inHeader *fission.InHeader, accessIn *fissi globals.stats.DoAccessUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO + // Note that with setting defaultPermissions to true, this call should never be made + errno = syscall.ENOSYS return } From 478681c5d8555e75b33ac111e504a7241029504e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 16:24:09 -0700 Subject: [PATCH 126/258] Properly declare "default" (and about to be overwritten) openHandleStruct's --- iclient/iclientpkg/fission.go | 6 +++--- iclient/iclientpkg/inode.go | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 8cebf4dd..cd4b1a73 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -144,7 +144,7 @@ Retry: return } - obtainExclusiveLock, errno = inode.ensureInodeHeadV1NonNil(obtainExclusiveLock) + obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) if errno == 0 { if obtainExclusiveLock { inodeLockRequest.unlockAll() @@ -461,7 +461,7 @@ Retry: return } - obtainExclusiveLock, errno = inode.ensureInodeHeadV1NonNil(obtainExclusiveLock) + obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) if errno == 0 { if obtainExclusiveLock { inodeLockRequest.unlockAll() @@ -677,7 +677,7 @@ Retry: return } - _, errno = inode.ensureInodeHeadV1NonNil(true) + _, errno = inode.ensureInodeHeadV1IsNonNil(true) if errno != 0 { inodeLockRequest.unlockAll() writeOut = nil diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 1132bfe1..569d92cc 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -231,8 +231,11 @@ func lookupInode(inodeNumber uint64) (inode *inodeStruct) { func createOpenHandle(inodeNumber uint64) (openHandle *openHandleStruct) { openHandle = &openHandleStruct{ - inodeNumber: inodeNumber, - fissionFH: fetchNonce(), + inodeNumber: inodeNumber, + fissionFH: fetchNonce(), + fissionFlagsAppend: false, // To be filled in by caller + fissionFlagsRead: false, // To be filled in by caller + fissionFlagsWrite: false, // To be filled in by caller } globals.Lock() @@ -442,14 +445,15 @@ func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { } } -// ensureInodeHeadV1NonNil will ensure inode.inodeHeadV1 is non-nil. As such, if it is -// currently nil at entry, ensureInodeHeadV1NonNil() will need exclusive access to the -// inode. In this case, if the exclusive lock is not held, ensureInodeHeadV1NonNil() -// will return without error but indicate an exclusive lock is needed. Note that if -// inode.inodeHeadV1 was non-nil or an exclusive lock was already held, ensureInodeHeadV1NonNil() +// ensureInodeHeadV1IsNonNil will ensure inode.inodeHeadV1 is non-nil. As +// such, if it is currently nil at entry, ensureInodeHeadV1IsNonNil() will +// need exclusive access to the inode. In this case, if the exclusive lock +// is not held, ensureInodeHeadV1IsNonNil() will return without error but +// indicate an exclusive lock is needed. Note that if inode.inodeHeadV1 was +// non-nil or an exclusive lock was already held, ensureInodeHeadV1IsNonNil() // will return that it (no longer) needs an exclusive lock and without error. // -func (inode *inodeStruct) ensureInodeHeadV1NonNil(exclusivelyLocked bool) (needExclusiveLock bool, errno syscall.Errno) { +func (inode *inodeStruct) ensureInodeHeadV1IsNonNil(exclusivelyLocked bool) (needExclusiveLock bool, errno syscall.Errno) { var ( err error getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct From 8fceecb86666476d12d41ca9cf3ad4a00a7e1e17 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 19 Oct 2021 16:31:35 -0700 Subject: [PATCH 127/258] Made run-unit-tests.yaml get all required steps in corresponding .sh file --- .github/workflows/run-unit-tests.sh | 18 ++++++++++++++++++ .github/workflows/run-unit-tests.yml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100755 .github/workflows/run-unit-tests.sh diff --git a/.github/workflows/run-unit-tests.sh b/.github/workflows/run-unit-tests.sh new file mode 100755 index 00000000..61512f30 --- /dev/null +++ b/.github/workflows/run-unit-tests.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +set -e +set -x + +cd /gopathroot/src/github.com/NVIDIA/proxyfs + +go get github.com/mattn/goveralls + +make ci + +# $COVERALLS_TOKEN must be configured in the CI/CD environment +if [ -n "$COVERALLS_TOKEN" ] && [ -n "$GIT_BRANCH" ]; then + goveralls -coverprofile coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN || true +fi diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index aa9084e3..491663df 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -9,7 +9,7 @@ jobs: fetch-depth: 0 submodules: false - name: Build, run unit tests, and get coverage - run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "cd /gopathroot/src/github.com/NVIDIA/proxyfs ; make ci" + run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs golang:1.17.2 /bin/bash -c "/gopathroot/src/github.com/NVIDIA/proxyfs/.github/workflows/run-unit-tests.sh" - name: Slack Notification uses: rtCamp/action-slack-notify@v2 env: From 06d2a2f01901007e2d4b7e36443af01d47c2bdba Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 21 Oct 2021 13:36:34 -0700 Subject: [PATCH 128/258] Do{Open|Release}{|Dir}() now properly manage FH's Also added RPC & Auth tracing --- go.mod | 2 +- go.sum | 2 + iclient/iclientpkg/fission.go | 329 ++++++++++++++++++++++++++++++---- iclient/iclientpkg/globals.go | 4 +- iclient/iclientpkg/inode.go | 4 + iclient/iclientpkg/rpc.go | 54 ++++++ 6 files changed, 360 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index 2a444fa7..03911183 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 + golang.org/x/sys v0.0.0-20211020174200-9d6173849985 ) require ( diff --git a/go.sum b/go.sum index 8d268e72..cc93a7fd 100644 --- a/go.sum +++ b/go.sum @@ -620,6 +620,8 @@ golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0 golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index cd4b1a73..4d064bd7 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -11,6 +11,7 @@ import ( "github.com/NVIDIA/sortedmap" "github.com/NVIDIA/proxyfs/ilayout" + "github.com/NVIDIA/proxyfs/imgr/imgrpkg" ) const ( @@ -100,7 +101,6 @@ func (dummy *globalsStruct) DoForget(inHeader *fission.InHeader, forgetIn *fissi globals.stats.DoForgetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO return } @@ -129,14 +129,14 @@ func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fis Retry: inodeLockRequest = newLockRequest() - inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) + inodeLockRequest.inodeNumber = inHeader.NodeID inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { goto Retry } - inode = lookupInode(uint64(inHeader.NodeID)) + inode = lookupInode(inHeader.NodeID) if nil == inode { inodeLockRequest.unlockAll() getAttrOut = nil @@ -183,10 +183,10 @@ Retry: }, } - inodeLockRequest.unlockAll() - fixAttrSizes(&getAttrOut.Attr) + inodeLockRequest.unlockAll() + errno = 0 return } @@ -369,7 +369,14 @@ func (dummy *globalsStruct) DoLink(inHeader *fission.InHeader, linkIn *fission.L func (dummy *globalsStruct) DoOpen(inHeader *fission.InHeader, openIn *fission.OpenIn) (openOut *fission.OpenOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + adjustInodeTableEntryOpenCountRequest *imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct + adjustInodeTableEntryOpenCountResponse *imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoOpen(inHeader: %+v, openIn: %+v)", inHeader, openIn) @@ -381,9 +388,70 @@ func (dummy *globalsStruct) DoOpen(inHeader *fission.InHeader, openIn *fission.O globals.stats.DoOpenUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - openOut = nil - errno = syscall.ENOSYS + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + openOut = nil + errno = syscall.ENOENT + return + } + + obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) + if errno == 0 { + if obtainExclusiveLock { + inodeLockRequest.unlockAll() + goto Retry + } + } else { + inodeLockRequest.unlockAll() + openOut = nil + return + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeFile { + inodeLockRequest.unlockAll() + openOut = nil + errno = syscall.ENXIO + } + + adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ + MountID: globals.mountID, + InodeNumber: inode.inodeNumber, + Adjustment: 1, + } + adjustInodeTableEntryOpenCountResponse = &imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct{} + + err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + if nil != err { + logWarnf("rpcAdjustInodeTableEntryOpenCount() returned err: %v", err) + } + + openHandle = createOpenHandle(inode.inodeNumber) + + openHandle.fissionFlagsAppend = (openIn.Flags & syscall.O_APPEND) == syscall.O_APPEND + openHandle.fissionFlagsRead = ((openIn.Flags & syscall.O_ACCMODE) == syscall.O_RDONLY) || ((openIn.Flags & syscall.O_ACCMODE) == syscall.O_RDWR) + openHandle.fissionFlagsWrite = ((openIn.Flags & syscall.O_ACCMODE) == syscall.O_RDWR) || ((openIn.Flags & syscall.O_ACCMODE) == syscall.O_WRONLY) + + openOut = &fission.OpenOut{ + FH: openHandle.fissionFH, + OpenFlags: 0, + Padding: 0, + } + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -426,7 +494,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R Retry: inodeLockRequest = newLockRequest() - inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) + inodeLockRequest.inodeNumber = inHeader.NodeID inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { @@ -440,7 +508,7 @@ Retry: errno = syscall.EBADF return } - if openHandle.inodeNumber != uint64(inHeader.NodeID) { + if openHandle.inodeNumber != inHeader.NodeID { inodeLockRequest.unlockAll() readOut = nil errno = syscall.EBADF @@ -642,7 +710,7 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission Retry: inodeLockRequest = newLockRequest() - inodeLockRequest.inodeNumber = uint64(inHeader.NodeID) + inodeLockRequest.inodeNumber = inHeader.NodeID inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { @@ -656,7 +724,7 @@ Retry: errno = syscall.EBADF return } - if openHandle.inodeNumber != uint64(inHeader.NodeID) { + if openHandle.inodeNumber != inHeader.NodeID { inodeLockRequest.unlockAll() writeOut = nil errno = syscall.EBADF @@ -736,7 +804,14 @@ func (dummy *globalsStruct) DoStatFS(inHeader *fission.InHeader) (statFSOut *fis func (dummy *globalsStruct) DoRelease(inHeader *fission.InHeader, releaseIn *fission.ReleaseIn) (errno syscall.Errno) { var ( - startTime time.Time = time.Now() + adjustInodeTableEntryOpenCountRequest *imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct + adjustInodeTableEntryOpenCountResponse *imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoRelease(inHeader: %+v, releaseIn: %+v)", inHeader, releaseIn) @@ -748,8 +823,67 @@ func (dummy *globalsStruct) DoRelease(inHeader *fission.InHeader, releaseIn *fis globals.stats.DoReleaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS + openHandle = lookupOpenHandleByFissionFH(releaseIn.FH) + if nil == openHandle { + errno = syscall.EBADF + return + } + if openHandle.inodeNumber != inHeader.NodeID { + errno = syscall.EBADF + return + } + + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + + obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) + if errno == 0 { + if obtainExclusiveLock { + inodeLockRequest.unlockAll() + goto Retry + } + } else { + inodeLockRequest.unlockAll() + return + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeFile { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + } + + adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ + MountID: globals.mountID, + InodeNumber: inode.inodeNumber, + Adjustment: -1, + } + adjustInodeTableEntryOpenCountResponse = &imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct{} + + err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + if nil != err { + logWarnf("rpcAdjustInodeTableEntryOpenCount() returned err: %v", err) + } + + openHandle.destroy() + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -883,8 +1017,6 @@ func (dummy *globalsStruct) DoInit(inHeader *fission.InHeader, initIn *fission.I globals.stats.DoInitUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - initOut = &fission.InitOut{ Major: initIn.Major, Minor: initIn.Minor, @@ -901,7 +1033,14 @@ func (dummy *globalsStruct) DoInit(inHeader *fission.InHeader, initIn *fission.I func (dummy *globalsStruct) DoOpenDir(inHeader *fission.InHeader, openDirIn *fission.OpenDirIn) (openDirOut *fission.OpenDirOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + adjustInodeTableEntryOpenCountRequest *imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct + adjustInodeTableEntryOpenCountResponse *imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoOpenDir(inHeader: %+v, openDirIn: %+v)", inHeader, openDirIn) @@ -913,9 +1052,76 @@ func (dummy *globalsStruct) DoOpenDir(inHeader *fission.InHeader, openDirIn *fis globals.stats.DoOpenDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - openDirOut = nil - errno = syscall.ENOSYS + if ((openDirIn.Flags & syscall.O_APPEND) == syscall.O_APPEND) || ((openDirIn.Flags & syscall.O_ACCMODE) != syscall.O_RDONLY) { + openDirOut = nil + errno = syscall.EACCES + return + } + + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + openDirOut = nil + errno = syscall.ENOENT + return + } + + obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) + if errno == 0 { + if obtainExclusiveLock { + inodeLockRequest.unlockAll() + goto Retry + } + } else { + inodeLockRequest.unlockAll() + openDirOut = nil + return + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + openDirOut = nil + errno = syscall.ENOTDIR + } + + adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ + MountID: globals.mountID, + InodeNumber: inode.inodeNumber, + Adjustment: 1, + } + adjustInodeTableEntryOpenCountResponse = &imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct{} + + err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + if nil != err { + logWarnf("rpcAdjustInodeTableEntryOpenCount() returned err: %v", err) + } + + openHandle = createOpenHandle(inode.inodeNumber) + + openHandle.fissionFlagsAppend = false + openHandle.fissionFlagsRead = true + openHandle.fissionFlagsWrite = false + + openDirOut = &fission.OpenDirOut{ + FH: openHandle.fissionFH, + OpenFlags: 0, + Padding: 0, + } + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -941,7 +1147,14 @@ func (dummy *globalsStruct) DoReadDir(inHeader *fission.InHeader, readDirIn *fis func (dummy *globalsStruct) DoReleaseDir(inHeader *fission.InHeader, releaseDirIn *fission.ReleaseDirIn) (errno syscall.Errno) { var ( - startTime time.Time = time.Now() + adjustInodeTableEntryOpenCountRequest *imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct + adjustInodeTableEntryOpenCountResponse *imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoReleaseDir(inHeader: %+v, releaseDirIn: %+v)", inHeader, releaseDirIn) @@ -953,8 +1166,67 @@ func (dummy *globalsStruct) DoReleaseDir(inHeader *fission.InHeader, releaseDirI globals.stats.DoReleaseDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS + openHandle = lookupOpenHandleByFissionFH(releaseDirIn.FH) + if nil == openHandle { + errno = syscall.EBADF + return + } + if openHandle.inodeNumber != inHeader.NodeID { + errno = syscall.EBADF + return + } + + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + + obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) + if errno == 0 { + if obtainExclusiveLock { + inodeLockRequest.unlockAll() + goto Retry + } + } else { + inodeLockRequest.unlockAll() + return + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + } + + adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ + MountID: globals.mountID, + InodeNumber: inode.inodeNumber, + Adjustment: -1, + } + adjustInodeTableEntryOpenCountResponse = &imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct{} + + err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + if nil != err { + logWarnf("rpcAdjustInodeTableEntryOpenCount() returned err: %v", err) + } + + openHandle.destroy() + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -1085,9 +1357,6 @@ func (dummy *globalsStruct) DoInterrupt(inHeader *fission.InHeader, interruptIn defer func() { globals.stats.DoInterruptUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - - // TODO - return } func (dummy *globalsStruct) DoBMap(inHeader *fission.InHeader, bMapIn *fission.BMapIn) (bMapOut *fission.BMapOut, errno syscall.Errno) { @@ -1142,7 +1411,6 @@ func (dummy *globalsStruct) DoPoll(inHeader *fission.InHeader, pollIn *fission.P globals.stats.DoPollUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO pollOut = nil errno = syscall.ENOSYS return @@ -1162,7 +1430,6 @@ func (dummy *globalsStruct) DoBatchForget(inHeader *fission.InHeader, batchForge globals.stats.DoBatchForgetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO return } @@ -1180,7 +1447,6 @@ func (dummy *globalsStruct) DoFAllocate(inHeader *fission.InHeader, fAllocateIn globals.stats.DoFAllocateUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO errno = syscall.ENOSYS return } @@ -1238,7 +1504,6 @@ func (dummy *globalsStruct) DoLSeek(inHeader *fission.InHeader, lSeekIn *fission globals.stats.DoLSeekUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO lSeekOut = nil errno = syscall.ENOSYS return diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 09cd5a6b..a96492b5 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -93,8 +93,8 @@ type openHandleStruct struct { inodeNumber uint64 fissionFH uint64 // As returned in fission.{CreateOut|OpenOut|OpenDirOut}.FH fissionFlagsAppend bool // As supplied in fission.{CreateIn|OpenIn}.Flags [true if (.Flags & syscall.O_APPEND) == syscall.O_APPEND] - fissionFlagsRead bool // As supplied in fission.{CreateIn|OpenIn|OpenDirIn}.Flags [true if (.Flags & (syscall.O_RDONLY|syscall.O_RDWR|syscall.O_WRONLY) is one of syscall.{O_RDONLY|O_RDWR}] - fissionFlagsWrite bool // As supplied in fission.{CreateIn|OpenIn|OpenDirIn}.Flags [true if (.Flags & (syscall.O_RDONLY|syscall.O_RDWR|syscall.O_WRONLY) is one of syscall.{O_RDWR|O_WRONLY}] + fissionFlagsRead bool // As supplied in fission.{CreateIn|OpenIn|OpenDirIn}.Flags [true if (.Flags & (syscall.O_ACCMODE) is one of syscall.{O_RDONLY|O_RDWR}] + fissionFlagsWrite bool // As supplied in fission.{CreateIn|OpenIn|OpenDirIn}.Flags [true if (.Flags & (syscall.O_ACCMODE) is one of syscall.{O_RDWR|O_WRONLY}] } type inodeStruct struct { diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 569d92cc..05c9a741 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -229,6 +229,10 @@ func lookupInode(inodeNumber uint64) (inode *inodeStruct) { return } +// createOpenHandle allocates an openHandleStruct and inserts it into the globals openHandle +// maps by both inodeNumber and fissionFH. Note that the fissionFlags* fields all default to +// false. Callers are expected to modify as necessary. +// func createOpenHandle(inodeNumber uint64) (openHandle *openHandleStruct) { openHandle = &openHandleStruct{ inodeNumber: inodeNumber, diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index 7e810180..01bfbf55 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -208,11 +208,15 @@ func updateSwithAuthTokenAndSwiftStorageURL() { globals.Unlock() + logTracef("==> iauth.PerformAuth(authPlugInPath: %s, authInString: %s", globals.config.AuthPlugInPath, globals.swiftAuthInString) + globals.swiftAuthToken, globals.swiftStorageURL, err = iauth.PerformAuth(globals.config.AuthPlugInPath, globals.swiftAuthInString) if nil != err { logFatalf("iauth.PerformAuth() failed: %v", err) } + logTracef("<== iauth.PerformAuth(authToken: %s, storageURL: %s)", globals.swiftAuthToken, globals.swiftStorageURL) + globals.Lock() globals.swiftAuthWaitGroup.Done() globals.swiftAuthWaitGroup = nil @@ -253,6 +257,11 @@ func rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *im startTime time.Time = time.Now() ) + logTracef("==> rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest: %+v)", adjustInodeTableEntryOpenCountRequest) + defer func() { + logTracef("<== rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountResponse: %+v, err: %v)", adjustInodeTableEntryOpenCountResponse, err) + }() + defer func() { globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -267,6 +276,11 @@ func rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest *imgrpkg.DeleteInodeT startTime time.Time = time.Now() ) + logTracef("==> rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest: %+v)", deleteInodeTableEntryRequest) + defer func() { + logTracef("<== rpcDeleteInodeTableEntry(deleteInodeTableEntryResponse: %+v, err: %v)", deleteInodeTableEntryResponse, err) + }() + defer func() { globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -281,6 +295,11 @@ func rpcFetchNonceRange(fetchNonceRangeRequest *imgrpkg.FetchNonceRangeRequestSt startTime time.Time = time.Now() ) + logTracef("==> rpcFetchNonceRange(fetchNonceRangeRequest: %+v)", fetchNonceRangeRequest) + defer func() { + logTracef("<== rpcFetchNonceRange(fetchNonceRangeResponse: %+v, err: %v)", fetchNonceRangeResponse, err) + }() + defer func() { globals.stats.FetchNonceRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -295,6 +314,11 @@ func rpcFlush(flushRequest *imgrpkg.FlushRequestStruct, flushResponse *imgrpkg.F startTime time.Time = time.Now() ) + logTracef("==> rpcFlush(flushRequest: %+v)", flushRequest) + defer func() { + logTracef("<== rpcFlush(flushResponse: %+v, err: %v)", flushResponse, err) + }() + defer func() { globals.stats.FlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -309,6 +333,11 @@ func rpcGetInodeTableEntry(getInodeTableEntryRequest *imgrpkg.GetInodeTableEntry startTime time.Time = time.Now() ) + logTracef("==> rpcGetInodeTableEntry(getInodeTableEntryRequest: %+v)", getInodeTableEntryRequest) + defer func() { + logTracef("<== rpcGetInodeTableEntry(getInodeTableEntryResponse: %+v, err: %v)", getInodeTableEntryResponse, err) + }() + defer func() { globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -323,6 +352,11 @@ func rpcLease(leaseRequest *imgrpkg.LeaseRequestStruct, leaseResponse *imgrpkg.L startTime time.Time = time.Now() ) + logTracef("==> rpcLease(leaseRequest: %+v)", leaseRequest) + defer func() { + logTracef("<== rpcLease(leaseResponse: %+v, err: %v)", leaseResponse, err) + }() + defer func() { globals.stats.LeaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -337,6 +371,11 @@ func rpcMount(mountRequest *imgrpkg.MountRequestStruct, mountResponse *imgrpkg.M startTime time.Time = time.Now() ) + logTracef("==> rpcMount(mountRequest: %+v)", mountRequest) + defer func() { + logTracef("<== rpcMount(mountResponse: %+v, err: %v)", mountResponse, err) + }() + defer func() { globals.stats.MountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -351,6 +390,11 @@ func rpcPutInodeTableEntries(putInodeTableEntriesRequest *imgrpkg.PutInodeTableE startTime time.Time = time.Now() ) + logTracef("==> rpcPutInodeTableEntries(rpcPutInodeTableEntries: %+v)", rpcPutInodeTableEntries) + defer func() { + logTracef("<== rpcPutInodeTableEntries(putInodeTableEntriesResponse: %+v, err: %v)", putInodeTableEntriesResponse, err) + }() + defer func() { globals.stats.PutInodeTableEntriesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -365,6 +409,11 @@ func rpcRenewMount(renewMountRequest *imgrpkg.RenewMountRequestStruct, renewMoun startTime time.Time = time.Now() ) + logTracef("==> rpcRenewMount(renewMountRequest: %+v)", renewMountRequest) + defer func() { + logTracef("<== rpcRenewMount(renewMountResponse: %+v, err: %v)", renewMountResponse, err) + }() + defer func() { globals.stats.DoGetAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() @@ -379,6 +428,11 @@ func rpcUnmount(unmountRequest *imgrpkg.UnmountRequestStruct, unmountResponse *i startTime time.Time = time.Now() ) + logTracef("==> rpcUnmount(unmountRequest: %+v)", unmountRequest) + defer func() { + logTracef("<== rpcUnmount(unmountResponse: %+v, err: %v)", unmountResponse, err) + }() + defer func() { globals.stats.UnmountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() From 7b0c3cfa8789be9d6648f43c962156ce4a56b68f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 21 Oct 2021 18:10:31 -0700 Subject: [PATCH 129/258] Beginning work on DoReadDir{|Plus}() --- iclient/iclientpkg/fission.go | 274 ++++++++++++++++++++++++++++------ iclient/iclientpkg/inode.go | 84 ++++------- 2 files changed, 259 insertions(+), 99 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 4d064bd7..6ac21832 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -28,8 +28,8 @@ const ( fission.InitFlagsAtomicOTrunc | fission.InitFlagsBigWrites | fission.InitFlagsAutoInvalData | - fission.InitFlagsDoReadDirPlus | - fission.InitFlagsReaddirplusAuto | + // fission.InitFlagsDoReadDirPlus | // UNDO once DoReadDir() is completed + // fission.InitFlagsReaddirplusAuto | // UNDO so that I can then test DoReadDirPlus() fission.InitFlagsParallelDirops | fission.InitFlagsMaxPages | fission.InitFlagsExplicitInvalData @@ -106,6 +106,7 @@ func (dummy *globalsStruct) DoForget(inHeader *fission.InHeader, forgetIn *fissi func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fission.GetAttrIn) (getAttrOut *fission.GetAttrOut, errno syscall.Errno) { var ( + err error inode *inodeStruct inodeLockRequest *inodeLockRequestStruct modificationTimeNSec uint32 @@ -144,16 +145,20 @@ Retry: return } - obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) - if errno == 0 { + if nil == inode.inodeHeadV1 { if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + getAttrOut = nil + errno = syscall.ENOENT + return + } + } else { inodeLockRequest.unlockAll() + obtainExclusiveLock = true goto Retry } - } else { - inodeLockRequest.unlockAll() - getAttrOut = nil - return } modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.ModificationTime.UnixNano())) @@ -407,22 +412,27 @@ Retry: return } - obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) - if errno == 0 { + if nil == inode.inodeHeadV1 { if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + openOut = nil + errno = syscall.ENOENT + return + } + } else { inodeLockRequest.unlockAll() + obtainExclusiveLock = true goto Retry } - } else { - inodeLockRequest.unlockAll() - openOut = nil - return } if inode.inodeHeadV1.InodeType != ilayout.InodeTypeFile { inodeLockRequest.unlockAll() openOut = nil errno = syscall.ENXIO + return } adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ @@ -529,16 +539,20 @@ Retry: return } - obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) - if errno == 0 { + if nil == inode.inodeHeadV1 { if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.ENOENT + return + } + } else { inodeLockRequest.unlockAll() + obtainExclusiveLock = true goto Retry } - } else { - inodeLockRequest.unlockAll() - readOut = nil - return } switch inode.inodeHeadV1.InodeType { @@ -693,6 +707,7 @@ Retry: func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission.WriteIn) (writeOut *fission.WriteOut, errno syscall.Errno) { var ( + err error inode *inodeStruct inodeLockRequest *inodeLockRequestStruct openHandle *openHandleStruct @@ -745,11 +760,14 @@ Retry: return } - _, errno = inode.ensureInodeHeadV1IsNonNil(true) - if errno != 0 { - inodeLockRequest.unlockAll() - writeOut = nil - return + if nil == inode.inodeHeadV1 { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + writeOut = nil + errno = syscall.ENOENT + return + } } switch inode.inodeHeadV1.InodeType { @@ -851,20 +869,25 @@ Retry: return } - obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) - if errno == 0 { + if nil == inode.inodeHeadV1 { if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } else { inodeLockRequest.unlockAll() + obtainExclusiveLock = true goto Retry } - } else { - inodeLockRequest.unlockAll() - return } if inode.inodeHeadV1.InodeType != ilayout.InodeTypeFile { inodeLockRequest.unlockAll() errno = syscall.EBADF + return } adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ @@ -1077,22 +1100,27 @@ Retry: return } - obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) - if errno == 0 { + if nil == inode.inodeHeadV1 { if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + openDirOut = nil + errno = syscall.ENOENT + return + } + } else { inodeLockRequest.unlockAll() + obtainExclusiveLock = true goto Retry } - } else { - inodeLockRequest.unlockAll() - openDirOut = nil - return } if inode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { inodeLockRequest.unlockAll() openDirOut = nil errno = syscall.ENOTDIR + return } adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ @@ -1127,7 +1155,12 @@ Retry: func (dummy *globalsStruct) DoReadDir(inHeader *fission.InHeader, readDirIn *fission.ReadDirIn) (readDirOut *fission.ReadDirOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoReadDir(inHeader: %+v, readDirIn: %+v)", inHeader, readDirIn) @@ -1139,7 +1172,77 @@ func (dummy *globalsStruct) DoReadDir(inHeader *fission.InHeader, readDirIn *fis globals.stats.DoReadDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO + openHandle = lookupOpenHandleByFissionFH(readDirIn.FH) + if nil == openHandle { + readDirOut = nil + errno = syscall.EBADF + return + } + if openHandle.inodeNumber != inHeader.NodeID { + readDirOut = nil + errno = syscall.EBADF + return + } + + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + readDirOut = nil + errno = syscall.EBADF + return + } + + if nil == inode.inodeHeadV1 { + if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + readDirOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + readDirOut = nil + errno = syscall.EBADF + return + } + + if nil == inode.payload { + if obtainExclusiveLock { + err = inode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + readDirOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + inodeLockRequest.unlockAll() // TODO readDirOut = nil errno = syscall.ENOSYS return @@ -1194,20 +1297,24 @@ Retry: return } - obtainExclusiveLock, errno = inode.ensureInodeHeadV1IsNonNil(obtainExclusiveLock) - if errno == 0 { + if nil == inode.inodeHeadV1 { if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } else { inodeLockRequest.unlockAll() + obtainExclusiveLock = true goto Retry } - } else { - inodeLockRequest.unlockAll() - return } if inode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { inodeLockRequest.unlockAll() - errno = syscall.EBADF + return } adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ @@ -1453,7 +1560,12 @@ func (dummy *globalsStruct) DoFAllocate(inHeader *fission.InHeader, fAllocateIn func (dummy *globalsStruct) DoReadDirPlus(inHeader *fission.InHeader, readDirPlusIn *fission.ReadDirPlusIn) (readDirPlusOut *fission.ReadDirPlusOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoReadDirPlus(inHeader: %+v, readDirPlusIn: %+v)", inHeader, readDirPlusIn) @@ -1465,7 +1577,77 @@ func (dummy *globalsStruct) DoReadDirPlus(inHeader *fission.InHeader, readDirPlu globals.stats.DoReadDirPlusUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO + openHandle = lookupOpenHandleByFissionFH(readDirPlusIn.FH) + if nil == openHandle { + readDirPlusOut = nil + errno = syscall.EBADF + return + } + if openHandle.inodeNumber != inHeader.NodeID { + readDirPlusOut = nil + errno = syscall.EBADF + return + } + + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + readDirPlusOut = nil + errno = syscall.EBADF + return + } + + if nil == inode.inodeHeadV1 { + if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + readDirPlusOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + readDirPlusOut = nil + errno = syscall.EBADF + return + } + + if nil == inode.payload { + if obtainExclusiveLock { + err = inode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + readDirPlusOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + inodeLockRequest.unlockAll() // TODO readDirPlusOut = nil errno = syscall.ENOSYS return diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 05c9a741..98d0ec2f 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "sync" - "syscall" "github.com/NVIDIA/sortedmap" @@ -364,6 +363,37 @@ Retry: goto Retry } +func (inode *inodeStruct) populateInodeHeadV1() (err error) { + var ( + getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct + getInodeTableEntryResponse *imgrpkg.GetInodeTableEntryResponseStruct + inodeHeadV1Buf []byte + ) + + getInodeTableEntryRequest = &imgrpkg.GetInodeTableEntryRequestStruct{ + MountID: globals.mountID, + InodeNumber: inode.inodeNumber, + } + getInodeTableEntryResponse = &imgrpkg.GetInodeTableEntryResponseStruct{} + + err = rpcGetInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) + if nil != err { + return + } + + inodeHeadV1Buf, err = objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength) + if nil != err { + logFatalf("objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber: %v, getInodeTableEntryResponse.InodeHeadLength: %v) failed: %v", getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength, err) + } + + inode.inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) + if nil != err { + logFatalf("ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) failed: %v", err) + } + + return +} + func (inode *inodeStruct) newPayload() (err error) { switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: @@ -448,55 +478,3 @@ func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced } } - -// ensureInodeHeadV1IsNonNil will ensure inode.inodeHeadV1 is non-nil. As -// such, if it is currently nil at entry, ensureInodeHeadV1IsNonNil() will -// need exclusive access to the inode. In this case, if the exclusive lock -// is not held, ensureInodeHeadV1IsNonNil() will return without error but -// indicate an exclusive lock is needed. Note that if inode.inodeHeadV1 was -// non-nil or an exclusive lock was already held, ensureInodeHeadV1IsNonNil() -// will return that it (no longer) needs an exclusive lock and without error. -// -func (inode *inodeStruct) ensureInodeHeadV1IsNonNil(exclusivelyLocked bool) (needExclusiveLock bool, errno syscall.Errno) { - var ( - err error - getInodeTableEntryRequest *imgrpkg.GetInodeTableEntryRequestStruct - getInodeTableEntryResponse *imgrpkg.GetInodeTableEntryResponseStruct - inodeHeadV1Buf []byte - ) - - if nil == inode.inodeHeadV1 { - if exclusivelyLocked { - needExclusiveLock = false - - getInodeTableEntryRequest = &imgrpkg.GetInodeTableEntryRequestStruct{ - MountID: globals.mountID, - InodeNumber: inode.inodeNumber, - } - getInodeTableEntryResponse = &imgrpkg.GetInodeTableEntryResponseStruct{} - - err = rpcGetInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) - if nil != err { - errno = syscall.ENOENT - return - } - - inodeHeadV1Buf, err = objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength) - if nil != err { - logFatalf("objectGETTail(getInodeTableEntryResponse.InodeHeadObjectNumber: %v, getInodeTableEntryResponse.InodeHeadLength: %v) failed: %v", getInodeTableEntryResponse.InodeHeadObjectNumber, getInodeTableEntryResponse.InodeHeadLength, err) - } - - inode.inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) - if nil != err { - logFatalf("ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) failed: %v", err) - } - } else { - needExclusiveLock = true - } - } else { - needExclusiveLock = false - } - - errno = 0 - return -} From dfeef4759e98cfedd5127657cf487b655899314d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 21 Oct 2021 19:42:38 -0700 Subject: [PATCH 130/258] DoReadDir() done... next up: DoReadDirPlus()... --- iclient/iclientpkg/fission.go | 129 +++++++++++++++++++++++++++++++--- 1 file changed, 118 insertions(+), 11 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 6ac21832..ebe25266 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1155,12 +1155,24 @@ Retry: func (dummy *globalsStruct) DoReadDir(inHeader *fission.InHeader, readDirIn *fission.ReadDirIn) (readDirOut *fission.ReadDirOut, errno syscall.Errno) { var ( - err error - inode *inodeStruct - inodeLockRequest *inodeLockRequestStruct - obtainExclusiveLock bool - openHandle *openHandleStruct - startTime time.Time = time.Now() + dirEntCountMax uint64 + dirEntMinSize uint64 + dirEntSize uint64 + dirEntSliceSize uint64 + dirEntType uint32 + directoryEntryIndex int + directoryEntryKeyV1 string + directoryEntryKeyV1AsKey sortedmap.Key + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + directoryLen int + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + ok bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoReadDir(inHeader: %+v, readDirIn: %+v)", inHeader, readDirIn) @@ -1242,9 +1254,88 @@ Retry: } } - inodeLockRequest.unlockAll() // TODO - readDirOut = nil - errno = syscall.ENOSYS + dirEntMinSize = fission.DirEntFixedPortionSize + 1 + fission.DirEntAlignment - 1 + dirEntMinSize /= fission.DirEntAlignment + dirEntMinSize *= fission.DirEntAlignment + dirEntCountMax = uint64(readDirIn.Size) / dirEntMinSize + + readDirOut = &fission.ReadDirOut{ + DirEnt: make([]fission.DirEnt, 0, dirEntCountMax), + } + + if dirEntCountMax == 0 { + inodeLockRequest.unlockAll() + errno = 0 + return + } + + directoryLen, err = inode.payload.Len() + if nil != err { + logFatalf("inode.payload.Len() failed: %v", err) + } + + if readDirIn.Offset >= uint64(directoryLen) { + inodeLockRequest.unlockAll() + errno = 0 + return + } + + directoryEntryIndex = int(readDirIn.Offset) + dirEntSliceSize = 0 + + for directoryEntryIndex < directoryLen { + directoryEntryKeyV1AsKey, directoryEntryValueV1AsValue, ok, err = inode.payload.GetByIndex(directoryEntryIndex) + if nil != err { + logFatalf("inode.payload.GetByIndex(directoryEntryIndex) failed: %v", err) + } + if !ok { + logFatalf("inode.payload.GetByIndex(directoryEntryIndex) returned !ok") + } + + directoryEntryKeyV1, ok = directoryEntryKeyV1AsKey.(string) + if !ok { + logFatalf("directoryEntryKeyV1AsKey.(string) returned !ok") + } + + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + dirEntSize = fission.DirEntFixedPortionSize + uint64(len(directoryEntryKeyV1)) + fission.DirEntAlignment - 1 + dirEntSize /= fission.DirEntAlignment + dirEntSize *= fission.DirEntAlignment + + dirEntSliceSize += dirEntSize + if dirEntSliceSize > uint64(readDirIn.Size) { + break + } + + directoryEntryIndex++ + + switch directoryEntryValueV1.InodeType { + case ilayout.InodeTypeDir: + dirEntType = syscall.S_IFDIR + case ilayout.InodeTypeFile: + dirEntType = syscall.S_IFREG + case ilayout.InodeTypeSymLink: + dirEntType = syscall.S_IFLNK + default: + logFatalf("directoryEntryValueV1.InodeType (%v) unknown", directoryEntryValueV1.InodeType) + } + + readDirOut.DirEnt = append(readDirOut.DirEnt, fission.DirEnt{ + Ino: directoryEntryValueV1.InodeNumber, + Off: uint64(directoryEntryIndex), + NameLen: uint32(len(directoryEntryKeyV1)), + Type: dirEntType, + Name: []byte(directoryEntryKeyV1), + }) + } + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -1560,6 +1651,8 @@ func (dummy *globalsStruct) DoFAllocate(inHeader *fission.InHeader, fAllocateIn func (dummy *globalsStruct) DoReadDirPlus(inHeader *fission.InHeader, readDirPlusIn *fission.ReadDirPlusIn) (readDirPlusOut *fission.ReadDirPlusOut, errno syscall.Errno) { var ( + dirEntAlignSize uint64 + dirEntCountMax uint64 err error inode *inodeStruct inodeLockRequest *inodeLockRequestStruct @@ -1647,9 +1740,23 @@ Retry: } } + dirEntAlignSize = fission.DirEntPlusFixedPortionSize + fission.DirEntAlignment - 1 + dirEntAlignSize /= fission.DirEntAlignment + dirEntAlignSize *= fission.DirEntAlignment + dirEntCountMax = uint64(readDirPlusIn.Size) / dirEntAlignSize + + readDirPlusOut = &fission.ReadDirPlusOut{ + DirEntPlus: make([]fission.DirEntPlus, 0, dirEntCountMax), + } + + if dirEntCountMax == 0 { + inodeLockRequest.unlockAll() + errno = 0 + return + } + inodeLockRequest.unlockAll() // TODO - readDirPlusOut = nil - errno = syscall.ENOSYS + errno = 0 return } From e3d9d980dc0742657867816d6924649775e1be38 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 21 Oct 2021 20:45:46 -0700 Subject: [PATCH 131/258] DoReadDirPlus() complete --- iclient/iclientpkg/fission.go | 238 +++++++++++++++++++++++++++++----- 1 file changed, 207 insertions(+), 31 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index ebe25266..671cba5c 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -4,6 +4,8 @@ package iclientpkg import ( + "runtime" + "sync" "syscall" "time" @@ -28,8 +30,8 @@ const ( fission.InitFlagsAtomicOTrunc | fission.InitFlagsBigWrites | fission.InitFlagsAutoInvalData | - // fission.InitFlagsDoReadDirPlus | // UNDO once DoReadDir() is completed - // fission.InitFlagsReaddirplusAuto | // UNDO so that I can then test DoReadDirPlus() + fission.InitFlagsDoReadDirPlus | + fission.InitFlagsReaddirplusAuto | fission.InitFlagsParallelDirops | fission.InitFlagsMaxPages | fission.InitFlagsExplicitInvalData @@ -1159,7 +1161,6 @@ func (dummy *globalsStruct) DoReadDir(inHeader *fission.InHeader, readDirIn *fis dirEntMinSize uint64 dirEntSize uint64 dirEntSliceSize uint64 - dirEntType uint32 directoryEntryIndex int directoryEntryKeyV1 string directoryEntryKeyV1AsKey sortedmap.Key @@ -1313,22 +1314,11 @@ Retry: directoryEntryIndex++ - switch directoryEntryValueV1.InodeType { - case ilayout.InodeTypeDir: - dirEntType = syscall.S_IFDIR - case ilayout.InodeTypeFile: - dirEntType = syscall.S_IFREG - case ilayout.InodeTypeSymLink: - dirEntType = syscall.S_IFLNK - default: - logFatalf("directoryEntryValueV1.InodeType (%v) unknown", directoryEntryValueV1.InodeType) - } - readDirOut.DirEnt = append(readDirOut.DirEnt, fission.DirEnt{ Ino: directoryEntryValueV1.InodeNumber, Off: uint64(directoryEntryIndex), NameLen: uint32(len(directoryEntryKeyV1)), - Type: dirEntType, + Type: dirEntType(directoryEntryValueV1.InodeType), Name: []byte(directoryEntryKeyV1), }) } @@ -1651,14 +1641,26 @@ func (dummy *globalsStruct) DoFAllocate(inHeader *fission.InHeader, fAllocateIn func (dummy *globalsStruct) DoReadDirPlus(inHeader *fission.InHeader, readDirPlusIn *fission.ReadDirPlusIn) (readDirPlusOut *fission.ReadDirPlusOut, errno syscall.Errno) { var ( - dirEntAlignSize uint64 - dirEntCountMax uint64 - err error - inode *inodeStruct - inodeLockRequest *inodeLockRequestStruct - obtainExclusiveLock bool - openHandle *openHandleStruct - startTime time.Time = time.Now() + dirEntPlusCountMax uint64 + dirEntPlusIndex int + dirEntPlusMinSize uint64 + dirEntPlusSize uint64 + dirEntPlusSliceSize uint64 + directoryEntryIndex int + directoryEntryKeyV1 string + directoryEntryKeyV1AsKey sortedmap.Key + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + directoryLen int + err error + helperErrorCount uint64 + helperWG sync.WaitGroup + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + ok bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoReadDirPlus(inHeader: %+v, readDirPlusIn: %+v)", inHeader, readDirPlusIn) @@ -1740,26 +1742,186 @@ Retry: } } - dirEntAlignSize = fission.DirEntPlusFixedPortionSize + fission.DirEntAlignment - 1 - dirEntAlignSize /= fission.DirEntAlignment - dirEntAlignSize *= fission.DirEntAlignment - dirEntCountMax = uint64(readDirPlusIn.Size) / dirEntAlignSize + dirEntPlusMinSize = fission.DirEntPlusFixedPortionSize + 1 + fission.DirEntAlignment - 1 + dirEntPlusMinSize /= fission.DirEntAlignment + dirEntPlusMinSize *= fission.DirEntAlignment + dirEntPlusCountMax = uint64(readDirPlusIn.Size) / dirEntPlusMinSize readDirPlusOut = &fission.ReadDirPlusOut{ - DirEntPlus: make([]fission.DirEntPlus, 0, dirEntCountMax), + DirEntPlus: make([]fission.DirEntPlus, 0, dirEntPlusCountMax), } - if dirEntCountMax == 0 { + if dirEntPlusCountMax == 0 { inodeLockRequest.unlockAll() errno = 0 return } - inodeLockRequest.unlockAll() // TODO - errno = 0 + directoryLen, err = inode.payload.Len() + if nil != err { + logFatalf("inode.payload.Len() failed: %v", err) + } + + if readDirPlusIn.Offset >= uint64(directoryLen) { + inodeLockRequest.unlockAll() + errno = 0 + return + } + + directoryEntryIndex = int(readDirPlusIn.Offset) + dirEntPlusSliceSize = 0 + + for directoryEntryIndex < directoryLen { + directoryEntryKeyV1AsKey, directoryEntryValueV1AsValue, ok, err = inode.payload.GetByIndex(directoryEntryIndex) + if nil != err { + logFatalf("inode.payload.GetByIndex(directoryEntryIndex) failed: %v", err) + } + if !ok { + logFatalf("inode.payload.GetByIndex(directoryEntryIndex) returned !ok") + } + + directoryEntryKeyV1, ok = directoryEntryKeyV1AsKey.(string) + if !ok { + logFatalf("directoryEntryKeyV1AsKey.(string) returned !ok") + } + + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + dirEntPlusSize = fission.DirEntFixedPortionSize + uint64(len(directoryEntryKeyV1)) + fission.DirEntAlignment - 1 + dirEntPlusSize /= fission.DirEntAlignment + dirEntPlusSize *= fission.DirEntAlignment + + dirEntPlusSliceSize += dirEntPlusSize + if dirEntPlusSliceSize > uint64(readDirPlusIn.Size) { + break + } + + directoryEntryIndex++ + + readDirPlusOut.DirEntPlus = append(readDirPlusOut.DirEntPlus, fission.DirEntPlus{ + EntryOut: fission.EntryOut{ + NodeID: directoryEntryValueV1.InodeNumber, + Generation: 0, + EntryValidSec: globals.fuseEntryValidDurationSec, + EntryValidNSec: globals.fuseEntryValidDurationNSec, + AttrValidSec: globals.fuseAttrValidDurationSec, + AttrValidNSec: globals.fuseAttrValidDurationNSec, + // Attr to be filled in below + }, + DirEnt: fission.DirEnt{ + Ino: directoryEntryValueV1.InodeNumber, + Off: uint64(directoryEntryIndex), + NameLen: uint32(len(directoryEntryKeyV1)), + Type: dirEntType(directoryEntryValueV1.InodeType), + Name: []byte(directoryEntryKeyV1), + }, + }) + } + + inodeLockRequest.unlockAll() + + helperErrorCount = 0 + + for dirEntPlusIndex = range readDirPlusOut.DirEntPlus { + helperWG.Add(1) + go doReadDirPlusHelper(readDirPlusOut.DirEntPlus[dirEntPlusIndex].EntryOut.NodeID, &readDirPlusOut.DirEntPlus[dirEntPlusIndex].EntryOut.Attr, &helperWG, &helperErrorCount) + } + + helperWG.Wait() + + if helperErrorCount == 0 { + errno = 0 + } else { + readDirPlusOut = &fission.ReadDirPlusOut{ + DirEntPlus: make([]fission.DirEntPlus, 0, 0), + } + errno = syscall.EIO + } + return } +func doReadDirPlusHelper(inodeNumber uint64, fissionAttr *fission.Attr, helperWG *sync.WaitGroup, helperErrorCount *uint64) { + var ( + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + modificationTimeNSec uint32 + modificationTimeSec uint64 + obtainExclusiveLock bool + statusChangeTimeNSec uint32 + statusChangeTimeSec uint64 + ) + + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inodeNumber + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inodeNumber) + if nil == inode { + inodeLockRequest.unlockAll() + globals.Lock() + *helperErrorCount++ + globals.Unlock() + helperWG.Done() + runtime.Goexit() + } + + if nil == inode.inodeHeadV1 { + if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + globals.Lock() + *helperErrorCount++ + globals.Unlock() + helperWG.Done() + runtime.Goexit() + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.ModificationTime.UnixNano())) + statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.StatusChangeTime.UnixNano())) + + fissionAttr.Ino = inode.inodeHeadV1.InodeNumber + fissionAttr.Size = inode.inodeHeadV1.Size // Possibly overwritten by fixAttrSizes() + fissionAttr.Blocks = 0 // Computed by fixAttrSizes() + fissionAttr.ATimeSec = modificationTimeSec + fissionAttr.MTimeSec = modificationTimeSec + fissionAttr.CTimeSec = statusChangeTimeSec + fissionAttr.ATimeNSec = modificationTimeNSec + fissionAttr.MTimeNSec = modificationTimeNSec + fissionAttr.CTimeNSec = statusChangeTimeNSec + fissionAttr.Mode = computeAttrMode(inode.inodeHeadV1.InodeType, inode.inodeHeadV1.Mode) + fissionAttr.NLink = uint32(len(inode.inodeHeadV1.LinkTable)) + fissionAttr.UID = uint32(inode.inodeHeadV1.UserID) + fissionAttr.GID = uint32(inode.inodeHeadV1.GroupID) + fissionAttr.RDev = attrRDev + fissionAttr.BlkSize = attrBlockSize // Possibly overwritten by fixAttrSizes() + fissionAttr.Padding = 0 + + fixAttrSizes(fissionAttr) + + inodeLockRequest.unlockAll() + + helperWG.Done() +} + func (dummy *globalsStruct) DoRename2(inHeader *fission.InHeader, rename2In *fission.Rename2In) (errno syscall.Errno) { var ( startTime time.Time = time.Now() @@ -1809,6 +1971,20 @@ func unixTimeToNs(sec uint64, nsec uint32) (ns uint64) { return } +func dirEntType(iLayoutInodeType uint8) (dirEntType uint32) { + switch iLayoutInodeType { + case ilayout.InodeTypeDir: + dirEntType = syscall.S_IFDIR + case ilayout.InodeTypeFile: + dirEntType = syscall.S_IFREG + case ilayout.InodeTypeSymLink: + dirEntType = syscall.S_IFLNK + default: + logFatalf("iLayoutInodeType (%v) unknown", iLayoutInodeType) + } + return +} + func computeAttrMode(iLayoutInodeType uint8, iLayoutMode uint16) (attrMode uint32) { attrMode = uint32(iLayoutMode) switch iLayoutInodeType { From f6dac76b2be24500d95c98a9a038306f51ffa8d2 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 22 Oct 2021 12:18:41 -0700 Subject: [PATCH 132/258] Implemented DoLookup() --- iclient/iclientpkg/fission.go | 290 +++++++++++++++++++++++----------- 1 file changed, 200 insertions(+), 90 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 671cba5c..51b0acfe 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -4,7 +4,7 @@ package iclientpkg import ( - "runtime" + "fmt" "sync" "syscall" "time" @@ -71,7 +71,14 @@ func performUnmountFUSE() (err error) { func (dummy *globalsStruct) DoLookup(inHeader *fission.InHeader, lookupIn *fission.LookupIn) (lookupOut *fission.LookupOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + ok bool + startTime time.Time = time.Now() ) logTracef("==> DoLookup(inHeader: %+v, lookupIn: %+v)", inHeader, lookupIn) @@ -83,9 +90,103 @@ func (dummy *globalsStruct) DoLookup(inHeader *fission.InHeader, lookupIn *fissi globals.stats.DoLookupUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - lookupOut = nil - errno = syscall.ENOSYS + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + lookupOut = nil + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + lookupOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + lookupOut = nil + errno = syscall.ENOTDIR + return + } + + if nil == inode.payload { + if obtainExclusiveLock { + err = inode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + lookupOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + directoryEntryValueV1AsValue, ok, err = inode.payload.GetByKey(string(lookupIn.Name[:])) + if nil != err { + logFatalf("inode.payload.GetByKey(string(lookupIn.Name[:])) failed: %v", err) + } + if !ok { + inodeLockRequest.unlockAll() + lookupOut = nil + errno = syscall.ENOENT + return + } + + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + inodeLockRequest.unlockAll() + + lookupOut = &fission.LookupOut{ + EntryOut: fission.EntryOut{ + NodeID: directoryEntryValueV1.InodeNumber, + Generation: 0, + EntryValidSec: globals.fuseEntryValidDurationSec, + EntryValidNSec: globals.fuseEntryValidDurationNSec, + AttrValidSec: globals.fuseAttrValidDurationSec, + AttrValidNSec: globals.fuseAttrValidDurationNSec, + // Attr to be filled in below + }, + } + + err = doAttrFetch(directoryEntryValueV1.InodeNumber, &lookupOut.EntryOut.Attr) + + if nil == err { + errno = 0 + } else { + lookupOut = nil + errno = syscall.EIO + } + return } @@ -1653,8 +1754,8 @@ func (dummy *globalsStruct) DoReadDirPlus(inHeader *fission.InHeader, readDirPlu directoryEntryValueV1AsValue sortedmap.Value directoryLen int err error - helperErrorCount uint64 - helperWG sync.WaitGroup + gorAttrFetchErrorCount uint64 + gorAttrFetchWG sync.WaitGroup inode *inodeStruct inodeLockRequest *inodeLockRequestStruct obtainExclusiveLock bool @@ -1823,16 +1924,16 @@ Retry: inodeLockRequest.unlockAll() - helperErrorCount = 0 + gorAttrFetchErrorCount = 0 for dirEntPlusIndex = range readDirPlusOut.DirEntPlus { - helperWG.Add(1) - go doReadDirPlusHelper(readDirPlusOut.DirEntPlus[dirEntPlusIndex].EntryOut.NodeID, &readDirPlusOut.DirEntPlus[dirEntPlusIndex].EntryOut.Attr, &helperWG, &helperErrorCount) + gorAttrFetchWG.Add(1) + go gorAttrFetch(readDirPlusOut.DirEntPlus[dirEntPlusIndex].EntryOut.NodeID, &readDirPlusOut.DirEntPlus[dirEntPlusIndex].EntryOut.Attr, &gorAttrFetchWG, &gorAttrFetchErrorCount) } - helperWG.Wait() + gorAttrFetchWG.Wait() - if helperErrorCount == 0 { + if gorAttrFetchErrorCount == 0 { errno = 0 } else { readDirPlusOut = &fission.ReadDirPlusOut{ @@ -1844,84 +1945,6 @@ Retry: return } -func doReadDirPlusHelper(inodeNumber uint64, fissionAttr *fission.Attr, helperWG *sync.WaitGroup, helperErrorCount *uint64) { - var ( - err error - inode *inodeStruct - inodeLockRequest *inodeLockRequestStruct - modificationTimeNSec uint32 - modificationTimeSec uint64 - obtainExclusiveLock bool - statusChangeTimeNSec uint32 - statusChangeTimeSec uint64 - ) - - obtainExclusiveLock = false - -Retry: - inodeLockRequest = newLockRequest() - inodeLockRequest.inodeNumber = inodeNumber - inodeLockRequest.exclusive = obtainExclusiveLock - inodeLockRequest.addThisLock() - if len(inodeLockRequest.locksHeld) == 0 { - goto Retry - } - - inode = lookupInode(inodeNumber) - if nil == inode { - inodeLockRequest.unlockAll() - globals.Lock() - *helperErrorCount++ - globals.Unlock() - helperWG.Done() - runtime.Goexit() - } - - if nil == inode.inodeHeadV1 { - if obtainExclusiveLock { - err = inode.populateInodeHeadV1() - if nil != err { - inodeLockRequest.unlockAll() - globals.Lock() - *helperErrorCount++ - globals.Unlock() - helperWG.Done() - runtime.Goexit() - } - } else { - inodeLockRequest.unlockAll() - obtainExclusiveLock = true - goto Retry - } - } - - modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.ModificationTime.UnixNano())) - statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.StatusChangeTime.UnixNano())) - - fissionAttr.Ino = inode.inodeHeadV1.InodeNumber - fissionAttr.Size = inode.inodeHeadV1.Size // Possibly overwritten by fixAttrSizes() - fissionAttr.Blocks = 0 // Computed by fixAttrSizes() - fissionAttr.ATimeSec = modificationTimeSec - fissionAttr.MTimeSec = modificationTimeSec - fissionAttr.CTimeSec = statusChangeTimeSec - fissionAttr.ATimeNSec = modificationTimeNSec - fissionAttr.MTimeNSec = modificationTimeNSec - fissionAttr.CTimeNSec = statusChangeTimeNSec - fissionAttr.Mode = computeAttrMode(inode.inodeHeadV1.InodeType, inode.inodeHeadV1.Mode) - fissionAttr.NLink = uint32(len(inode.inodeHeadV1.LinkTable)) - fissionAttr.UID = uint32(inode.inodeHeadV1.UserID) - fissionAttr.GID = uint32(inode.inodeHeadV1.GroupID) - fissionAttr.RDev = attrRDev - fissionAttr.BlkSize = attrBlockSize // Possibly overwritten by fixAttrSizes() - fissionAttr.Padding = 0 - - fixAttrSizes(fissionAttr) - - inodeLockRequest.unlockAll() - - helperWG.Done() -} - func (dummy *globalsStruct) DoRename2(inHeader *fission.InHeader, rename2In *fission.Rename2In) (errno syscall.Errno) { var ( startTime time.Time = time.Now() @@ -2000,6 +2023,93 @@ func computeAttrMode(iLayoutInodeType uint8, iLayoutMode uint16) (attrMode uint3 return } +func gorAttrFetch(inodeNumber uint64, fissionAttr *fission.Attr, gorAttrFetchWG *sync.WaitGroup, gorAttrFetchErrorCount *uint64) { + var ( + err error + ) + + err = doAttrFetch(inodeNumber, fissionAttr) + if nil != err { + globals.Lock() + *gorAttrFetchErrorCount++ + globals.Unlock() + } + + gorAttrFetchWG.Done() +} + +func doAttrFetch(inodeNumber uint64, fissionAttr *fission.Attr) (err error) { + var ( + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + modificationTimeNSec uint32 + modificationTimeSec uint64 + obtainExclusiveLock bool + statusChangeTimeNSec uint32 + statusChangeTimeSec uint64 + ) + + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inodeNumber + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inodeNumber) + if nil == inode { + inodeLockRequest.unlockAll() + err = fmt.Errorf("lookupInode(inodeNumber) returned nil") + return + } + + if nil == inode.inodeHeadV1 { + if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + err = fmt.Errorf("inode.populateInodeHeadV1() failed: %v", err) + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.ModificationTime.UnixNano())) + statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.StatusChangeTime.UnixNano())) + + fissionAttr.Ino = inode.inodeHeadV1.InodeNumber + fissionAttr.Size = inode.inodeHeadV1.Size // Possibly overwritten by fixAttrSizes() + fissionAttr.Blocks = 0 // Computed by fixAttrSizes() + fissionAttr.ATimeSec = modificationTimeSec + fissionAttr.MTimeSec = modificationTimeSec + fissionAttr.CTimeSec = statusChangeTimeSec + fissionAttr.ATimeNSec = modificationTimeNSec + fissionAttr.MTimeNSec = modificationTimeNSec + fissionAttr.CTimeNSec = statusChangeTimeNSec + fissionAttr.Mode = computeAttrMode(inode.inodeHeadV1.InodeType, inode.inodeHeadV1.Mode) + fissionAttr.NLink = uint32(len(inode.inodeHeadV1.LinkTable)) + fissionAttr.UID = uint32(inode.inodeHeadV1.UserID) + fissionAttr.GID = uint32(inode.inodeHeadV1.GroupID) + fissionAttr.RDev = attrRDev + fissionAttr.BlkSize = attrBlockSize // Possibly overwritten by fixAttrSizes() + fissionAttr.Padding = 0 + + inodeLockRequest.unlockAll() + + fixAttrSizes(fissionAttr) + + err = nil + return +} + func fixAttrSizes(attr *fission.Attr) { if syscall.S_IFREG == (attr.Mode & syscall.S_IFMT) { attr.Blocks = attr.Size + (uint64(attrBlockSize) - 1) From d524f6cb05462c0bc3ffd67981909ca8f10df1c2 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 22 Oct 2021 16:47:15 -0700 Subject: [PATCH 133/258] Implemented AdjustInodeTableEntryOpenCount() imgrpkg RPC ...though action on delete/openCount==0 still a TODO... --- iclient/iclientpkg/fission.go | 8 +-- imgr/imgrpkg/api.go | 41 +++++++++---- imgr/imgrpkg/globals.go | 2 + imgr/imgrpkg/retry-rpc.go | 108 +++++++++++++++++++++++++++++++--- imgr/imgrpkg/volume.go | 1 + 5 files changed, 135 insertions(+), 25 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 51b0acfe..30ce7579 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -547,7 +547,7 @@ Retry: err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) if nil != err { - logWarnf("rpcAdjustInodeTableEntryOpenCount() returned err: %v", err) + logFatal(err) } openHandle = createOpenHandle(inode.inodeNumber) @@ -1002,7 +1002,7 @@ Retry: err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) if nil != err { - logWarnf("rpcAdjustInodeTableEntryOpenCount() returned err: %v", err) + logFatal(err) } openHandle.destroy() @@ -1235,7 +1235,7 @@ Retry: err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) if nil != err { - logWarnf("rpcAdjustInodeTableEntryOpenCount() returned err: %v", err) + logFatal(err) } openHandle = createOpenHandle(inode.inodeNumber) @@ -1508,7 +1508,7 @@ Retry: err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) if nil != err { - logWarnf("rpcAdjustInodeTableEntryOpenCount() returned err: %v", err) + logFatal(err) } openHandle.destroy() diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 92833e65..27552df9 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -182,13 +182,14 @@ func LogInfof(format string, args ...interface{}) { // E* specifies the prefix of an error string returned by any RetryRPC API // const ( - EAuthTokenRejected = "EAuthTokenRejected:" - ELeaseRequestDenied = "ELeaseRequestDenied:" - EMissingLease = "EMissingLease:" - EVolumeBeingDeleted = "EVolumeBeingDeleted:" - EUnknownInodeNumber = "EUnknownInodeNumber:" - EUnknownMountID = "EUnknownMountID:" - EUnknownVolumeName = "EUnknownVolumeName:" + EAuthTokenRejected = "EAuthTokenRejected:" + EBadOpenCountAdjustment = "EBadOpenCountAdjustment:" + ELeaseRequestDenied = "ELeaseRequestDenied:" + EMissingLease = "EMissingLease:" + EVolumeBeingDeleted = "EVolumeBeingDeleted:" + EUnknownInodeNumber = "EUnknownInodeNumber:" + EUnknownMountID = "EUnknownMountID:" + EUnknownVolumeName = "EUnknownVolumeName:" ETODO = "ETODO:" ) @@ -232,6 +233,8 @@ type RenewMountResponseStruct struct{} // RenewMount updates the AuthToken for the specified MountID. // +// Possible errors: EAuthTokenRejected EUnknownMountID +// func (dummy *RetryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse *RenewMountResponseStruct) (err error) { return renewMount(renewMountRequest, renewMountResponse) } @@ -249,6 +252,8 @@ type UnmountResponseStruct struct{} // Unmount requests that the given MountID be released (and implicitly releases // any Leases held by the MountID). // +// Possible errors: EAuthTokenRejected EUnknownMountID +// func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountResponseStruct) (err error) { return unmount(unmountRequest, unmountResponse) } @@ -271,6 +276,8 @@ type FetchNonceRangeResponseStruct struct { // FetchNonceRange requests a range of uint64 nonce values (i.e. values that will // never be reused). // +// Possible errors: EAuthTokenRejected EUnknownMountID +// func (dummy *RetryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeResponse *FetchNonceRangeResponseStruct) (err error) { return fetchNonceRange(fetchNonceRangeRequest, fetchNonceRangeResponse) } @@ -292,6 +299,8 @@ type GetInodeTableEntryResponseStruct struct { // GetInodeTableEntry requests the Inode information for the specified Inode // (which must have an active Shared or Exclusive Lease granted to the MountID). // +// Possible errors: EAuthTokenRejected EMissingLease EUnknownInodeNumber EUnknownMountID +// func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { return getInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) } @@ -332,6 +341,8 @@ type PutInodeTableEntriesResponseStruct struct{} // PutInodeTableEntries requests an atomic update of the listed Inodes (which must // each have an active Exclusive Lease granted to the MountID). // +// Possible errors: EAuthTokenRejected EMissingLease EUnknownMountID +// func (dummy *RetryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { return putInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesResponse) } @@ -352,6 +363,8 @@ type DeleteInodeTableEntryResponseStruct struct{} // unless/until the OpenCount for the Inode drops to zero, the Inode will // still exist. // +// Possible errors: ETODO +// func (dummy *RetryRPCServerStruct) DeleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { return deleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) } @@ -372,10 +385,12 @@ type AdjustInodeTableEntryOpenCountResponseStruct struct { } // AdjustInodeTableEntryOpenCount requests the specified Inode's OpenCount be -// adjusted. A (Shared or Exclusive) Lease must be granted to the MountID. If -// the adjustment results in an OpenCount of zero and the Inode has been marked -// for deletion by a prior call to DeleteInodeTableEntry, the Inode will be -// deleted. +// adjusted. If the referenced InodeNumber is non-zero, a (Shared or Exclusive) +// Lease must be granted to the MountID. If the adjustment results in an OpenCount +// of zero and the Inode has been marked for deletion by a prior call to +// DeleteInodeTableEntry, the Inode will be deleted. +// +// Possible errors: EAuthTokenRejected EBadOpenCountAdjustment EMissingLease EUnknownMountID // func (dummy *RetryRPCServerStruct) AdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { return adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) @@ -393,7 +408,7 @@ type FlushResponseStruct struct{} // Flush that the results of prior PutInodeTableEntries requests be persisted. // -// Possible errors: EUnknownMountID +// Possible errors: EAuthTokenRejected EUnknownMountID // func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { return flush(flushRequest, flushResponse) @@ -442,6 +457,8 @@ type LeaseResponseStruct struct { // Lease is a blocking Lease Request. // +// Possible errors: EAuthTokenRejected EUnknownMountID +// func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { return lease(leaseRequest, leaseResponse) } diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index b64d518c..89001940 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -217,6 +217,7 @@ type mountStruct struct { authToken string // lastAuthTime time.Time // used to periodically check TTL of authToken listElement *list.Element // LRU element on either volumeStruct.{healthy|leasesExpired|authTokenExpired}MountList + inodeOpenMap map[uint64]uint64 // key == inodeNumber; value == open count for this mountStruct for this inodeNumber } type inodeTableLayoutElementStruct struct { @@ -245,6 +246,7 @@ type volumeStruct struct { checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG checkPointPutObjectNumber uint64 // checkPointPutObjectBuffer *bytes.Buffer // if nil, no CheckPoint data to PUT has yet accumulated + inodeOpenMap map[uint64]uint64 // key == inodeNumber; value == open count for all mountStruct's for this inodeNumber inodeLeaseMap map[uint64]*inodeLeaseStruct // key == inodeLeaseStruct.inodeNumber leaseHandlerWG sync.WaitGroup // .Add(1) each inodeLease insertion into inodeLeaseMap // .Done() each inodeLease after it is removed from inodeLeaseMap diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 2c47aeb8..fcaea596 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -157,6 +157,7 @@ retryGenerateMountID: authTokenExpired: false, authToken: mountRequest.AuthToken, lastAuthTime: startTime, + inodeOpenMap: make(map[uint64]uint64), } volume.mountMap[mountIDAsString] = mount @@ -211,7 +212,6 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * mount *mountStruct ok bool startTime time.Time = time.Now() - volume *volumeStruct ) defer func() { @@ -227,20 +227,18 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * return } - volume = mount.volume - mount.authToken = renewMountRequest.AuthToken _, err = swiftObjectGet(mount.volume.storageURL, mount.authToken, ilayout.CheckPointObjectNumber) if nil == err { if mount.leasesExpired { - volume.leasesExpiredMountList.MoveToBack(mount.listElement) + mount.volume.leasesExpiredMountList.MoveToBack(mount.listElement) } else { if mount.authTokenExpired { - _ = volume.authTokenExpiredMountList.Remove(mount.listElement) - mount.listElement = volume.healthyMountList.PushBack(mount) + _ = mount.volume.authTokenExpiredMountList.Remove(mount.listElement) + mount.listElement = mount.volume.healthyMountList.PushBack(mount) } else { - volume.healthyMountList.MoveToBack(mount.listElement) + mount.volume.healthyMountList.MoveToBack(mount.listElement) } } } else { @@ -446,19 +444,111 @@ func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRe globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + // TODO: Mark for deletion... and, if inodeOpenCount == 0, kick off delete of inode now + return fmt.Errorf(ETODO + " deleteInodeTableEntry") } func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { var ( - startTime time.Time = time.Now() + leaseRequest *leaseRequestStruct + mount *mountStruct + ok bool + newMountOpenCount uint64 + newVolumeOpenCount uint64 + oldMountOpenCount uint64 + oldVolumeOpenCount uint64 + startTime time.Time = time.Now() ) defer func() { globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf(ETODO + " adjustInodeTableEntryOpenCount") + globals.Lock() + + mount, ok = globals.mountMap[adjustInodeTableEntryOpenCountRequest.MountID] + if !ok { + globals.Unlock() + err = fmt.Errorf("%s %s", EUnknownMountID, adjustInodeTableEntryOpenCountRequest.MountID) + return + } + + if mount.authTokenHasExpired() { + globals.Unlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + return + } + + leaseRequest, ok = mount.leaseRequestMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] + if !ok || ((leaseRequestStateSharedGranted != leaseRequest.requestState) && (leaseRequestStateExclusiveGranted != leaseRequest.requestState)) { + globals.Unlock() + err = fmt.Errorf("%s %016X", EMissingLease, adjustInodeTableEntryOpenCountRequest.InodeNumber) + return + } + + if adjustInodeTableEntryOpenCountRequest.Adjustment >= 0 { + oldMountOpenCount, ok = mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] + if ok { + oldVolumeOpenCount, ok = mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] + if !ok { + logFatalf("mount.volumeinodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] returned !ok after mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] returned ok") + } + } else { + oldMountOpenCount = 0 + oldVolumeOpenCount, ok = mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] + if !ok { + oldVolumeOpenCount = 0 + } + } + + if adjustInodeTableEntryOpenCountRequest.Adjustment == 0 { + newMountOpenCount = oldMountOpenCount + newVolumeOpenCount = oldVolumeOpenCount + } else { + newMountOpenCount = oldMountOpenCount + uint64(adjustInodeTableEntryOpenCountRequest.Adjustment) + newVolumeOpenCount = oldVolumeOpenCount + uint64(adjustInodeTableEntryOpenCountRequest.Adjustment) + + mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = newMountOpenCount + mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = newVolumeOpenCount + } + } else { // adjustInodeTableEntryOpenCountRequest.Adjustment < 0 + oldMountOpenCount, ok = mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] + if !ok || (oldMountOpenCount < uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment)) { + globals.Unlock() + err = fmt.Errorf("%s %016X %v", EBadOpenCountAdjustment, adjustInodeTableEntryOpenCountRequest.InodeNumber, adjustInodeTableEntryOpenCountRequest.Adjustment) + return + } + oldVolumeOpenCount, ok = mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] + if !ok || (oldVolumeOpenCount < uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment)) { + logFatalf("mount.volumeinodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] returned !ok || oldVolumeOpenCount < uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment)") + } + + newMountOpenCount = oldMountOpenCount - uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment) + newVolumeOpenCount = oldVolumeOpenCount - uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment) + + if newMountOpenCount == 0 { + delete(mount.inodeOpenMap, adjustInodeTableEntryOpenCountRequest.InodeNumber) + } else { + mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = newMountOpenCount + } + + if newVolumeOpenCount == 0 { + delete(mount.volume.inodeOpenMap, adjustInodeTableEntryOpenCountRequest.InodeNumber) + + // TODO: Check for pending delete... and, if so, actuially delete the inode + } else { + mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = newVolumeOpenCount + } + } + + globals.Unlock() + + adjustInodeTableEntryOpenCountResponse.CurrentOpenCountThisMount = newMountOpenCount + adjustInodeTableEntryOpenCountResponse.CurrentOpenCountAllMounts = newVolumeOpenCount + + err = nil + return } func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index a316d897..3bae27e6 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -630,6 +630,7 @@ func putVolume(name string, storageURL string, authToken string) (err error) { checkPointControlChan: nil, checkPointPutObjectNumber: 0, checkPointPutObjectBuffer: nil, + inodeOpenMap: make(map[uint64]uint64), inodeLeaseMap: make(map[uint64]*inodeLeaseStruct), } From 3b28b9b0ebdaa163d6ad02eb5a3ff672edc00a5f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 22 Oct 2021 17:01:13 -0700 Subject: [PATCH 134/258] Clarified/corrected descriptions for iclientpkg.inodeStruct.putObject{Number|Buffer} --- iclient/iclientpkg/globals.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index a96492b5..7a388e74 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -113,8 +113,11 @@ type inodeStruct struct { superBlockInodeObjectSizeAdjustment int64 // superBlockInodeBytesReferencedAdjustment int64 // dereferencedObjectNumberArray []uint64 // - putObjectNumber uint64 // For DirInode & FileInode: - putObjectBuffer []byte // ObjectNumber and buffer to PUT during next flush + putObjectNumber uint64 // ObjectNumber to PUT during flush + putObjectBuffer []byte // PUT content to send to .putObjectNumber'd Object: + // For DirInode: marshaled .payload & ilayout.InodeHeadV1Struct + // For FileInode: file extents, marshaled .payload, & ilayout.InodeHeadV1Struct + // For SymLinkInode: marshaled ilayout.InodeHeadV1Struct } type inodeHeldLockStruct struct { From 238330c3f6dd16ced0aaa27aa0fe0f9e23e0fac6 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 22 Oct 2021 17:11:47 -0700 Subject: [PATCH 135/258] Adjusted suggested ICLIENT.FUSE{Entry|Attr}ValidDuration from 0s to 250ms This dramatically cuts down on the number of redundant DoGetAttr() upcalls iclient receives --- iclient/dev.conf | 6 +++--- iclient/iclient.conf | 4 ++-- iclient/iclientpkg/api.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/iclient/dev.conf b/iclient/dev.conf index ae3d641f..e34775e1 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -8,9 +8,9 @@ FUSEAllowOther: true FUSEMaxBackground: 1000 FUSECongestionThreshhold: 0 FUSEMaxWrite: 131076 -FUSEEntryValidDuration: 0s -FUSEAttrValidDuration: 0s -AuthPlugInPath: iauth/iauth-swift/iauth-swift.so +FUSEEntryValidDuration: 250ms +FUSEAttrValidDuration: 250ms +AuthPlugInPath: iauth/iauth-swift/iauth-swift.so AuthPlugInEnvName: AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} SwiftTimeout: 10m diff --git a/iclient/iclient.conf b/iclient/iclient.conf index 96aa1d30..f1ce2094 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -8,8 +8,8 @@ FUSEAllowOther: true FUSEMaxBackground: 1000 FUSECongestionThreshhold: 0 FUSEMaxWrite: 131076 -FUSEEntryValidDuration: 0s -FUSEAttrValidDuration: 0s +FUSEEntryValidDuration: 250ms +FUSEAttrValidDuration: 250ms AuthPlugInPath: iauth-swift.so AuthPlugInEnvName: AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index 74a635e3..20b5e366 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -14,8 +14,8 @@ // FUSEMaxBackground: 1000 // FUSECongestionThreshhold: 0 // FUSEMaxWrite: 131076 -// FUSEEntryValidDuration: 0s -// FUSEAttrValidDuration: 0s +// FUSEEntryValidDuration: 250ms +// FUSEAttrValidDuration: 250ms // AuthPlugInPath: iauth-swift.so // AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here // AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} From 065e428e038a459ab36482a8bc8a6a5b435cfc8d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 25 Oct 2021 15:50:52 -0700 Subject: [PATCH 136/258] Finished up {Dir|File}Inode B+Tree callbacks --- go.mod | 2 +- go.sum | 2 + iclient/iclientpkg/globals.go | 1 + iclient/iclientpkg/inode.go | 162 +++++++++++++++++++++++++++++++--- iclient/iclientpkg/lease.go | 1 + 5 files changed, 157 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 03911183..279fc1ef 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20211020174200-9d6173849985 + golang.org/x/sys v0.0.0-20211022215931-8e5104632af7 ) require ( diff --git a/go.sum b/go.sum index cc93a7fd..9c561665 100644 --- a/go.sum +++ b/go.sum @@ -622,6 +622,8 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjq golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo= golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211022215931-8e5104632af7 h1:e2q1CMOFXDvurT2sa2yhJAkuA2n8Rd9tMDd7Tcfvs6M= +golang.org/x/sys v0.0.0-20211022215931-8e5104632af7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 7a388e74..cf20a8c0 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -100,6 +100,7 @@ type openHandleStruct struct { type inodeStruct struct { sync.WaitGroup // Signaled to indicate .markedForDelete == true triggered removal has completed inodeNumber uint64 // + dirty bool // markedForDelete bool // If true, remove from globalsStruct.inodeTable upon last dereference leaseState inodeLeaseStateType // listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 98d0ec2f..a3fa6552 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -6,6 +6,7 @@ package iclientpkg import ( "encoding/json" "fmt" + "log" "sync" "github.com/NVIDIA/sortedmap" @@ -114,37 +115,131 @@ func (inode *inodeStruct) GetNode(objectNumber uint64, objectOffset uint64, obje } func (inode *inodeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + var ( + layoutMapEntry layoutMapEntryStruct + ok bool + ) + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: - err = fmt.Errorf("TODO") + // Fall through case ilayout.InodeTypeFile: - err = fmt.Errorf("TODO") + // Fall through default: - err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + logFatalf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + inode.ensureLayoutMapIsActive() + + inode.ensurePutObjectIsActive() + + layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] + if !ok { + log.Fatalf("inode.layoutMap[old inode.putObjectNumber] returned !ok") } + objectNumber = inode.putObjectNumber + objectOffset = uint64(len(inode.putObjectBuffer)) + + layoutMapEntry.objectSize += uint64(len(nodeByteSlice)) + layoutMapEntry.bytesReferenced += uint64(len(nodeByteSlice)) + + inode.layoutMap[inode.putObjectNumber] = layoutMapEntry + + inode.putObjectBuffer = append(inode.putObjectBuffer, nodeByteSlice...) + + err = nil return } func (inode *inodeStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + var ( + layoutMapEntry layoutMapEntryStruct + ok bool + ) + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: - err = fmt.Errorf("TODO") + // Fall through case ilayout.InodeTypeFile: - err = fmt.Errorf("TODO") + // Fall through default: - err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + logFatalf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) + } + + inode.ensureLayoutMapIsActive() + + layoutMapEntry, ok = inode.layoutMap[objectNumber] + if !ok { + log.Fatalf("inode.layoutMap[old inode.putObjectNumber] returned !ok") + } + if objectLength > layoutMapEntry.bytesReferenced { + log.Fatalf("objectLength > layoutMapEntry.bytesReferenced") + } + if (objectOffset + objectLength) > layoutMapEntry.objectSize { + log.Fatalf("(objectOffset + objectLength) > layoutMapEntry.objectSize") + } + + // It's ok to update lauoutMap... but note that the above checks don't protect against all double deallocations + + if (objectLength == layoutMapEntry.bytesReferenced) && (objectNumber != inode.putObjectNumber) { + // Note that we skip the special case where we are currently + // discarding the only referenced bytes in an active putObjectBuffer + // since a subsequent flush will write (at least) the inodeHeadV1 there + + delete(inode.layoutMap, objectNumber) + + inode.superBlockInodeObjectCountAdjustment-- + inode.superBlockInodeObjectSizeAdjustment -= int64(layoutMapEntry.objectSize) + inode.superBlockInodeBytesReferencedAdjustment -= int64(objectLength) + + inode.dereferencedObjectNumberArray = append(inode.dereferencedObjectNumberArray, objectNumber) + } else { + layoutMapEntry.bytesReferenced -= objectLength + + inode.layoutMap[objectNumber] = layoutMapEntry + + inode.superBlockInodeBytesReferencedAdjustment -= int64(objectLength) } + err = nil return } func (inode *inodeStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + var ( + keyAsString string + keyAsUint64 uint64 + nextPos int + ok bool + packedKeyLen int + ) + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: - err = fmt.Errorf("TODO") + keyAsString, ok = key.(string) + if ok { + packedKeyLen = 8 + len(keyAsString) + packedKey = make([]byte, packedKeyLen) + nextPos, err = ilayout.PutLEStringToBuf(packedKey, 0, keyAsString) + if (nil == err) && (nextPos != packedKeyLen) { + err = fmt.Errorf("nextPos != packedKeyLen") + } + } else { + err = fmt.Errorf("key.(string) returned !ok") + } case ilayout.InodeTypeFile: - err = fmt.Errorf("TODO") + keyAsUint64, ok = key.(uint64) + if ok { + packedKeyLen = 8 + packedKey = make([]byte, packedKeyLen) + nextPos, err = ilayout.PutLEUint64ToBuf(packedKey, 0, keyAsUint64) + if (nil == err) && (nextPos != packedKeyLen) { + err = fmt.Errorf("nextPos != packedKeyLen") + } + } else { + err = fmt.Errorf("key.(uint64) returned !ok") + } default: err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } @@ -176,11 +271,27 @@ func (inode *inodeStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, byte } func (inode *inodeStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + var ( + valueAsDirectoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + valueAsExtentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct + ok bool + ) + switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: - err = fmt.Errorf("TODO") + valueAsDirectoryEntryValueV1, ok = value.(*ilayout.DirectoryEntryValueV1Struct) + if ok { + packedValue, err = valueAsDirectoryEntryValueV1.MarshalDirectoryEntryValueV1() + } else { + err = fmt.Errorf("value.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } case ilayout.InodeTypeFile: - err = fmt.Errorf("TODO") + valueAsExtentMapEntryValueV1, ok = value.(*ilayout.ExtentMapEntryValueV1Struct) + if ok { + packedValue, err = valueAsExtentMapEntryValueV1.MarshalExtentMapEntryValueV1() + } else { + err = fmt.Errorf("value.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } default: err = fmt.Errorf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } @@ -478,3 +589,34 @@ func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced } } + +func (inode *inodeStruct) ensureLayoutMapIsActive() { + if inode.layoutMap == nil { + inode.convertInodeHeadV1LayoutToLayoutMap() + } +} + +func (inode *inodeStruct) ensurePutObjectIsActive() { + var ( + layoutMapEntry layoutMapEntryStruct + ok bool + ) + + if inode.putObjectNumber == 0 { + inode.putObjectNumber = fetchNonce() + inode.putObjectBuffer = make([]byte, 0) + + layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] + if ok { + log.Fatalf("inode.layoutMap[inode.putObjectNumber] returned ok") + } + layoutMapEntry = layoutMapEntryStruct{ + objectSize: 0, + bytesReferenced: 0, + } + + inode.layoutMap[inode.putObjectNumber] = layoutMapEntry + + inode.superBlockInodeObjectCountAdjustment++ + } +} diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 425ec8ce..34dba4a2 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -98,6 +98,7 @@ Retry: } else { inode = &inodeStruct{ inodeNumber: inodeLockRequest.inodeNumber, + dirty: false, markedForDelete: false, leaseState: inodeLeaseStateNone, listElement: nil, From fa3f738b52b990ec5700d28aecef0d6073a7f1db Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 25 Oct 2021 17:12:20 -0700 Subject: [PATCH 137/258] Added support for statfs() in iclient (and all the underlying RPCs/mechanisms thru imgr --- iclient/dev.conf | 1 + iclient/iclient.conf | 1 + iclient/iclientpkg/api.go | 11 ++++----- iclient/iclientpkg/fission.go | 34 ++++++++++++++++++++++++---- iclient/iclientpkg/globals.go | 7 ++++++ iclient/iclientpkg/rpc.go | 21 +++++++++++++++++- imgr/dev.conf | 4 ++-- imgr/imgrpkg/api.go | 23 +++++++++++++++++++ imgr/imgrpkg/globals.go | 1 + imgr/imgrpkg/retry-rpc.go | 42 +++++++++++++++++++++++++++++++++++ imgr/imgrpkg/volume.go | 19 ++++++++++++++++ 11 files changed, 152 insertions(+), 12 deletions(-) diff --git a/iclient/dev.conf b/iclient/dev.conf index e34775e1..11b20fbf 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -4,6 +4,7 @@ [ICLIENT] VolumeName: testvol MountPointDirPath: /mnt +FUSEBlockSize: 512 FUSEAllowOther: true FUSEMaxBackground: 1000 FUSECongestionThreshhold: 0 diff --git a/iclient/iclient.conf b/iclient/iclient.conf index f1ce2094..e28731c7 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -4,6 +4,7 @@ [ICLIENT] VolumeName: testvol MountPointDirPath: /mnt +FUSEBlockSize: 512 FUSEAllowOther: true FUSEMaxBackground: 1000 FUSECongestionThreshhold: 0 diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index 20b5e366..adcd8cae 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -10,6 +10,7 @@ // [ICLIENT] // VolumeName: testvol // MountPointDirPath: /mnt +// FUSEBlockSize: 512 # Several tools/applications actually require fission.KStatFS.BSize to be 512 // FUSEAllowOther: true // FUSEMaxBackground: 1000 // FUSECongestionThreshhold: 0 @@ -17,19 +18,19 @@ // FUSEEntryValidDuration: 250ms // FUSEAttrValidDuration: 250ms // AuthPlugInPath: iauth-swift.so -// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here +// AuthPlugInEnvName: # Only used if not defining AuthPlugInEnvValue here // AuthPlugInEnvValue: {"AuthURL":"http://swift:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"\u002C"Container":"con"} // SwiftTimeout: 10m // SwiftRetryLimit: 4 // SwiftRetryDelay: 100ms -// SwiftRetryDelayVariance: 25 # Percentage (1-100) of SwitRetryDelay +// SwiftRetryDelayVariance: 25 # Percentage (1-100) of SwitRetryDelay // SwiftRetryExponentialBackoff: 1.4 // SwiftConnectionPoolSize: 128 // RetryRPCPublicIPAddr: imgr // RetryRPCPort: 32356 // RetryRPCDeadlineIO: 60s // RetryRPCKeepAlivePeriod: 60s -// RetryRPCCACertFilePath: # Defaults to /dev/null +// RetryRPCCACertFilePath: # Defaults to /dev/null // MaxSharedLeases: 500 // MaxExclusiveLeases: 100 # Caps pending FileFlush data at 1GiB (with FileFlushTriggerSize of 10MiB) // InodePayloadEvictLowLimit: 100000 @@ -44,8 +45,8 @@ // LogToConsole: true // TraceEnabled: false // FUSELogEnabled: false -// HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) -// HTTPServerPort: # Defaults to disabling the embedded HTTP Server +// HTTPServerIPAddr: # Defaults to 0.0.0.0 (i.e. all interfaces) +// HTTPServerPort: # Defaults to disabling the embedded HTTP Server // // Most of the config keys are required and must have values. One set of exceptions // are the HTTPServer{IPAddr|Port} keys that, if not present (or HTTPServerPort is diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 30ce7579..774f833c 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -5,6 +5,7 @@ package iclientpkg import ( "fmt" + "math" "sync" "syscall" "time" @@ -905,7 +906,10 @@ Retry: func (dummy *globalsStruct) DoStatFS(inHeader *fission.InHeader) (statFSOut *fission.StatFSOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + startTime time.Time = time.Now() + volumeStatusRequest *imgrpkg.VolumeStatusRequestStruct + volumeStatusResponse *imgrpkg.VolumeStatusResponseStruct ) logTracef("==> DoStatFS(inHeader: %+v)", inHeader) @@ -917,9 +921,31 @@ func (dummy *globalsStruct) DoStatFS(inHeader *fission.InHeader) (statFSOut *fis globals.stats.DoStatFSUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - statFSOut = nil - errno = syscall.ENOSYS + volumeStatusRequest = &imgrpkg.VolumeStatusRequestStruct{ + MountID: globals.mountID, + } + volumeStatusResponse = &imgrpkg.VolumeStatusResponseStruct{} + + err = rpcVolumeStatus(volumeStatusRequest, volumeStatusResponse) + if nil != err { + logFatal(err) + } + + statFSOut = &fission.StatFSOut{ + KStatFS: fission.KStatFS{ + Blocks: (volumeStatusResponse.ObjectSize + uint64(globals.config.FUSEBlockSize) - 1) / uint64(globals.config.FUSEBlockSize), + BFree: math.MaxUint64, + BAvail: math.MaxUint64, + Files: volumeStatusResponse.NumInodes, + FFree: math.MaxUint64, + BSize: globals.config.FUSEBlockSize, + FRSize: globals.config.FUSEBlockSize, + Padding: 0, + Spare: [6]uint32{0, 0, 0, 0, 0, 0}, + }, + } + + errno = 0 return } diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index cf20a8c0..1c95e493 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -25,6 +25,7 @@ import ( type configStruct struct { VolumeName string MountPointDirPath string + FUSEBlockSize uint32 FUSEAllowOther bool FUSEMaxBackground uint16 FUSECongestionThreshhold uint16 @@ -169,6 +170,7 @@ type statsStruct struct { PutInodeTableEntriesUsecs bucketstats.BucketLog2Round // (*imgrpkg.RetryRPCServerStruct)PutInodeTableEntries() RenewMountUsecs bucketstats.BucketLog2Round // (*imgrpkg.RetryRPCServerStruct)RenewMount() UnmountUsecs bucketstats.BucketLog2Round // (*imgrpkg.RetryRPCServerStruct)Unmount() + VolumeStatusUsecs bucketstats.BucketLog2Round // (*imgrpkg.RetryRPCServerStruct)VolumeStatus() DoLookupUsecs bucketstats.BucketLog2Round // (*globalsStruct)DoLookup() DoForgetUsecs bucketstats.BucketLog2Round // (*globalsStruct)DoForget() @@ -273,6 +275,10 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err if nil != err { logFatal(err) } + globals.config.FUSEBlockSize, err = confMap.FetchOptionValueUint32("ICLIENT", "FUSEBlockSize") + if nil != err { + logFatal(err) + } globals.config.FUSEAllowOther, err = confMap.FetchOptionValueBool("ICLIENT", "FUSEAllowOther") if nil != err { logFatal(err) @@ -498,6 +504,7 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err func uninitializeGlobals() (err error) { globals.config.VolumeName = "" globals.config.MountPointDirPath = "" + globals.config.FUSEBlockSize = 0 globals.config.FUSEAllowOther = false globals.config.FUSEMaxBackground = 0 globals.config.FUSECongestionThreshhold = 0 diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index 01bfbf55..8bc58eaa 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -390,7 +390,7 @@ func rpcPutInodeTableEntries(putInodeTableEntriesRequest *imgrpkg.PutInodeTableE startTime time.Time = time.Now() ) - logTracef("==> rpcPutInodeTableEntries(rpcPutInodeTableEntries: %+v)", rpcPutInodeTableEntries) + logTracef("==> rpcPutInodeTableEntries(putInodeTableEntriesRequest: %+v)", putInodeTableEntriesRequest) defer func() { logTracef("<== rpcPutInodeTableEntries(putInodeTableEntriesResponse: %+v, err: %v)", putInodeTableEntriesResponse, err) }() @@ -442,6 +442,25 @@ func rpcUnmount(unmountRequest *imgrpkg.UnmountRequestStruct, unmountResponse *i return } +func rpcVolumeStatus(volumeStatusRequest *imgrpkg.VolumeStatusRequestStruct, volumeStatusResponse *imgrpkg.VolumeStatusResponseStruct) (err error) { + var ( + startTime time.Time = time.Now() + ) + + logTracef("==> rpcVolumeStatus(volumeStatusRequest: %+v)", volumeStatusRequest) + defer func() { + logTracef("<== rpcVolumeStatus(volumeStatusResponse: %+v, err: %v)", volumeStatusResponse, err) + }() + + defer func() { + globals.stats.VolumeStatusUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = performMountRenewableRPC("VolumeStatus", volumeStatusRequest, volumeStatusResponse) + + return +} + func objectGETRange(objectNumber uint64, offset uint64, length uint64) (buf []byte, err error) { var ( rangeHeader string diff --git a/imgr/dev.conf b/imgr/dev.conf index bbac8c44..f4eff423 100644 --- a/imgr/dev.conf +++ b/imgr/dev.conf @@ -12,8 +12,8 @@ RetryRPCAckTrim: 100ms RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s -RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, -RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +RetryRPCCertFilePath: +RetryRPCKeyFilePath: CheckPointInterval: 10s diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 27552df9..0918ec9c 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -258,6 +258,29 @@ func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, return unmount(unmountRequest, unmountResponse) } +// VolumeStatusRequestStruct is the request object for VolumeStatus. +// +type VolumeStatusRequestStruct struct { + MountID string +} + +// VolumeStatusResponseStruct is the response object for VolumeStatus. +// +type VolumeStatusResponseStruct struct { + NumInodes uint64 + ObjectCount uint64 + ObjectSize uint64 + BytesReferenced uint64 +} + +// VolumeStatus requests the current status of the mounted volume. +// +// Possible errors: EAuthTokenRejected EUnknownMountID +// +func (dummy *RetryRPCServerStruct) VolumeStatus(volumeStatusRequest *VolumeStatusRequestStruct, volumeStatusResponse *VolumeStatusResponseStruct) (err error) { + return volumeStatus(volumeStatusRequest, volumeStatusResponse) +} + // FetchNonceRangeRequestStruct is the request object for FetchNonceRange. // // Possible errors: EAuthTokenRejected EUnknownMountID diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 89001940..57ee1435 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -104,6 +104,7 @@ type statsStruct struct { PutInodeTableEntriesUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).PutInodeTableEntries() RenewMountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).RenewMount() UnmountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).Unmount() + VolumeStatusUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).VolumeStatus() VolumeCheckPointUsecs bucketstats.BucketLog2Round diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index fcaea596..3d9e5f19 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -262,6 +262,48 @@ func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountRespo return fmt.Errorf(ETODO + " unmount") } +func volumeStatus(volumeStatusRequest *VolumeStatusRequestStruct, volumeStatusResponse *VolumeStatusResponseStruct) (err error) { + var ( + bytesReferenced uint64 + mount *mountStruct + numInodes uint64 + objectCount uint64 + objectSize uint64 + ok bool + startTime time.Time = time.Now() + ) + + defer func() { + globals.stats.VolumeStatusUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + globals.Lock() + + mount, ok = globals.mountMap[volumeStatusRequest.MountID] + if !ok { + globals.Unlock() + err = fmt.Errorf("%s %s", EUnknownMountID, volumeStatusRequest.MountID) + return + } + + if mount.authTokenHasExpired() { + globals.Unlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + return + } + + numInodes, objectCount, objectSize, bytesReferenced = mount.volume.statusWhileLocked() + + globals.Unlock() + + volumeStatusResponse.NumInodes = numInodes + volumeStatusResponse.ObjectCount = objectCount + volumeStatusResponse.ObjectSize = objectSize + volumeStatusResponse.BytesReferenced = bytesReferenced + + return +} + func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeResponse *FetchNonceRangeResponseStruct) (err error) { var ( mount *mountStruct diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 3bae27e6..5606d8d3 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -1100,6 +1100,25 @@ func (volume *volumeStruct) fetchInodeHead(inodeHeadObjectNumber uint64, inodeHe return } +func (volume *volumeStruct) statusWhileLocked() (numInodes uint64, objectCount uint64, objectSize uint64, bytesReferenced uint64) { + var ( + err error + inodeTableLen int + ) + + inodeTableLen, err = volume.inodeTable.Len() + if nil != err { + logFatalf("volume.inodeTable.Len() failed: %v\n", err) + } + numInodes = uint64(inodeTableLen) + + objectCount = volume.superBlock.InodeObjectCount + objectSize = volume.superBlock.InodeObjectSize + bytesReferenced = volume.superBlock.InodeBytesReferenced + + return +} + func (volume *volumeStruct) fetchNonceRangeWhileLocked() (nextNonce uint64, numNoncesFetched uint64, err error) { var ( authOK bool From 586f7efedcd8714786e6f57420be39bc2da21b4f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 26 Oct 2021 20:04:19 -0700 Subject: [PATCH 138/258] Completed inode flush logic in iclient Also added optional "-f" to format volume in imgr/mkmount.sh & iclient/iclient.sh --- docker-compose.yml | 2 +- iclient/iclient.sh | 11 ++- iclient/iclientpkg/inode.go | 156 +++++++++++++++++++++++++++++++++--- iclient/iclientpkg/rpc.go | 52 ++++++++++++ imgr/imgrpkg/volume.go | 6 +- imgr/mkmount.sh | 11 ++- 6 files changed, 220 insertions(+), 18 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fb4107a7..dc0c0bf7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,5 +128,5 @@ services: published: 15347 protocol: tcp mode: host - command: ["./iclient.sh", "-s"] + command: ["./iclient.sh", "-fs"] \ No newline at end of file diff --git a/iclient/iclient.sh b/iclient/iclient.sh index 69ebe6aa..69888a8b 100755 --- a/iclient/iclient.sh +++ b/iclient/iclient.sh @@ -2,14 +2,18 @@ Usage="$(basename "$0") - Ask swift/imgr to format and serve testvol... and then sleep where: + -f format volume -h show this help text -s supply static AuthToken to imgr" +DoFormat=false StaticAuthToken=false -while getopts 'hs' option +while getopts 'fhs' option do case "$option" in + f) DoFormat=true + ;; h) echo "$Usage" exit 0 ;; @@ -40,7 +44,10 @@ do DollarQuestionMark=$? done -curl -v -s -f imgr:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" +if $DoFormat +then + curl -v -s -f imgr:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" +fi if $StaticAuthToken then diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index a3fa6552..14b82559 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -129,8 +129,6 @@ func (inode *inodeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, ob logFatalf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - inode.ensureLayoutMapIsActive() - inode.ensurePutObjectIsActive() layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] @@ -606,17 +604,155 @@ func (inode *inodeStruct) ensurePutObjectIsActive() { inode.putObjectNumber = fetchNonce() inode.putObjectBuffer = make([]byte, 0) - layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] - if ok { - log.Fatalf("inode.layoutMap[inode.putObjectNumber] returned ok") + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeSymLink { + inode.ensureLayoutMapIsActive() + + layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] + if ok { + log.Fatalf("inode.layoutMap[inode.putObjectNumber] returned ok") + } + layoutMapEntry = layoutMapEntryStruct{ + objectSize: 0, + bytesReferenced: 0, + } + + inode.layoutMap[inode.putObjectNumber] = layoutMapEntry + } + + inode.superBlockInodeObjectCountAdjustment++ + } +} + +// flush marshals the current state of an assumed to be dirty inode into its .putObjectBuffer. +// This .putObjectBuffer is then passed to objectPUT() following which the inode is marked clean. +// +// The return value indicates the size of the marshaled .inodeHeadV1 such that, along with the +// .superBlockInode*, .dereferencedObjectNumberArray, and .putObjectNumber fields, will be used +// to construct this inode's portion of an imgrpkg.PutInodeTableEntriesRequestStruct. +// +// The caller is assumed to mark the inode clean and reset the .superBlockInode*, +// .dereferencedObjectNumberArray, and .putObject{Number|Buffer} fields. +// +func (inode *inodeStruct) flush() (inodeHeadLength uint64) { + var ( + err error + inodeHeadV1Buf []byte + ) + + inode.ensurePutObjectIsActive() + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeSymLink { + if inode.payload != nil { + inode.inodeHeadV1.PayloadObjectNumber, inode.inodeHeadV1.PayloadObjectOffset, inode.inodeHeadV1.PayloadObjectLength, err = inode.payload.Flush(false) + if nil != err { + logFatalf("inode.payload.Flush(false) failed: %v", err) + } + + err = inode.payload.Prune() + if nil != err { + logFatalf("inode.payload.Prune() failed: %v", err) + } } - layoutMapEntry = layoutMapEntryStruct{ - objectSize: 0, - bytesReferenced: 0, + + if inode.layoutMap != nil { + inode.convertLayoutMapToInodeHeadV1Layout() } + } - inode.layoutMap[inode.putObjectNumber] = layoutMapEntry + inodeHeadV1Buf, err = inode.inodeHeadV1.MarshalInodeHeadV1() + if nil != err { + logFatalf("inode.inodeHeadV1.MarshalInodeHeadV1() failed: %v", err) + } - inode.superBlockInodeObjectCountAdjustment++ + inode.putObjectBuffer = append(inode.putObjectBuffer, inodeHeadV1Buf...) + + err = objectPUT(inode.putObjectNumber, inode.putObjectBuffer) + if nil != err { + logFatalf("objectPUT(inode.putObjectNumber, inode.putObjectBuffer) failed: %v", err) + } + + inodeHeadLength = uint64(len(inodeHeadV1Buf)) + return +} + +func flushInodeNumbersInSlice(inodeNumberSlice []uint64) { + var ( + inodeNumber uint64 + inodeSlice []*inodeStruct + inodeSliceIndex int + ok bool + ) + + inodeSlice = make([]*inodeStruct, len(inodeNumberSlice)) + + globals.Lock() + + for inodeSliceIndex, inodeNumber = range inodeNumberSlice { + inodeSlice[inodeSliceIndex], ok = globals.inodeTable[inodeNumber] + if !ok { + logFatalf("globals.inodeTable[inodeNumber: %016X] returned !ok", inodeNumber) + } + } + + globals.Unlock() + + flushInodesInSlice(inodeSlice) +} + +func flushInodesInSlice(inodeSlice []*inodeStruct) { + var ( + dereferencedObjectNumber uint64 + err error + inode *inodeStruct + inodeHeadLength uint64 + inodeSliceIndex int + putInodeTableEntriesRequest *imgrpkg.PutInodeTableEntriesRequestStruct + putInodeTableEntriesResponse *imgrpkg.PutInodeTableEntriesResponseStruct + ) + + putInodeTableEntriesRequest = &imgrpkg.PutInodeTableEntriesRequestStruct{ + MountID: globals.mountID, + UpdatedInodeTableEntryArray: make([]imgrpkg.PutInodeTableEntryStruct, len(inodeSlice)), + SuperBlockInodeObjectCountAdjustment: 0, + SuperBlockInodeObjectSizeAdjustment: 0, + SuperBlockInodeBytesReferencedAdjustment: 0, + DereferencedObjectNumberArray: make([]uint64, 0), + } + putInodeTableEntriesResponse = &imgrpkg.PutInodeTableEntriesResponseStruct{} + + for inodeSliceIndex, inode = range inodeSlice { + if !inode.dirty { + logFatalf("inode.dirty for inodeNumber %016X is false", inode.inodeNumber) + } + + inodeHeadLength = inode.flush() + + putInodeTableEntriesRequest.UpdatedInodeTableEntryArray[inodeSliceIndex].InodeNumber = inode.inodeNumber + putInodeTableEntriesRequest.UpdatedInodeTableEntryArray[inodeSliceIndex].InodeHeadObjectNumber = inode.putObjectNumber + putInodeTableEntriesRequest.UpdatedInodeTableEntryArray[inodeSliceIndex].InodeHeadLength = inodeHeadLength + + putInodeTableEntriesRequest.SuperBlockInodeObjectCountAdjustment += inode.superBlockInodeObjectCountAdjustment + putInodeTableEntriesRequest.SuperBlockInodeObjectSizeAdjustment += inode.superBlockInodeObjectSizeAdjustment + putInodeTableEntriesRequest.SuperBlockInodeBytesReferencedAdjustment += inode.superBlockInodeBytesReferencedAdjustment + + for _, dereferencedObjectNumber = range inode.dereferencedObjectNumberArray { + putInodeTableEntriesRequest.DereferencedObjectNumberArray = append(putInodeTableEntriesRequest.DereferencedObjectNumberArray, dereferencedObjectNumber) + } + + inode.dirty = false + + inode.superBlockInodeObjectCountAdjustment = 0 + inode.superBlockInodeObjectSizeAdjustment = 0 + inode.superBlockInodeBytesReferencedAdjustment = 0 + + inode.dereferencedObjectNumberArray = make([]uint64, 0) + + inode.putObjectNumber = 0 + inode.putObjectBuffer = nil + } + + err = rpcPutInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesResponse) + if nil != err { + logFatalf("rpcPutInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesResponse) failed: %v", err) } } diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index 8bc58eaa..fd330f87 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -4,6 +4,7 @@ package iclientpkg import ( + "bytes" "fmt" "io/ioutil" "math/rand" @@ -536,3 +537,54 @@ func objectGETWithRangeHeader(objectNumber uint64, rangeHeader string) (buf []by retryIndex++ } } + +func objectPUT(objectNumber uint64, buf []byte) (err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + retryDelay time.Duration + retryIndex uint64 + ) + + retryIndex = 0 + + for { + httpRequest, err = http.NewRequest("PUT", fetchSwiftStorageURL()+"/"+ilayout.GetObjectNameAsString(objectNumber), bytes.NewBuffer(buf)) + if nil != err { + return + } + + httpRequest.Header["User-Agent"] = []string{HTTPUserAgent} + httpRequest.Header["X-Auth-Token"] = []string{fetchSwiftAuthToken()} + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + logFatalf("globals.httpClient.Do(httpRequest) failed: %v", err) + } + + _, err = ioutil.ReadAll(httpResponse.Body) + _ = httpResponse.Body.Close() + if nil != err { + logFatalf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + err = nil + return + } + + if retryIndex >= globals.config.SwiftRetryLimit { + err = fmt.Errorf("objectPUT(objectNumber: %v,) reached SwiftRetryLimit", objectNumber) + logWarn(err) + return + } + + if http.StatusUnauthorized == httpResponse.StatusCode { + updateSwithAuthTokenAndSwiftStorageURL() + } + + retryDelay = globals.swiftRetryDelay[retryIndex].nominal - time.Duration(rand.Int63n(int64(globals.swiftRetryDelay[retryIndex].variance))) + time.Sleep(retryDelay) + retryIndex++ + } +} diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 5606d8d3..392ba218 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -748,8 +748,8 @@ func (volume *volumeStruct) doCheckPoint() (err error) { } volume.superBlock.InodeTableRootObjectNumber = inodeTableRootObjectNumber - volume.superBlock.InodeTableRootObjectNumber = inodeTableRootObjectOffset - volume.superBlock.InodeTableRootObjectNumber = inodeTableRootObjectLength + volume.superBlock.InodeTableRootObjectOffset = inodeTableRootObjectOffset + volume.superBlock.InodeTableRootObjectLength = inodeTableRootObjectLength volume.superBlock.InodeTableLayout = make([]ilayout.InodeTableLayoutEntryV1Struct, 0, len(volume.inodeTableLayout)) @@ -800,7 +800,7 @@ func (volume *volumeStruct) doCheckPoint() (err error) { _, _ = volume.checkPointPutObjectBuffer.Write(superBlockV1Buf) volume.checkPoint.SuperBlockObjectNumber = volume.checkPointPutObjectNumber - volume.checkPoint.SuperBlockLength = uint64(volume.checkPointPutObjectBuffer.Len()) + volume.checkPoint.SuperBlockLength = uint64(len(superBlockV1Buf)) authOK, err = volume.swiftObjectPut(volume.checkPointPutObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) if nil != err { diff --git a/imgr/mkmount.sh b/imgr/mkmount.sh index 7c5cde45..770bba84 100755 --- a/imgr/mkmount.sh +++ b/imgr/mkmount.sh @@ -2,14 +2,18 @@ Usage="$(basename "$0") - Ask swift/imgr to format and serve testvol where: + -f format volume -h show this help text -s supply static AuthToken to imgr" +DoFormat=false StaticAuthToken=false -while getopts 'hs' option +while getopts 'fhs' option do case "$option" in + f) DoFormat=true + ;; h) echo "$Usage" exit 0 ;; @@ -44,7 +48,10 @@ do DollarQuestionMark=$? done -curl -s -f dev:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" > /dev/null +if $DoFormat +then + curl -s -f dev:15346/volume -X POST -d "{\"StorageURL\":\"http://swift:8080/v1/AUTH_test/con\",\"AuthToken\":\"$AuthToken\"}" > /dev/null +fi if $StaticAuthToken then From 66b4a040f23706cb2da94a905e9052ffef34c826 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 26 Oct 2021 20:15:00 -0700 Subject: [PATCH 139/258] Pick up latest go.mod modifications --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 279fc1ef..f04fbde2 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20211022215931-8e5104632af7 + golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 ) require ( diff --git a/go.sum b/go.sum index 9c561665..6e3f5ee3 100644 --- a/go.sum +++ b/go.sum @@ -624,6 +624,8 @@ golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwk golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211022215931-8e5104632af7 h1:e2q1CMOFXDvurT2sa2yhJAkuA2n8Rd9tMDd7Tcfvs6M= golang.org/x/sys v0.0.0-20211022215931-8e5104632af7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From fdfdb13b5a4e968fe0874c69b4f45ce5907e9143 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 27 Oct 2021 11:45:28 -0700 Subject: [PATCH 140/258] Now SymLink inodes are supported... --- iclient/iclientpkg/fission.go | 232 +++++++++++++++++++++++++++++++-- imgr/imgrpkg/html_templates.go | 2 +- 2 files changed, 222 insertions(+), 12 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 774f833c..9bc812d2 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -4,6 +4,7 @@ package iclientpkg import ( + "container/list" "fmt" "math" "sync" @@ -322,7 +323,11 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis func (dummy *globalsStruct) DoReadLink(inHeader *fission.InHeader) (readLinkOut *fission.ReadLinkOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + startTime time.Time = time.Now() ) logTracef("==> DoReadLink(inHeader: %+v)", inHeader) @@ -334,15 +339,71 @@ func (dummy *globalsStruct) DoReadLink(inHeader *fission.InHeader) (readLinkOut globals.stats.DoReadLinkUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - readLinkOut = nil - errno = syscall.ENOSYS + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + readLinkOut = nil + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + readLinkOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeSymLink { + inodeLockRequest.unlockAll() + readLinkOut = nil + errno = syscall.EINVAL + return + } + + readLinkOut = &fission.ReadLinkOut{ + Data: []byte(inode.inodeHeadV1.SymLinkTarget), + } + + inodeLockRequest.unlockAll() + + errno = 0 return } func (dummy *globalsStruct) DoSymLink(inHeader *fission.InHeader, symLinkIn *fission.SymLinkIn) (symLinkOut *fission.SymLinkOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + dirInode *inodeStruct + err error + inodeLockRequest *inodeLockRequestStruct + modificationTimeNSec uint32 + modificationTimeSec uint64 + ok bool + startTime time.Time = time.Now() + statusChangeTimeNSec uint32 + statusChangeTimeSec uint64 + symLinkInode *inodeStruct + symLinkInodeNumber uint64 ) logTracef("==> DoSymLink(inHeader: %+v, symLinkIn: %+v)", inHeader, symLinkIn) @@ -354,9 +415,158 @@ func (dummy *globalsStruct) DoSymLink(inHeader *fission.InHeader, symLinkIn *fis globals.stats.DoSymLinkUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - symLinkOut = nil - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + dirInode = lookupInode(inHeader.NodeID) + if nil == dirInode { + inodeLockRequest.unlockAll() + symLinkOut = nil + errno = syscall.ENOENT + return + } + + if nil == dirInode.inodeHeadV1 { + err = dirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + symLinkOut = nil + errno = syscall.ENOENT + return + } + } + + if dirInode.payload == nil { + err = dirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + symLinkOut = nil + errno = syscall.ENOENT + return + } + } + + _, ok, err = dirInode.payload.GetByKey(string(symLinkIn.Name[:])) + if nil != err { + logFatalf("dirInode.payload.GetByKey(string(symLinkIn.Name[:])) failed: %v", err) + } + if ok { + inodeLockRequest.unlockAll() + symLinkOut = nil + errno = syscall.EEXIST + return + } + + symLinkInodeNumber = fetchNonce() + + symLinkInode = &inodeStruct{ + inodeNumber: symLinkInodeNumber, + dirty: true, + markedForDelete: false, + leaseState: inodeLeaseStateNone, + listElement: nil, + heldList: list.New(), + requestList: list.New(), + inodeHeadV1: &ilayout.InodeHeadV1Struct{ + InodeNumber: symLinkInodeNumber, + InodeType: ilayout.InodeTypeSymLink, + LinkTable: []ilayout.InodeLinkTableEntryStruct{ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: dirInode.inodeNumber, + ParentDirEntryName: string(symLinkIn.Name[:]), + }}, + Size: 0, + ModificationTime: startTime, + StatusChangeTime: startTime, + Mode: ilayout.InodeModeMask, + UserID: uint64(inHeader.UID), + GroupID: uint64(inHeader.GID), + StreamTable: make([]ilayout.InodeStreamTableEntryStruct, 0), + PayloadObjectNumber: 0, + PayloadObjectOffset: 0, + PayloadObjectLength: 0, + SymLinkTarget: string(symLinkIn.Data[:]), + Layout: nil, + }, + payload: nil, + layoutMap: nil, + superBlockInodeObjectCountAdjustment: 0, + superBlockInodeObjectSizeAdjustment: 0, + superBlockInodeBytesReferencedAdjustment: 0, + dereferencedObjectNumberArray: make([]uint64, 0), + putObjectNumber: 0, + putObjectBuffer: nil, + } + + inodeLockRequest.inodeNumber = symLinkInodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + dirInode.dirty = true + + dirInode.inodeHeadV1.ModificationTime = startTime + dirInode.inodeHeadV1.StatusChangeTime = startTime + + ok, err = dirInode.payload.Put(string( + symLinkIn.Name[:]), + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: symLinkInodeNumber, + InodeType: ilayout.InodeTypeSymLink, + }) + if nil != err { + logFatalf("dirInode.payload.Put(string(symLinkIn.Name[:]),) failed: %v", err) + } + if !ok { + logFatalf("dirInode.payload.Put(string(symLinkIn.Name[:]),) returned !ok") + } + + flushInodesInSlice([]*inodeStruct{dirInode, symLinkInode}) + + modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(startTime.UnixNano())) + statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(startTime.UnixNano())) + + symLinkOut = &fission.SymLinkOut{ + EntryOut: fission.EntryOut{ + NodeID: symLinkInode.inodeHeadV1.InodeNumber, + Generation: 0, + EntryValidSec: globals.fuseEntryValidDurationSec, + AttrValidSec: globals.fuseAttrValidDurationSec, + EntryValidNSec: globals.fuseEntryValidDurationNSec, + AttrValidNSec: globals.fuseAttrValidDurationNSec, + Attr: fission.Attr{ + Ino: symLinkInode.inodeHeadV1.InodeNumber, + Size: symLinkInode.inodeHeadV1.Size, // Possibly overwritten by fixAttrSizes() + Blocks: 0, // Computed by fixAttrSizes() + ATimeSec: modificationTimeSec, + MTimeSec: modificationTimeSec, + CTimeSec: statusChangeTimeSec, + ATimeNSec: modificationTimeNSec, + MTimeNSec: modificationTimeNSec, + CTimeNSec: statusChangeTimeNSec, + Mode: computeAttrMode(symLinkInode.inodeHeadV1.InodeType, symLinkInode.inodeHeadV1.Mode), + NLink: uint32(len(symLinkInode.inodeHeadV1.LinkTable)), + UID: uint32(symLinkInode.inodeHeadV1.UserID), + GID: uint32(symLinkInode.inodeHeadV1.GroupID), + RDev: attrRDev, + BlkSize: attrBlockSize, // Possibly overwritten by fixAttrSizes() + Padding: 0, + }, + }, + } + + fixAttrSizes(&symLinkOut.EntryOut.Attr) + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -798,8 +1008,8 @@ Retry: default: // TODO - need to actually read from the cache (in a coherent/cooperative way) // TODO - need to handle case where extent crosses cache line boundary - // UNDO - but for now, we will simply append zeroes - readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length, readPlanEntry.Length)...) // UNDO + // TODO - but for now, we will simply append zeroes + readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length, readPlanEntry.Length)...) // TODO } } @@ -899,7 +1109,7 @@ Retry: inodeLockRequest.unlockAll() - writeOut = nil // UNDO + writeOut = nil // TODO errno = syscall.ENOSYS return } diff --git a/imgr/imgrpkg/html_templates.go b/imgr/imgrpkg/html_templates.go index fc54e747..96391f66 100644 --- a/imgr/imgrpkg/html_templates.go +++ b/imgr/imgrpkg/html_templates.go @@ -479,7 +479,7 @@ const inodeTemplate string = `

Inode - 1 + %[3]v

Volume testvol

From eea4bda24e329398391940105e65dea24fc81650 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 27 Oct 2021 12:25:47 -0700 Subject: [PATCH 141/258] Now Dir inodes are supported... --- iclient/iclientpkg/fission.go | 228 ++++++++++++++++++++++++++++++++-- 1 file changed, 218 insertions(+), 10 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 9bc812d2..f599f8b5 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -476,10 +476,12 @@ Retry: inodeHeadV1: &ilayout.InodeHeadV1Struct{ InodeNumber: symLinkInodeNumber, InodeType: ilayout.InodeTypeSymLink, - LinkTable: []ilayout.InodeLinkTableEntryStruct{ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: dirInode.inodeNumber, - ParentDirEntryName: string(symLinkIn.Name[:]), - }}, + LinkTable: []ilayout.InodeLinkTableEntryStruct{ + ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: dirInode.inodeNumber, + ParentDirEntryName: string(symLinkIn.Name[:]), + }, + }, Size: 0, ModificationTime: startTime, StatusChangeTime: startTime, @@ -515,8 +517,8 @@ Retry: dirInode.inodeHeadV1.ModificationTime = startTime dirInode.inodeHeadV1.StatusChangeTime = startTime - ok, err = dirInode.payload.Put(string( - symLinkIn.Name[:]), + ok, err = dirInode.payload.Put( + string(symLinkIn.Name[:]), &ilayout.DirectoryEntryValueV1Struct{ InodeNumber: symLinkInodeNumber, InodeType: ilayout.InodeTypeSymLink, @@ -591,7 +593,17 @@ func (dummy *globalsStruct) DoMkNod(inHeader *fission.InHeader, mkNodIn *fission func (dummy *globalsStruct) DoMkDir(inHeader *fission.InHeader, mkDirIn *fission.MkDirIn) (mkDirOut *fission.MkDirOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + childDirInode *inodeStruct + childDirInodeNumber uint64 + err error + inodeLockRequest *inodeLockRequestStruct + modificationTimeNSec uint32 + modificationTimeSec uint64 + ok bool + parentDirInode *inodeStruct + startTime time.Time = time.Now() + statusChangeTimeNSec uint32 + statusChangeTimeSec uint64 ) logTracef("==> DoMkDir(inHeader: %+v, mkDirIn: %+v)", inHeader, mkDirIn) @@ -603,9 +615,205 @@ func (dummy *globalsStruct) DoMkDir(inHeader *fission.InHeader, mkDirIn *fission globals.stats.DoMkDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - mkDirOut = nil - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + parentDirInode = lookupInode(inHeader.NodeID) + if nil == parentDirInode { + inodeLockRequest.unlockAll() + mkDirOut = nil + errno = syscall.ENOENT + return + } + + if nil == parentDirInode.inodeHeadV1 { + err = parentDirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + mkDirOut = nil + errno = syscall.ENOENT + return + } + } + + if parentDirInode.payload == nil { + err = parentDirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + mkDirOut = nil + errno = syscall.ENOENT + return + } + } + + _, ok, err = parentDirInode.payload.GetByKey(string(mkDirIn.Name[:])) + if nil != err { + logFatalf("parentDirInode.payload.GetByKey(string(mkDirIn.Name[:])) failed: %v", err) + } + if ok { + inodeLockRequest.unlockAll() + mkDirOut = nil + errno = syscall.EEXIST + return + } + + childDirInodeNumber = fetchNonce() + + childDirInode = &inodeStruct{ + inodeNumber: childDirInodeNumber, + dirty: true, + markedForDelete: false, + leaseState: inodeLeaseStateNone, + listElement: nil, + heldList: list.New(), + requestList: list.New(), + inodeHeadV1: &ilayout.InodeHeadV1Struct{ + InodeNumber: childDirInodeNumber, + InodeType: ilayout.InodeTypeDir, + LinkTable: []ilayout.InodeLinkTableEntryStruct{ + ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: parentDirInode.inodeNumber, + ParentDirEntryName: string(mkDirIn.Name[:]), + }, + ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: childDirInodeNumber, + ParentDirEntryName: ".", + }, + }, + Size: 0, + ModificationTime: startTime, + StatusChangeTime: startTime, + Mode: ilayout.InodeModeMask, + UserID: uint64(inHeader.UID), + GroupID: uint64(inHeader.GID), + StreamTable: make([]ilayout.InodeStreamTableEntryStruct, 0), + PayloadObjectNumber: 0, + PayloadObjectOffset: 0, + PayloadObjectLength: 0, + SymLinkTarget: "", + Layout: nil, + }, + payload: nil, + layoutMap: nil, + superBlockInodeObjectCountAdjustment: 0, + superBlockInodeObjectSizeAdjustment: 0, + superBlockInodeBytesReferencedAdjustment: 0, + dereferencedObjectNumberArray: make([]uint64, 0), + putObjectNumber: 0, + putObjectBuffer: nil, + } + + err = childDirInode.newPayload() + if nil != err { + logFatalf("childDirInode.newPayload() failed: %v\n", err) + } + + ok, err = childDirInode.payload.Put( + ".", + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: childDirInodeNumber, + InodeType: ilayout.InodeTypeDir, + }, + ) + if nil != err { + logFatalf("parentDirInode.payload.Put(\".\",) failed: %v", err) + } + if !ok { + logFatalf("parentDirInode.payload.Put(\".\",) returned !ok") + } + + ok, err = childDirInode.payload.Put( + "..", + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: parentDirInode.inodeNumber, + InodeType: ilayout.InodeTypeDir, + }, + ) + if nil != err { + logFatalf("parentDirInode.payload.Put(\"..\",) failed: %v", err) + } + if !ok { + logFatalf("parentDirInode.payload.Put(\"..\",) returned !ok") + } + + inodeLockRequest.inodeNumber = childDirInodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + parentDirInode.dirty = true + + parentDirInode.inodeHeadV1.LinkTable = append( + parentDirInode.inodeHeadV1.LinkTable, + ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: childDirInode.inodeNumber, + ParentDirEntryName: "..", + }, + ) + + parentDirInode.inodeHeadV1.ModificationTime = startTime + parentDirInode.inodeHeadV1.StatusChangeTime = startTime + + ok, err = parentDirInode.payload.Put( + string(mkDirIn.Name[:]), + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: childDirInodeNumber, + InodeType: ilayout.InodeTypeDir, + }) + if nil != err { + logFatalf("parentDirInode.payload.Put(string(mkDirIn.Name[:]),) failed: %v", err) + } + if !ok { + logFatalf("parentDirInode.payload.Put(string(mkDirIn.Name[:]),) returned !ok") + } + + flushInodesInSlice([]*inodeStruct{parentDirInode, childDirInode}) + + modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(startTime.UnixNano())) + statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(startTime.UnixNano())) + + mkDirOut = &fission.MkDirOut{ + EntryOut: fission.EntryOut{ + NodeID: childDirInode.inodeHeadV1.InodeNumber, + Generation: 0, + EntryValidSec: globals.fuseEntryValidDurationSec, + AttrValidSec: globals.fuseAttrValidDurationSec, + EntryValidNSec: globals.fuseEntryValidDurationNSec, + AttrValidNSec: globals.fuseAttrValidDurationNSec, + Attr: fission.Attr{ + Ino: childDirInode.inodeHeadV1.InodeNumber, + Size: childDirInode.inodeHeadV1.Size, // Possibly overwritten by fixAttrSizes() + Blocks: 0, // Computed by fixAttrSizes() + ATimeSec: modificationTimeSec, + MTimeSec: modificationTimeSec, + CTimeSec: statusChangeTimeSec, + ATimeNSec: modificationTimeNSec, + MTimeNSec: modificationTimeNSec, + CTimeNSec: statusChangeTimeNSec, + Mode: computeAttrMode(childDirInode.inodeHeadV1.InodeType, childDirInode.inodeHeadV1.Mode), + NLink: uint32(len(childDirInode.inodeHeadV1.LinkTable)), + UID: uint32(childDirInode.inodeHeadV1.UserID), + GID: uint32(childDirInode.inodeHeadV1.GroupID), + RDev: attrRDev, + BlkSize: attrBlockSize, // Possibly overwritten by fixAttrSizes() + Padding: 0, + }, + }, + } + + fixAttrSizes(&mkDirOut.EntryOut.Attr) + + inodeLockRequest.unlockAll() + + errno = 0 return } From 712f9397387970ca90e674f5831846f6c88e1da3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 27 Oct 2021 16:50:05 -0700 Subject: [PATCH 142/258] Converted imgr parsing of GET /volume//inode/ to assume decimal Inode# This matches the way the HTML/JS code was displaying/specifying InodeNumber's --- imgr/imgrpkg/http-server.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index be3bf4be..eb3cfe0d 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -654,9 +654,9 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ inodeGETStreamTableEntryValueByte byte inodeGETStreamTableEntryValueIndex int inodeHeadV1 *ilayout.InodeHeadV1Struct - inodeNumberAsHexDigits string - inodeNumberAsUint64 uint64 + inodeNumberAsDecimalString string inodeNumberAsKey sortedmap.Key + inodeNumberAsUint64 uint64 inodeTableEntryValueV1 *ilayout.InodeTableEntryValueV1Struct inodeTableEntryValueV1AsValue sortedmap.Value inodeTableIndex int @@ -917,11 +917,11 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeName = pathSplit[2] mustBeInode = pathSplit[3] - inodeNumberAsHexDigits = pathSplit[4] + inodeNumberAsDecimalString = pathSplit[4] switch mustBeInode { case "inode": - inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAsHexDigits, 16, 64) + inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAsDecimalString, 10, 64) if nil != err { responseWriter.WriteHeader(http.StatusBadRequest) } else { From 05fe29e91395cec390eb3e80d0c1467f30f21370 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 27 Oct 2021 18:43:31 -0700 Subject: [PATCH 143/258] And, finally, now file inodes are supported... --- iclient/iclientpkg/fission.go | 252 +++++++++++++++++++++++++++++++--- 1 file changed, 235 insertions(+), 17 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index f599f8b5..33dfe862 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -689,7 +689,7 @@ Retry: Size: 0, ModificationTime: startTime, StatusChangeTime: startTime, - Mode: ilayout.InodeModeMask, + Mode: uint16(mkDirIn.Mode & ^mkDirIn.UMask) & ilayout.InodeModeMask, UserID: uint64(inHeader.UID), GroupID: uint64(inHeader.GID), StreamTable: make([]ilayout.InodeStreamTableEntryStruct, 0), @@ -915,6 +915,9 @@ func (dummy *globalsStruct) DoOpen(inHeader *fission.InHeader, openIn *fission.O globals.stats.DoOpenUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + // TODO: Validate simply ignoring openIn.Flags containing fission.FOpenRequestEXCL is ok + // TODO: Need to handle openIn.Flags containing fission.FOpenRequestCREAT + obtainExclusiveLock = false Retry: @@ -957,6 +960,8 @@ Retry: return } + // TODO: Need to truncate file if openIn.Flags contains fission.FOpenRequestTRUNC + adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ MountID: globals.mountID, InodeNumber: inode.inodeNumber, @@ -2059,7 +2064,22 @@ func (dummy *globalsStruct) DoAccess(inHeader *fission.InHeader, accessIn *fissi func (dummy *globalsStruct) DoCreate(inHeader *fission.InHeader, createIn *fission.CreateIn) (createOut *fission.CreateOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + adjustInodeTableEntryOpenCountRequest *imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct + adjustInodeTableEntryOpenCountResponse *imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + dirInode *inodeStruct + err error + fileInode *inodeStruct + fileInodeNumber uint64 + inodeLockRequest *inodeLockRequestStruct + // modificationTimeNSec uint32 + // modificationTimeSec uint64 + ok bool + openHandle *openHandleStruct + startTime time.Time = time.Now() + // statusChangeTimeNSec uint32 + // statusChangeTimeSec uint64 ) logTracef("==> DoCreate(inHeader: %+v, createIn: %+v)", inHeader, createIn) @@ -2071,9 +2091,200 @@ func (dummy *globalsStruct) DoCreate(inHeader *fission.InHeader, createIn *fissi globals.stats.DoCreateUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - createOut = nil - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + dirInode = lookupInode(inHeader.NodeID) + if nil == dirInode { + inodeLockRequest.unlockAll() + createOut = nil + errno = syscall.ENOENT + return + } + + if nil == dirInode.inodeHeadV1 { + err = dirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + createOut = nil + errno = syscall.ENOENT + return + } + } + + if dirInode.payload == nil { + err = dirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + createOut = nil + errno = syscall.ENOENT + return + } + } + + directoryEntryValueV1AsValue, ok, err = dirInode.payload.GetByKey(string(createIn.Name[:])) + if nil != err { + logFatalf("dirInode.payload.GetByKey(string(createIn.Name[:])) failed: %v", err) + } + + if ok { + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + switch directoryEntryValueV1.InodeType { + case ilayout.InodeTypeDir: + inodeLockRequest.unlockAll() + createOut = nil + errno = syscall.EISDIR + return + case ilayout.InodeTypeFile: + // Fall through + case ilayout.InodeTypeSymLink: + inodeLockRequest.unlockAll() + createOut = nil + errno = syscall.EACCES + return + default: + logFatalf("directoryEntryValueV1.InodeType (%v) unknown", directoryEntryValueV1.InodeType) + } + + inodeLockRequest.inodeNumber = directoryEntryValueV1.InodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + inodeLockRequest.unlockAll() + goto Retry + } + + fileInode = lookupInode(directoryEntryValueV1.InodeNumber) + if nil == fileInode { + inodeLockRequest.unlockAll() + goto Retry + } + + if fileInode.inodeHeadV1.InodeType != ilayout.InodeTypeFile { + logFatalf("fileInode.inodeHeadV1.InodeType != ilayout.InodeTypeFile") + } + + // TODO: Need to truncate file (i.e. assume createIn.Flags contains fission.FOpenRequestTRUNC) + } else { // dirInode.payload.GetByKey(string(createIn.Name[:])) returned !ok + + fileInodeNumber = fetchNonce() + + fileInode = &inodeStruct{ + inodeNumber: fileInodeNumber, + dirty: true, + markedForDelete: false, + leaseState: inodeLeaseStateNone, + listElement: nil, + heldList: list.New(), + requestList: list.New(), + inodeHeadV1: &ilayout.InodeHeadV1Struct{ + InodeNumber: fileInodeNumber, + InodeType: ilayout.InodeTypeFile, + LinkTable: []ilayout.InodeLinkTableEntryStruct{ + ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: dirInode.inodeNumber, + ParentDirEntryName: string(createIn.Name[:]), + }, + }, + Size: 0, + ModificationTime: startTime, + StatusChangeTime: startTime, + Mode: uint16(createIn.Mode & ^createIn.UMask) & ilayout.InodeModeMask, + UserID: uint64(inHeader.UID), + GroupID: uint64(inHeader.GID), + StreamTable: make([]ilayout.InodeStreamTableEntryStruct, 0), + PayloadObjectNumber: 0, + PayloadObjectOffset: 0, + PayloadObjectLength: 0, + SymLinkTarget: "", + Layout: nil, + }, + payload: nil, + layoutMap: nil, + superBlockInodeObjectCountAdjustment: 0, + superBlockInodeObjectSizeAdjustment: 0, + superBlockInodeBytesReferencedAdjustment: 0, + dereferencedObjectNumberArray: make([]uint64, 0), + putObjectNumber: 0, + putObjectBuffer: nil, + } + + inodeLockRequest.inodeNumber = fileInodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + dirInode.dirty = true + + dirInode.inodeHeadV1.ModificationTime = startTime + dirInode.inodeHeadV1.StatusChangeTime = startTime + + ok, err = dirInode.payload.Put( + string(createIn.Name[:]), + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: fileInodeNumber, + InodeType: ilayout.InodeTypeFile, + }) + if nil != err { + logFatalf("dirInode.payload.Put(string(CreateIn.Name[:]),) failed: %v", err) + } + if !ok { + logFatalf("dirInode.payload.Put(string(CreateIn.Name[:]),) returned !ok") + } + + flushInodesInSlice([]*inodeStruct{dirInode, fileInode}) + } + + adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ + MountID: globals.mountID, + InodeNumber: fileInode.inodeNumber, + Adjustment: 1, + } + adjustInodeTableEntryOpenCountResponse = &imgrpkg.AdjustInodeTableEntryOpenCountResponseStruct{} + + err = rpcAdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + if nil != err { + logFatal(err) + } + + openHandle = createOpenHandle(fileInode.inodeNumber) + + openHandle.fissionFlagsAppend = false + openHandle.fissionFlagsRead = false + openHandle.fissionFlagsWrite = true + + createOut = &fission.CreateOut{ + EntryOut: fission.EntryOut{ + NodeID: fileInode.inodeHeadV1.InodeNumber, + Generation: 0, + EntryValidSec: globals.fuseEntryValidDurationSec, + AttrValidSec: globals.fuseAttrValidDurationSec, + EntryValidNSec: globals.fuseEntryValidDurationNSec, + AttrValidNSec: globals.fuseAttrValidDurationNSec, + // Attr to be filled in below + }, + FH: openHandle.fissionFH, + OpenFlags: 0, + Padding: 0, + } + + fileInode.doAttrFetch(&createOut.EntryOut.Attr) + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -2484,13 +2695,9 @@ func gorAttrFetch(inodeNumber uint64, fissionAttr *fission.Attr, gorAttrFetchWG func doAttrFetch(inodeNumber uint64, fissionAttr *fission.Attr) (err error) { var ( - inode *inodeStruct - inodeLockRequest *inodeLockRequestStruct - modificationTimeNSec uint32 - modificationTimeSec uint64 - obtainExclusiveLock bool - statusChangeTimeNSec uint32 - statusChangeTimeSec uint64 + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool ) obtainExclusiveLock = false @@ -2526,6 +2733,22 @@ Retry: } } + inode.doAttrFetch(fissionAttr) + + inodeLockRequest.unlockAll() + + err = nil + return +} + +func (inode *inodeStruct) doAttrFetch(fissionAttr *fission.Attr) { + var ( + modificationTimeNSec uint32 + modificationTimeSec uint64 + statusChangeTimeNSec uint32 + statusChangeTimeSec uint64 + ) + modificationTimeSec, modificationTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.ModificationTime.UnixNano())) statusChangeTimeSec, statusChangeTimeNSec = nsToUnixTime(uint64(inode.inodeHeadV1.StatusChangeTime.UnixNano())) @@ -2546,12 +2769,7 @@ Retry: fissionAttr.BlkSize = attrBlockSize // Possibly overwritten by fixAttrSizes() fissionAttr.Padding = 0 - inodeLockRequest.unlockAll() - fixAttrSizes(fissionAttr) - - err = nil - return } func fixAttrSizes(attr *fission.Attr) { From 32b742e572f4ed58c677b6bf0c1ee00a7c06b7ee Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 28 Oct 2021 17:06:35 -0700 Subject: [PATCH 144/258] Temporarily echo GolangURL in Dockerfile building of dev --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 53b08fa3..13c5abf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,6 +92,7 @@ ENV XDG_RUNTIME_DIR /tmp/runtime-root ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" ENV GolangURL "https://golang.org/dl/${GolangBasename}" WORKDIR /tmp +RUN echo $GolangURL RUN wget -nv $GolangURL RUN tar -C /usr/local -xzf $GolangBasename ENV PATH $PATH:/usr/local/go/bin From 011f6861f2eb7d4835b3f8c8d1bc6aa59a91426f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 28 Oct 2021 17:33:12 -0700 Subject: [PATCH 145/258] Let's echo GolangVersion in base as well... --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 13c5abf7..96459fb1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,6 +69,7 @@ FROM alpine:3.14.0 as base ARG GolangVersion=1.17.2 +RUN echo $GolangVersion ARG MakeTarget RUN apk add --no-cache libc6-compat From 1324c7c867118e687d92743ce96d3398e5f28fdd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 28 Oct 2021 17:43:28 -0700 Subject: [PATCH 146/258] For the time being, let's hard-code "1.17.2" into the GolangBasename This works around the apparent bug in our "runner" that fails to properly handle the ARG line --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 96459fb1..5123fc3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,6 +91,7 @@ RUN curl -sSL https://github.com/coreos/etcd/releases/download/v3.5.0/etcd-v3.5. ENV LIBGL_ALWAYS_INDIRECT 1 ENV XDG_RUNTIME_DIR /tmp/runtime-root ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" +ENV GolangBasename "go1.17.2.linux-amd64.tar.gz" ENV GolangURL "https://golang.org/dl/${GolangBasename}" WORKDIR /tmp RUN echo $GolangURL From 0f2e745c6d2ba060a4e59a90a73791f6fbc04f79 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 28 Oct 2021 17:49:20 -0700 Subject: [PATCH 147/258] Revert the last three "experiment" hack changes to Dockerfile --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5123fc3e..53b08fa3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,7 +69,6 @@ FROM alpine:3.14.0 as base ARG GolangVersion=1.17.2 -RUN echo $GolangVersion ARG MakeTarget RUN apk add --no-cache libc6-compat @@ -91,10 +90,8 @@ RUN curl -sSL https://github.com/coreos/etcd/releases/download/v3.5.0/etcd-v3.5. ENV LIBGL_ALWAYS_INDIRECT 1 ENV XDG_RUNTIME_DIR /tmp/runtime-root ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" -ENV GolangBasename "go1.17.2.linux-amd64.tar.gz" ENV GolangURL "https://golang.org/dl/${GolangBasename}" WORKDIR /tmp -RUN echo $GolangURL RUN wget -nv $GolangURL RUN tar -C /usr/local -xzf $GolangBasename ENV PATH $PATH:/usr/local/go/bin From 5e85db4f67308badace20997428b4dc364e4b139 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 13:50:46 -0700 Subject: [PATCH 148/258] Ensure GitLab runners properly parse the expansion of GolangURL --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 53b08fa3..12b7459a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,7 +92,7 @@ ENV XDG_RUNTIME_DIR /tmp/runtime-root ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" ENV GolangURL "https://golang.org/dl/${GolangBasename}" WORKDIR /tmp -RUN wget -nv $GolangURL +RUN wget -nv ${GolangURL} RUN tar -C /usr/local -xzf $GolangBasename ENV PATH $PATH:/usr/local/go/bin VOLUME /src From 997d00c90bc38faa8d26106b5bb25cef0f903563 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 14:19:07 -0700 Subject: [PATCH 149/258] Apparently staged images don't inherit ENVs... so move the ARGs to the stages that need them --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 12b7459a..f1449034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,11 +68,10 @@ # --env DISPLAY: tells Docker to set ENV DISPLAY for X apps (e.g. wireshark) FROM alpine:3.14.0 as base -ARG GolangVersion=1.17.2 -ARG MakeTarget RUN apk add --no-cache libc6-compat FROM base as dev +ARG GolangVersion=1.17.2 RUN apk add --no-cache bind-tools \ curl \ fuse \ @@ -99,6 +98,7 @@ VOLUME /src WORKDIR /src FROM dev as build +ARG MakeTarget VOLUME /src COPY . /src WORKDIR /src From 0475ed8f9ac73e03b1ec82b10fed2f18ac48e07b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 14:35:49 -0700 Subject: [PATCH 150/258] Switch to canonical format for an ENV line in Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1449034..356349c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -86,10 +86,10 @@ RUN apk add --no-cache bind-tools \ RUN curl -sSL https://github.com/coreos/etcd/releases/download/v3.5.0/etcd-v3.5.0-linux-amd64.tar.gz \ | tar -vxz -C /usr/local/bin --strip=1 etcd-v3.5.0-linux-amd64/etcd etcd-v3.5.0-linux-amd64/etcdctl \ && chown root:root /usr/local/bin/etcd /usr/local/bin/etcdctl -ENV LIBGL_ALWAYS_INDIRECT 1 -ENV XDG_RUNTIME_DIR /tmp/runtime-root -ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" -ENV GolangURL "https://golang.org/dl/${GolangBasename}" +ENV LIBGL_ALWAYS_INDIRECT=1 +ENV XDG_RUNTIME_DIR="/tmp/runtime-root" +ENV GolangBasename="go${GolangVersion}.linux-amd64.tar.gz" +ENV GolangURL="https://golang.org/dl/${GolangBasename}" WORKDIR /tmp RUN wget -nv ${GolangURL} RUN tar -C /usr/local -xzf $GolangBasename From 6e28c3f5577bd9974f81b61775a59e3677d94ab3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 15:00:44 -0700 Subject: [PATCH 151/258] In case COPY --from=/... does not honor that leading '/' --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 356349c6..7d6a3f95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,6 +104,7 @@ COPY . /src WORKDIR /src RUN make clean RUN make $MakeTarget +WORKDIR / FROM base as imgr COPY --from=build /src/icert/icert ./ From 6de778dc989c5b5e877f6bdd00e8ac69e61200c3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 15:06:45 -0700 Subject: [PATCH 152/258] Let's try to debug the COPY --from=build issue --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7d6a3f95..89c54dde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,6 +105,8 @@ WORKDIR /src RUN make clean RUN make $MakeTarget WORKDIR / +RUN ls -l /src/icert/icert +RUN ls -l src/icert/icert FROM base as imgr COPY --from=build /src/icert/icert ./ From c09f3e18c3c80a1b724f675b11ab23765118fec3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 15:20:51 -0700 Subject: [PATCH 153/258] Trying to discover if icert should be there... --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 89c54dde..947065f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,11 +102,10 @@ ARG MakeTarget VOLUME /src COPY . /src WORKDIR /src +RUN ls -l icert RUN make clean RUN make $MakeTarget -WORKDIR / -RUN ls -l /src/icert/icert -RUN ls -l src/icert/icert +RUN ls -l icert FROM base as imgr COPY --from=build /src/icert/icert ./ From 90f85d72dad73d1471ef6ff806a3b4e4d080f864 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 15:30:11 -0700 Subject: [PATCH 154/258] Where did building of icert go? --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 947065f8..460a625d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,6 +106,8 @@ RUN ls -l icert RUN make clean RUN make $MakeTarget RUN ls -l icert +RUN find / -name icert +RUN echo ${GOPATH} FROM base as imgr COPY --from=build /src/icert/icert ./ From 7748d9e6a9f4b4cbd2127a95b25afe4aa2686529 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 15:36:39 -0700 Subject: [PATCH 155/258] Let's try dumping where everything went... --- GoMakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GoMakefile b/GoMakefile index 47341683..04b486d7 100644 --- a/GoMakefile +++ b/GoMakefile @@ -11,7 +11,7 @@ bench: go test -bench $(gosubdir) build: - go build -gcflags "-N -l" $(gosubdir) + go build -work -a -p 1 -x -gcflags "-N -l" $(gosubdir) clean: @set -e; \ From 804af77065aa0dde98ed4d497af2adecf0744b30 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 15:53:18 -0700 Subject: [PATCH 156/258] Trying to find icert... where did it go? --- Dockerfile | 3 +++ GoMakefile | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 460a625d..c60a5082 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,6 +108,9 @@ RUN make $MakeTarget RUN ls -l icert RUN find / -name icert RUN echo ${GOPATH} +RUN echo ${GOBIN} +RUN echo ${GOROOT} +RUN echo ${PWD} FROM base as imgr COPY --from=build /src/icert/icert ./ diff --git a/GoMakefile b/GoMakefile index 04b486d7..47341683 100644 --- a/GoMakefile +++ b/GoMakefile @@ -11,7 +11,7 @@ bench: go test -bench $(gosubdir) build: - go build -work -a -p 1 -x -gcflags "-N -l" $(gosubdir) + go build -gcflags "-N -l" $(gosubdir) clean: @set -e; \ From 5ec8b71918009cfe93d3272f35bb568ca8fe0024 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 15:57:17 -0700 Subject: [PATCH 157/258] Yet another try... --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index c60a5082..38b90d87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -111,6 +111,8 @@ RUN echo ${GOPATH} RUN echo ${GOBIN} RUN echo ${GOROOT} RUN echo ${PWD} +RUN (cd icert ; ls -l ; make build ; ls -l) +RUN (cd icert ; ls -l ; go build ; ls -l) FROM base as imgr COPY --from=build /src/icert/icert ./ From 6bb83b40f6d2cc892ba56f186b0709f2f53e7a56 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 16:10:41 -0700 Subject: [PATCH 158/258] Clean-out all the debugging stuff in Dockerfile... --- Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 38b90d87..356349c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,17 +102,8 @@ ARG MakeTarget VOLUME /src COPY . /src WORKDIR /src -RUN ls -l icert RUN make clean RUN make $MakeTarget -RUN ls -l icert -RUN find / -name icert -RUN echo ${GOPATH} -RUN echo ${GOBIN} -RUN echo ${GOROOT} -RUN echo ${PWD} -RUN (cd icert ; ls -l ; make build ; ls -l) -RUN (cd icert ; ls -l ; go build ; ls -l) FROM base as imgr COPY --from=build /src/icert/icert ./ From 27d5d63f65749b17671cf97be594faa5d9809532 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 16:51:09 -0700 Subject: [PATCH 159/258] The "build" image shouldn't have a VOLUME /src statement... it makes a copy --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 356349c6..f9d73605 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,6 @@ WORKDIR /src FROM dev as build ARG MakeTarget -VOLUME /src COPY . /src WORKDIR /src RUN make clean From cc4a9e4807afad53e63e6dd749a174a44d99e9c8 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Oct 2021 17:06:51 -0700 Subject: [PATCH 160/258] Avoid inherited VOLUME /src from "dev" into "build" by using a different path... --- Dockerfile | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index f9d73605..219be4cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ # 1) builds an image capable of building all elements # 2) /src is shared with context dir of host # --target build: -# 1) clones the context dir at /src +# 1) clones the context dir at /clone # 2) performs a make clean to clear out non-source-controlled artifacts # 3) performs a make $MakeTarget # --target imgr: @@ -63,8 +63,6 @@ # 1) bind mounts the context into /src in the container # 2) /src will be a read-write'able equivalent to the context dir # 3) only useful for --target dev -# 4) /src will be a local copy instead for --target build -# 5) /src doesn't exist for --target build # --env DISPLAY: tells Docker to set ENV DISPLAY for X apps (e.g. wireshark) FROM alpine:3.14.0 as base @@ -99,23 +97,23 @@ WORKDIR /src FROM dev as build ARG MakeTarget -COPY . /src -WORKDIR /src +COPY . /clone +WORKDIR /clone RUN make clean RUN make $MakeTarget FROM base as imgr -COPY --from=build /src/icert/icert ./ +COPY --from=build /clone/icert/icert ./ RUN ./icert -ca -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 RUN ./icert -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 -cert cert.pem -key key.pem -dns imgr -COPY --from=build /src/imgr/imgr ./ -COPY --from=build /src/imgr/imgr.conf ./ +COPY --from=build /clone/imgr/imgr ./ +COPY --from=build /clone/imgr/imgr.conf ./ FROM imgr as iclient RUN rm icert caKey.pem cert.pem key.pem imgr imgr.conf RUN apk add --no-cache fuse -COPY --from=build /src/iclient/iclient ./ -COPY --from=build /src/iclient/iclient.conf ./ -COPY --from=build /src/iclient/iclient.sh ./ -COPY --from=build /src/iauth/iauth-swift/iauth-swift.so ./ +COPY --from=build /clone/iclient/iclient ./ +COPY --from=build /clone/iclient/iclient.conf ./ +COPY --from=build /clone/iclient/iclient.sh ./ +COPY --from=build /clone/iauth/iauth-swift/iauth-swift.so ./ RUN apk add --no-cache curl From b878533e236ad5bdb138cc3e7c92bf0f23c0aa26 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 1 Nov 2021 10:37:59 -0700 Subject: [PATCH 161/258] Fix GET of FileInode to imgr's HTTP Server --- iclient/iclientpkg/fission.go | 17 +++++++++-------- imgr/imgrpkg/http-server.go | 6 +++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 33dfe862..32e59a5e 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -2073,13 +2073,9 @@ func (dummy *globalsStruct) DoCreate(inHeader *fission.InHeader, createIn *fissi fileInode *inodeStruct fileInodeNumber uint64 inodeLockRequest *inodeLockRequestStruct - // modificationTimeNSec uint32 - // modificationTimeSec uint64 - ok bool - openHandle *openHandleStruct - startTime time.Time = time.Now() - // statusChangeTimeNSec uint32 - // statusChangeTimeSec uint64 + ok bool + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoCreate(inHeader: %+v, createIn: %+v)", inHeader, createIn) @@ -2207,7 +2203,7 @@ Retry: PayloadObjectOffset: 0, PayloadObjectLength: 0, SymLinkTarget: "", - Layout: nil, + Layout: make([]ilayout.InodeHeadLayoutEntryV1Struct, 0), }, payload: nil, layoutMap: nil, @@ -2226,6 +2222,11 @@ Retry: goto Retry } + err = fileInode.newPayload() + if nil != err { + logFatalf("fileInode.newPayload() failed: %v", err) + } + dirInode.dirty = true dirInode.inodeHeadV1.ModificationTime = startTime diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index eb3cfe0d..fb0cd673 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -1199,9 +1199,9 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ } for inodeGETLayoutEntryIndex = range inodeHeadV1.Layout { - inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].ObjectName = ilayout.GetObjectNameAsString(inodeHeadV1.Layout[inodeGETLayoutEntryIndex].ObjectNumber) - inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].ObjectSize = inodeHeadV1.Layout[inodeGETLayoutEntryIndex].ObjectSize - inodeGET.(*dirInodeGETStruct).Layout[inodeGETLayoutEntryIndex].BytesReferenced = inodeHeadV1.Layout[inodeGETLayoutEntryIndex].BytesReferenced + inodeGET.(*fileInodeGETStruct).Layout[inodeGETLayoutEntryIndex].ObjectName = ilayout.GetObjectNameAsString(inodeHeadV1.Layout[inodeGETLayoutEntryIndex].ObjectNumber) + inodeGET.(*fileInodeGETStruct).Layout[inodeGETLayoutEntryIndex].ObjectSize = inodeHeadV1.Layout[inodeGETLayoutEntryIndex].ObjectSize + inodeGET.(*fileInodeGETStruct).Layout[inodeGETLayoutEntryIndex].BytesReferenced = inodeHeadV1.Layout[inodeGETLayoutEntryIndex].BytesReferenced } case ilayout.InodeTypeSymLink: inodeGET = &symLinkInodeGETStruct{ From 74d4a05046aaf9dd6c0e30175bb2f6b589697da8 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 1 Nov 2021 12:03:23 -0700 Subject: [PATCH 162/258] Implemented most of iclient's DoRename{|2}() ...common code between the two is iclient/fission.go::doRenameCommon() --- iclient/iclientpkg/fission.go | 222 +++++++++++++++++++++++++++++++++- 1 file changed, 218 insertions(+), 4 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 32e59a5e..a9b3ed62 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -869,8 +869,7 @@ func (dummy *globalsStruct) DoRename(inHeader *fission.InHeader, renameIn *fissi globals.stats.DoRenameUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS + errno = doRenameCommon(inHeader.NodeID, string(renameIn.OldName[:]), renameIn.NewDir, string(renameIn.NewName[:]), startTime) return } @@ -2615,8 +2614,7 @@ func (dummy *globalsStruct) DoRename2(inHeader *fission.InHeader, rename2In *fis globals.stats.DoRename2Usecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS + errno = doRenameCommon(inHeader.NodeID, string(rename2In.OldName[:]), rename2In.NewDir, string(rename2In.NewName[:]), startTime) return } @@ -2784,3 +2782,219 @@ func fixAttrSizes(attr *fission.Attr) { attr.BlkSize = 0 } } + +func doRenameCommon(oldDirInodeNumber uint64, oldName string, newDirInodeNumber uint64, newName string, startTime time.Time) (errno syscall.Errno) { + var ( + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + err error + inodeLockRequest *inodeLockRequestStruct + inodeSliceToFlush []*inodeStruct = make([]*inodeStruct, 0) + newDirInode *inodeStruct + ok bool + oldDirInode *inodeStruct + renamedInode *inodeStruct + replacedInode *inodeStruct + ) + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = oldDirInodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + oldDirInode = lookupInode(oldDirInodeNumber) + if nil == oldDirInode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + if oldDirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.ENOTDIR + return + } + + inodeSliceToFlush = append(inodeSliceToFlush, oldDirInode) + + if nil == oldDirInode.inodeHeadV1 { + err = oldDirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if oldDirInode.payload == nil { + err = oldDirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + directoryEntryValueV1AsValue, ok, err = oldDirInode.payload.GetByKey(oldName) + if nil != err { + logFatalf("dirInode.payload.GetByKey(oldName) failed: %v", err) + } + if !ok { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + inodeLockRequest.inodeNumber = directoryEntryValueV1.InodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + inodeLockRequest.unlockAll() + goto Retry + } + + renamedInode = lookupInode(directoryEntryValueV1.InodeNumber) + if nil == renamedInode { + inodeLockRequest.unlockAll() + goto Retry + } + + inodeSliceToFlush = append(inodeSliceToFlush, renamedInode) + + if nil == renamedInode.inodeHeadV1 { + err = renamedInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if oldDirInodeNumber == newDirInodeNumber { + newDirInode = oldDirInode + + if oldName == newName { + inodeLockRequest.unlockAll() + errno = 0 + return + } + } else { + inodeLockRequest.inodeNumber = newDirInodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + newDirInode = lookupInode(newDirInodeNumber) + if nil == newDirInode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + if newDirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.ENOTDIR + return + } + + inodeSliceToFlush = append(inodeSliceToFlush, newDirInode) + + if nil == newDirInode.inodeHeadV1 { + err = newDirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if newDirInode.payload == nil { + err = newDirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + } + + directoryEntryValueV1AsValue, ok, err = newDirInode.payload.GetByKey(newName) + if nil != err { + logFatalf("dirInode.payload.GetByKey(newName) failed: %v", err) + } + + if ok { + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + inodeLockRequest.inodeNumber = directoryEntryValueV1.InodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + inodeLockRequest.unlockAll() + goto Retry + } + + replacedInode = lookupInode(directoryEntryValueV1.InodeNumber) + if nil == renamedInode { + inodeLockRequest.unlockAll() + goto Retry + } + + inodeSliceToFlush = append(inodeSliceToFlush, replacedInode) + + if nil == replacedInode.inodeHeadV1 { + err = replacedInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + } else { + replacedInode = nil + } + + if renamedInode.inodeHeadV1.InodeType == ilayout.InodeTypeDir { + if replacedInode != nil { + inodeLockRequest.unlockAll() + errno = syscall.EISDIR + return + } + + // TODO - remove renamedInode from oldDirInode's payload + // TODO - remove renamedInode's ".." link from oldDirInode + // TODO - replace renamedInode's ".." to point to newDirInode + // TODO - add renamedInodes's ".." link in newDirInode + // TODO - insert renamedInode in newDirInode's payload + } else { + if replacedInode != nil { + // TODO - first remove replacedInode + } + + // TODO - next remove renamedInode from oldDirInode's payload + // TODO - replace link entry in renamedInode to be from newDirInode + // TODO - finally insert renamedInode in newDirInode's payload + } + + fmt.Printf("\nUNDO: renamedInode: %+v\n", renamedInode) + fmt.Printf("UNDO: replacedInode: %+v\n", replacedInode) + fmt.Printf("UNDO: inodeSliceToFlush: %+v\n\n", inodeSliceToFlush) + + inodeLockRequest.unlockAll() + + // TODO + errno = syscall.ENOSYS + return +} From bd45aeec4591b669fa81a519dd22a40408baaad7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 1 Nov 2021 17:06:11 -0700 Subject: [PATCH 163/258] Promoted LinkTable & StreamTable to maps upon loading so that they are easy to search --- iclient/iclientpkg/fission.go | 144 ++++++++++++++++++++++++---------- iclient/iclientpkg/globals.go | 44 ++++++----- iclient/iclientpkg/inode.go | 136 +++++++++++++++++++++----------- iclient/iclientpkg/lease.go | 4 +- 4 files changed, 218 insertions(+), 110 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index a9b3ed62..33ea1aa2 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -474,29 +474,26 @@ Retry: heldList: list.New(), requestList: list.New(), inodeHeadV1: &ilayout.InodeHeadV1Struct{ - InodeNumber: symLinkInodeNumber, - InodeType: ilayout.InodeTypeSymLink, - LinkTable: []ilayout.InodeLinkTableEntryStruct{ - ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: dirInode.inodeNumber, - ParentDirEntryName: string(symLinkIn.Name[:]), - }, - }, + InodeNumber: symLinkInodeNumber, + InodeType: ilayout.InodeTypeSymLink, + LinkTable: nil, Size: 0, ModificationTime: startTime, StatusChangeTime: startTime, Mode: ilayout.InodeModeMask, UserID: uint64(inHeader.UID), GroupID: uint64(inHeader.GID), - StreamTable: make([]ilayout.InodeStreamTableEntryStruct, 0), + StreamTable: nil, PayloadObjectNumber: 0, PayloadObjectOffset: 0, PayloadObjectLength: 0, SymLinkTarget: string(symLinkIn.Data[:]), Layout: nil, }, + linkSet: make(map[ilayout.InodeLinkTableEntryStruct]struct{}), + streamMap: make(map[string][]byte), + layoutMap: make(map[uint64]layoutMapEntryStruct), payload: nil, - layoutMap: nil, superBlockInodeObjectCountAdjustment: 0, superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, @@ -505,6 +502,11 @@ Retry: putObjectBuffer: nil, } + symLinkInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: dirInode.inodeNumber, + ParentDirEntryName: string(symLinkIn.Name[:]), + }] = struct{}{} + inodeLockRequest.inodeNumber = symLinkInodeNumber inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() @@ -674,33 +676,26 @@ Retry: heldList: list.New(), requestList: list.New(), inodeHeadV1: &ilayout.InodeHeadV1Struct{ - InodeNumber: childDirInodeNumber, - InodeType: ilayout.InodeTypeDir, - LinkTable: []ilayout.InodeLinkTableEntryStruct{ - ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: parentDirInode.inodeNumber, - ParentDirEntryName: string(mkDirIn.Name[:]), - }, - ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: childDirInodeNumber, - ParentDirEntryName: ".", - }, - }, + InodeNumber: childDirInodeNumber, + InodeType: ilayout.InodeTypeDir, + LinkTable: nil, Size: 0, ModificationTime: startTime, StatusChangeTime: startTime, Mode: uint16(mkDirIn.Mode & ^mkDirIn.UMask) & ilayout.InodeModeMask, UserID: uint64(inHeader.UID), GroupID: uint64(inHeader.GID), - StreamTable: make([]ilayout.InodeStreamTableEntryStruct, 0), + StreamTable: nil, PayloadObjectNumber: 0, PayloadObjectOffset: 0, PayloadObjectLength: 0, SymLinkTarget: "", Layout: nil, }, + linkSet: make(map[ilayout.InodeLinkTableEntryStruct]struct{}), + streamMap: make(map[string][]byte), + layoutMap: make(map[uint64]layoutMapEntryStruct), payload: nil, - layoutMap: nil, superBlockInodeObjectCountAdjustment: 0, superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, @@ -709,6 +704,15 @@ Retry: putObjectBuffer: nil, } + childDirInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: parentDirInode.inodeNumber, + ParentDirEntryName: string(mkDirIn.Name[:]), + }] = struct{}{} + childDirInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: childDirInodeNumber, + ParentDirEntryName: ".", + }] = struct{}{} + err = childDirInode.newPayload() if nil != err { logFatalf("childDirInode.newPayload() failed: %v\n", err) @@ -751,13 +755,10 @@ Retry: parentDirInode.dirty = true - parentDirInode.inodeHeadV1.LinkTable = append( - parentDirInode.inodeHeadV1.LinkTable, - ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: childDirInode.inodeNumber, - ParentDirEntryName: "..", - }, - ) + parentDirInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: childDirInode.inodeNumber, + ParentDirEntryName: "..", + }] = struct{}{} parentDirInode.inodeHeadV1.ModificationTime = startTime parentDirInode.inodeHeadV1.StatusChangeTime = startTime @@ -875,7 +876,11 @@ func (dummy *globalsStruct) DoRename(inHeader *fission.InHeader, renameIn *fissi func (dummy *globalsStruct) DoLink(inHeader *fission.InHeader, linkIn *fission.LinkIn) (linkOut *fission.LinkOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + dirInode *inodeStruct + err error + inodeLockRequest *inodeLockRequestStruct + startTime time.Time = time.Now() + // targetInode *inodeStruct ) logTracef("==> DoLink(inHeader: %+v, linkIn: %+v)", inHeader, linkIn) @@ -887,6 +892,50 @@ func (dummy *globalsStruct) DoLink(inHeader *fission.InHeader, linkIn *fission.L globals.stats.DoLinkUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + dirInode = lookupInode(inHeader.NodeID) + if nil == dirInode { + inodeLockRequest.unlockAll() + linkOut = nil + errno = syscall.ENOENT + return + } + + if nil == dirInode.inodeHeadV1 { + err = dirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + linkOut = nil + errno = syscall.ENOENT + return + } + } + + if dirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + linkOut = nil + errno = syscall.ENOTDIR + return + } + + if dirInode.payload == nil { + err = dirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + linkOut = nil + errno = syscall.ENOENT + return + } + } + // TODO linkOut = nil errno = syscall.ENOSYS @@ -2113,6 +2162,13 @@ Retry: } } + if dirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + createOut = nil + errno = syscall.ENOTDIR + return + } + if dirInode.payload == nil { err = dirInode.oldPayload() if nil != err { @@ -2183,29 +2239,26 @@ Retry: heldList: list.New(), requestList: list.New(), inodeHeadV1: &ilayout.InodeHeadV1Struct{ - InodeNumber: fileInodeNumber, - InodeType: ilayout.InodeTypeFile, - LinkTable: []ilayout.InodeLinkTableEntryStruct{ - ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: dirInode.inodeNumber, - ParentDirEntryName: string(createIn.Name[:]), - }, - }, + InodeNumber: fileInodeNumber, + InodeType: ilayout.InodeTypeFile, + LinkTable: nil, Size: 0, ModificationTime: startTime, StatusChangeTime: startTime, Mode: uint16(createIn.Mode & ^createIn.UMask) & ilayout.InodeModeMask, UserID: uint64(inHeader.UID), GroupID: uint64(inHeader.GID), - StreamTable: make([]ilayout.InodeStreamTableEntryStruct, 0), + StreamTable: nil, PayloadObjectNumber: 0, PayloadObjectOffset: 0, PayloadObjectLength: 0, SymLinkTarget: "", - Layout: make([]ilayout.InodeHeadLayoutEntryV1Struct, 0), + Layout: nil, }, + linkSet: make(map[ilayout.InodeLinkTableEntryStruct]struct{}), + streamMap: make(map[string][]byte), + layoutMap: make(map[uint64]layoutMapEntryStruct), payload: nil, - layoutMap: nil, superBlockInodeObjectCountAdjustment: 0, superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, @@ -2214,6 +2267,11 @@ Retry: putObjectBuffer: nil, } + fileInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: dirInode.inodeNumber, + ParentDirEntryName: string(createIn.Name[:]), + }] = struct{}{} + inodeLockRequest.inodeNumber = fileInodeNumber inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 1c95e493..50732569 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -99,27 +99,29 @@ type openHandleStruct struct { } type inodeStruct struct { - sync.WaitGroup // Signaled to indicate .markedForDelete == true triggered removal has completed - inodeNumber uint64 // - dirty bool // - markedForDelete bool // If true, remove from globalsStruct.inodeTable upon last dereference - leaseState inodeLeaseStateType // - listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU - heldList *list.List // List of granted inodeHeldLockStruct's - requestList *list.List // List of pending inodeLockRequestStruct's - inodeHeadV1 *ilayout.InodeHeadV1Struct // - payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} - // For FileInode: ExtentMap B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} - layoutMap map[uint64]layoutMapEntryStruct // For DirInode & FileInode: Map form of .inodeHeadV1.Layout - superBlockInodeObjectCountAdjustment int64 // - superBlockInodeObjectSizeAdjustment int64 // - superBlockInodeBytesReferencedAdjustment int64 // - dereferencedObjectNumberArray []uint64 // - putObjectNumber uint64 // ObjectNumber to PUT during flush - putObjectBuffer []byte // PUT content to send to .putObjectNumber'd Object: - // For DirInode: marshaled .payload & ilayout.InodeHeadV1Struct - // For FileInode: file extents, marshaled .payload, & ilayout.InodeHeadV1Struct - // For SymLinkInode: marshaled ilayout.InodeHeadV1Struct + sync.WaitGroup // Signaled to indicate .markedForDelete == true triggered removal has completed + inodeNumber uint64 // + dirty bool // + markedForDelete bool // If true, remove from globalsStruct.inodeTable upon last dereference + leaseState inodeLeaseStateType // + listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU + heldList *list.List // List of granted inodeHeldLockStruct's + requestList *list.List // List of pending inodeLockRequestStruct's + inodeHeadV1 *ilayout.InodeHeadV1Struct // + linkSet map[ilayout.InodeLinkTableEntryStruct]struct{} // Set form of .inodeHeadV1.LinkTable; key == ilayout.InodeLinkTableEntryStruct + streamMap map[string][]byte // Map form of .inodeHeadV1.StreamTable; key == ilayout.InodeStreamTableEntryStruct.Name, value == ilayout.InodeStreamTableEntryStruct.Value + layoutMap map[uint64]layoutMapEntryStruct // For DirInode & FileInode: Map form of .inodeHeadV1.Layout; key == ilayout.InodeHeadLayoutEntryV1Struct.ObjectNumber + payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} + // For FileInode: ExtentMap B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} + superBlockInodeObjectCountAdjustment int64 // + superBlockInodeObjectSizeAdjustment int64 // + superBlockInodeBytesReferencedAdjustment int64 // + dereferencedObjectNumberArray []uint64 // + putObjectNumber uint64 // ObjectNumber to PUT during flush + putObjectBuffer []byte // PUT content to send to .putObjectNumber'd Object: + // For DirInode: marshaled .payload & ilayout.InodeHeadV1Struct + // For FileInode: file extents, marshaled .payload, & ilayout.InodeHeadV1Struct + // For SymLinkInode: marshaled ilayout.InodeHeadV1Struct } type inodeHeldLockStruct struct { diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 14b82559..10c7a55b 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -165,8 +165,6 @@ func (inode *inodeStruct) DiscardNode(objectNumber uint64, objectOffset uint64, logFatalf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - inode.ensureLayoutMapIsActive() - layoutMapEntry, ok = inode.layoutMap[objectNumber] if !ok { log.Fatalf("inode.layoutMap[old inode.putObjectNumber] returned !ok") @@ -500,9 +498,96 @@ func (inode *inodeStruct) populateInodeHeadV1() (err error) { logFatalf("ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) failed: %v", err) } + inode.convertInodeHeadV1LinkTableToLinkSet() + inode.convertInodeHeadV1StreamTableToStreamMap() + inode.convertInodeHeadV1LayoutToLayoutMap() + return } +func (inode *inodeStruct) convertInodeHeadV1LinkTableToLinkSet() { + var ( + ilayoutInodeLinkTableEntry ilayout.InodeLinkTableEntryStruct + ) + + inode.linkSet = make(map[ilayout.InodeLinkTableEntryStruct]struct{}) + + for _, ilayoutInodeLinkTableEntry = range inode.inodeHeadV1.LinkTable { + inode.linkSet[ilayoutInodeLinkTableEntry] = struct{}{} + } +} + +func (inode *inodeStruct) convertLinkSetToInodeHeadV1LinkTable() { + var ( + ilayoutInodeLinkTableEntry ilayout.InodeLinkTableEntryStruct + ) + + inode.inodeHeadV1.LinkTable = make([]ilayout.InodeLinkTableEntryStruct, 0, len(inode.linkSet)) + + for ilayoutInodeLinkTableEntry = range inode.linkSet { + inode.inodeHeadV1.LinkTable = append(inode.inodeHeadV1.LinkTable, ilayoutInodeLinkTableEntry) + } +} + +func (inode *inodeStruct) convertInodeHeadV1StreamTableToStreamMap() { + var ( + ilayoutInodeStreamTableEntry ilayout.InodeStreamTableEntryStruct + ) + + inode.streamMap = make(map[string][]byte) + + for _, ilayoutInodeStreamTableEntry = range inode.inodeHeadV1.StreamTable { + inode.streamMap[ilayoutInodeStreamTableEntry.Name] = ilayoutInodeStreamTableEntry.Value + } +} + +func (inode *inodeStruct) convertStreamMapToInodeHeadV1StreamTable() { + var ( + name string + value []byte + ) + + inode.inodeHeadV1.StreamTable = make([]ilayout.InodeStreamTableEntryStruct, 0, len(inode.streamMap)) + + for name, value = range inode.streamMap { + inode.inodeHeadV1.StreamTable = append(inode.inodeHeadV1.StreamTable, ilayout.InodeStreamTableEntryStruct{ + Name: name, + Value: value, + }) + } +} + +func (inode *inodeStruct) convertInodeHeadV1LayoutToLayoutMap() { + var ( + ilayoutInodeHeadLayoutEntryV1 ilayout.InodeHeadLayoutEntryV1Struct + ) + + inode.layoutMap = make(map[uint64]layoutMapEntryStruct) + + for _, ilayoutInodeHeadLayoutEntryV1 = range inode.inodeHeadV1.Layout { + inode.layoutMap[ilayoutInodeHeadLayoutEntryV1.ObjectNumber] = layoutMapEntryStruct{ + objectSize: ilayoutInodeHeadLayoutEntryV1.ObjectSize, + bytesReferenced: ilayoutInodeHeadLayoutEntryV1.BytesReferenced, + } + } +} + +func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { + var ( + ilayoutInodeHeadV1LayoutIndex uint64 = 0 + layoutMapEntry layoutMapEntryStruct + objectNumber uint64 + ) + + inode.inodeHeadV1.Layout = make([]ilayout.InodeHeadLayoutEntryV1Struct, len(inode.layoutMap)) + + for objectNumber, layoutMapEntry = range inode.layoutMap { + inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectNumber = objectNumber + inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectSize = layoutMapEntry.objectSize + inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced + } +} + func (inode *inodeStruct) newPayload() (err error) { switch inode.inodeHeadV1.InodeType { case ilayout.InodeTypeDir: @@ -557,43 +642,6 @@ func (inode *inodeStruct) oldPayload() (err error) { return } -func (inode *inodeStruct) convertInodeHeadV1LayoutToLayoutMap() { - var ( - ilayoutInodeHeadLayoutEntryV1 ilayout.InodeHeadLayoutEntryV1Struct - ) - - inode.layoutMap = make(map[uint64]layoutMapEntryStruct) - - for _, ilayoutInodeHeadLayoutEntryV1 = range inode.inodeHeadV1.Layout { - inode.layoutMap[ilayoutInodeHeadLayoutEntryV1.ObjectNumber] = layoutMapEntryStruct{ - objectSize: ilayoutInodeHeadLayoutEntryV1.ObjectSize, - bytesReferenced: ilayoutInodeHeadLayoutEntryV1.BytesReferenced, - } - } -} - -func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { - var ( - ilayoutInodeHeadV1LayoutIndex uint64 = 0 - layoutMapEntry layoutMapEntryStruct - objectNumber uint64 - ) - - inode.inodeHeadV1.Layout = make([]ilayout.InodeHeadLayoutEntryV1Struct, len(inode.layoutMap)) - - for objectNumber, layoutMapEntry = range inode.layoutMap { - inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectNumber = objectNumber - inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectSize = layoutMapEntry.objectSize - inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced - } -} - -func (inode *inodeStruct) ensureLayoutMapIsActive() { - if inode.layoutMap == nil { - inode.convertInodeHeadV1LayoutToLayoutMap() - } -} - func (inode *inodeStruct) ensurePutObjectIsActive() { var ( layoutMapEntry layoutMapEntryStruct @@ -605,8 +653,6 @@ func (inode *inodeStruct) ensurePutObjectIsActive() { inode.putObjectBuffer = make([]byte, 0) if inode.inodeHeadV1.InodeType != ilayout.InodeTypeSymLink { - inode.ensureLayoutMapIsActive() - layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] if ok { log.Fatalf("inode.layoutMap[inode.putObjectNumber] returned ok") @@ -653,12 +699,12 @@ func (inode *inodeStruct) flush() (inodeHeadLength uint64) { logFatalf("inode.payload.Prune() failed: %v", err) } } - - if inode.layoutMap != nil { - inode.convertLayoutMapToInodeHeadV1Layout() - } } + inode.convertLinkSetToInodeHeadV1LinkTable() + inode.convertStreamMapToInodeHeadV1StreamTable() + inode.convertLayoutMapToInodeHeadV1Layout() + inodeHeadV1Buf, err = inode.inodeHeadV1.MarshalInodeHeadV1() if nil != err { logFatalf("inode.inodeHeadV1.MarshalInodeHeadV1() failed: %v", err) diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 34dba4a2..bc807a24 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -105,8 +105,10 @@ Retry: heldList: list.New(), requestList: list.New(), inodeHeadV1: nil, - payload: nil, + linkSet: nil, + streamMap: nil, layoutMap: nil, + payload: nil, superBlockInodeObjectCountAdjustment: 0, superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, From 7b8ac3077d2699607a89bfe66268fc93497ef40a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 1 Nov 2021 18:40:00 -0700 Subject: [PATCH 164/258] Finished non-dirInode renames --- iclient/iclientpkg/fission.go | 61 +++++++++++++++++++++++++++++------ iclient/iclientpkg/inode.go | 28 +++------------- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 33ea1aa2..4c8d1f72 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -2876,6 +2876,7 @@ Retry: return } + oldDirInode.dirty = true inodeSliceToFlush = append(inodeSliceToFlush, oldDirInode) if nil == oldDirInode.inodeHeadV1 { @@ -2925,6 +2926,7 @@ Retry: goto Retry } + renamedInode.dirty = true inodeSliceToFlush = append(inodeSliceToFlush, renamedInode) if nil == renamedInode.inodeHeadV1 { @@ -2964,6 +2966,7 @@ Retry: return } + newDirInode.dirty = true inodeSliceToFlush = append(inodeSliceToFlush, newDirInode) if nil == newDirInode.inodeHeadV1 { @@ -3010,6 +3013,7 @@ Retry: goto Retry } + replacedInode.dirty = true inodeSliceToFlush = append(inodeSliceToFlush, replacedInode) if nil == replacedInode.inodeHeadV1 { @@ -3038,21 +3042,60 @@ Retry: // TODO - insert renamedInode in newDirInode's payload } else { if replacedInode != nil { - // TODO - first remove replacedInode + ok, err = newDirInode.payload.DeleteByKey(newName) + if nil != err { + logFatalf("newDirInode.payload.DeleteByKey(newName) failed: %v", err) + } + if !ok { + logFatalf("newDirInode.payload.DeleteByKey(newName) returned !ok") + } + + delete(replacedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: newDirInodeNumber, + ParentDirEntryName: newName, + }) + + if len(replacedInode.linkSet) == 0 { + logWarnf("TODO: we need to delete replacedInode") + } } - // TODO - next remove renamedInode from oldDirInode's payload - // TODO - replace link entry in renamedInode to be from newDirInode - // TODO - finally insert renamedInode in newDirInode's payload + ok, err = oldDirInode.payload.DeleteByKey(oldName) + if nil != err { + logFatalf("newDirInode.payload.DeleteByKey(newName) failed: %v", err) + } + if !ok { + logFatalf("newDirInode.payload.DeleteByKey(newName) returned !ok") + } + + delete(renamedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: oldDirInodeNumber, + ParentDirEntryName: oldName, + }) + + ok, err = newDirInode.payload.Put( + newName, + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: renamedInode.inodeNumber, + InodeType: renamedInode.inodeHeadV1.InodeType, + }) + if nil != err { + logFatalf("newDirInode.payload.Put(newName,) failed: %v", err) + } + if !ok { + logFatalf("newDirInode.payload.Put(newName,) returned !ok") + } + + renamedInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: newDirInodeNumber, + ParentDirEntryName: newName, + }] = struct{}{} } - fmt.Printf("\nUNDO: renamedInode: %+v\n", renamedInode) - fmt.Printf("UNDO: replacedInode: %+v\n", replacedInode) - fmt.Printf("UNDO: inodeSliceToFlush: %+v\n\n", inodeSliceToFlush) + flushInodesInSlice(inodeSliceToFlush) inodeLockRequest.unlockAll() - // TODO - errno = syscall.ENOSYS + errno = 0 return } diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 10c7a55b..3ab3f9bc 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -133,7 +133,10 @@ func (inode *inodeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, ob layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] if !ok { - log.Fatalf("inode.layoutMap[old inode.putObjectNumber] returned !ok") + layoutMapEntry = layoutMapEntryStruct{ + objectSize: 0, + bytesReferenced: 0, + } } objectNumber = inode.putObjectNumber @@ -643,28 +646,10 @@ func (inode *inodeStruct) oldPayload() (err error) { } func (inode *inodeStruct) ensurePutObjectIsActive() { - var ( - layoutMapEntry layoutMapEntryStruct - ok bool - ) - if inode.putObjectNumber == 0 { inode.putObjectNumber = fetchNonce() inode.putObjectBuffer = make([]byte, 0) - if inode.inodeHeadV1.InodeType != ilayout.InodeTypeSymLink { - layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] - if ok { - log.Fatalf("inode.layoutMap[inode.putObjectNumber] returned ok") - } - layoutMapEntry = layoutMapEntryStruct{ - objectSize: 0, - bytesReferenced: 0, - } - - inode.layoutMap[inode.putObjectNumber] = layoutMapEntry - } - inode.superBlockInodeObjectCountAdjustment++ } } @@ -747,7 +732,6 @@ func flushInodeNumbersInSlice(inodeNumberSlice []uint64) { func flushInodesInSlice(inodeSlice []*inodeStruct) { var ( - dereferencedObjectNumber uint64 err error inode *inodeStruct inodeHeadLength uint64 @@ -781,9 +765,7 @@ func flushInodesInSlice(inodeSlice []*inodeStruct) { putInodeTableEntriesRequest.SuperBlockInodeObjectSizeAdjustment += inode.superBlockInodeObjectSizeAdjustment putInodeTableEntriesRequest.SuperBlockInodeBytesReferencedAdjustment += inode.superBlockInodeBytesReferencedAdjustment - for _, dereferencedObjectNumber = range inode.dereferencedObjectNumberArray { - putInodeTableEntriesRequest.DereferencedObjectNumberArray = append(putInodeTableEntriesRequest.DereferencedObjectNumberArray, dereferencedObjectNumber) - } + putInodeTableEntriesRequest.DereferencedObjectNumberArray = append(putInodeTableEntriesRequest.DereferencedObjectNumberArray, inode.dereferencedObjectNumberArray...) inode.dirty = false From 044bee7fcfeb8d969cb6b0fc5a0a8a0fc1be9957 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 1 Nov 2021 19:05:28 -0700 Subject: [PATCH 165/258] Finished renames (directories were missing before) --- iclient/iclientpkg/fission.go | 71 +++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 4c8d1f72..b3691a63 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -3035,11 +3035,68 @@ Retry: return } - // TODO - remove renamedInode from oldDirInode's payload - // TODO - remove renamedInode's ".." link from oldDirInode - // TODO - replace renamedInode's ".." to point to newDirInode - // TODO - add renamedInodes's ".." link in newDirInode - // TODO - insert renamedInode in newDirInode's payload + if renamedInode.payload == nil { + err = renamedInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + ok, err = oldDirInode.payload.DeleteByKey(oldName) + if nil != err { + logFatalf("oldDirInode.payload.DeleteByKey(newName) failed: %v", err) + } + if !ok { + logFatalf("oldDirInode.payload.DeleteByKey(newName) returned !ok") + } + + delete(oldDirInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: renamedInode.inodeNumber, + ParentDirEntryName: "..", + }) + + delete(renamedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: oldDirInodeNumber, + ParentDirEntryName: oldName, + }) + + ok, err = renamedInode.payload.PatchByKey( + "..", + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: newDirInodeNumber, + InodeType: ilayout.InodeTypeDir, + }) + if nil != err { + logFatalf("renamedInode.payload.PatchByKey(\"..\",) failed: %v", err) + } + if !ok { + logFatalf("renamedInode.payload.PatchByKey(\"..\",) returned !ok") + } + + ok, err = newDirInode.payload.Put( + newName, + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: renamedInode.inodeNumber, + InodeType: ilayout.InodeTypeDir, + }) + if nil != err { + logFatalf("newDirInode.payload.Put(newName,) failed: %v", err) + } + if !ok { + logFatalf("newDirInode.payload.Put(newName,) returned !ok") + } + + newDirInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: renamedInode.inodeNumber, + ParentDirEntryName: "..", + }] = struct{}{} + + renamedInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: newDirInodeNumber, + ParentDirEntryName: newName, + }] = struct{}{} } else { if replacedInode != nil { ok, err = newDirInode.payload.DeleteByKey(newName) @@ -3062,10 +3119,10 @@ Retry: ok, err = oldDirInode.payload.DeleteByKey(oldName) if nil != err { - logFatalf("newDirInode.payload.DeleteByKey(newName) failed: %v", err) + logFatalf("oldDirInode.payload.DeleteByKey(newName) failed: %v", err) } if !ok { - logFatalf("newDirInode.payload.DeleteByKey(newName) returned !ok") + logFatalf("oldDirInode.payload.DeleteByKey(newName) returned !ok") } delete(renamedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ From 793d89df9d01102d3370d35bc85f9e616f9ee878 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 2 Nov 2021 17:36:26 -0700 Subject: [PATCH 166/258] Finished DoLink() --- iclient/iclientpkg/fission.go | 80 +++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index b3691a63..9672d07a 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -879,8 +879,9 @@ func (dummy *globalsStruct) DoLink(inHeader *fission.InHeader, linkIn *fission.L dirInode *inodeStruct err error inodeLockRequest *inodeLockRequestStruct + ok bool startTime time.Time = time.Now() - // targetInode *inodeStruct + targetInode *inodeStruct ) logTracef("==> DoLink(inHeader: %+v, linkIn: %+v)", inHeader, linkIn) @@ -936,9 +937,80 @@ Retry: } } - // TODO - linkOut = nil - errno = syscall.ENOSYS + _, ok, err = dirInode.payload.GetByKey(string(linkIn.Name[:])) + if nil != err { + logFatalf("dirInode.payload.GetByKey(string(linkIn.Name[:])) failed: %v", err) + } + if ok { + inodeLockRequest.unlockAll() + linkOut = nil + errno = syscall.EEXIST + return + } + + inodeLockRequest.inodeNumber = linkIn.OldNodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + targetInode = lookupInode(linkIn.OldNodeID) + if nil == dirInode { + inodeLockRequest.unlockAll() + linkOut = nil + errno = syscall.ENOENT + return + } + + if targetInode.inodeHeadV1.InodeType == ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + linkOut = nil + errno = syscall.EISDIR + return + } + + targetInode.dirty = true + + targetInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: dirInode.inodeNumber, + ParentDirEntryName: string(linkIn.Name[:]), + }] = struct{}{} + + dirInode.dirty = true + + ok, err = dirInode.payload.Put( + string(linkIn.Name[:]), + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: targetInode.inodeNumber, + InodeType: targetInode.inodeHeadV1.InodeType, + }) + if nil != err { + logFatalf("dirInode.payload.Put(string(linkIn.Name[:]),) failed: %v", err) + } + if !ok { + logFatalf("dirInode.payload.Put(string(linkIn.Name[:]),) returned !ok") + } + + flushInodesInSlice([]*inodeStruct{dirInode, targetInode}) + + linkOut = &fission.LinkOut{ + EntryOut: fission.EntryOut{ + NodeID: targetInode.inodeHeadV1.InodeNumber, + Generation: 0, + EntryValidSec: globals.fuseEntryValidDurationSec, + AttrValidSec: globals.fuseAttrValidDurationSec, + EntryValidNSec: globals.fuseEntryValidDurationNSec, + AttrValidNSec: globals.fuseAttrValidDurationNSec, + // Attr to be filled in below + }, + } + + targetInode.doAttrFetch(&linkOut.EntryOut.Attr) + + inodeLockRequest.unlockAll() + + errno = 0 return } From d1371ced7da19db70090edb100990252205977c3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 2 Nov 2021 17:53:16 -0700 Subject: [PATCH 167/258] Correctly update inode.inodeHeadV1.{Modification|StatusChange}Time in recent Do*() additions --- iclient/iclientpkg/fission.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 9672d07a..0fb677f9 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -755,6 +755,9 @@ Retry: parentDirInode.dirty = true + parentDirInode.inodeHeadV1.ModificationTime = startTime + parentDirInode.inodeHeadV1.StatusChangeTime = startTime + parentDirInode.linkSet[ilayout.InodeLinkTableEntryStruct{ ParentDirInodeNumber: childDirInode.inodeNumber, ParentDirEntryName: "..", @@ -972,6 +975,9 @@ Retry: targetInode.dirty = true + targetInode.inodeHeadV1.ModificationTime = startTime + targetInode.inodeHeadV1.StatusChangeTime = startTime + targetInode.linkSet[ilayout.InodeLinkTableEntryStruct{ ParentDirInodeNumber: dirInode.inodeNumber, ParentDirEntryName: string(linkIn.Name[:]), @@ -979,6 +985,9 @@ Retry: dirInode.dirty = true + dirInode.inodeHeadV1.ModificationTime = startTime + dirInode.inodeHeadV1.StatusChangeTime = startTime + ok, err = dirInode.payload.Put( string(linkIn.Name[:]), &ilayout.DirectoryEntryValueV1Struct{ @@ -2951,6 +2960,9 @@ Retry: oldDirInode.dirty = true inodeSliceToFlush = append(inodeSliceToFlush, oldDirInode) + oldDirInode.inodeHeadV1.ModificationTime = startTime + oldDirInode.inodeHeadV1.StatusChangeTime = startTime + if nil == oldDirInode.inodeHeadV1 { err = oldDirInode.populateInodeHeadV1() if nil != err { @@ -3001,6 +3013,9 @@ Retry: renamedInode.dirty = true inodeSliceToFlush = append(inodeSliceToFlush, renamedInode) + renamedInode.inodeHeadV1.ModificationTime = startTime + renamedInode.inodeHeadV1.StatusChangeTime = startTime + if nil == renamedInode.inodeHeadV1 { err = renamedInode.populateInodeHeadV1() if nil != err { @@ -3041,6 +3056,9 @@ Retry: newDirInode.dirty = true inodeSliceToFlush = append(inodeSliceToFlush, newDirInode) + newDirInode.inodeHeadV1.ModificationTime = startTime + newDirInode.inodeHeadV1.StatusChangeTime = startTime + if nil == newDirInode.inodeHeadV1 { err = newDirInode.populateInodeHeadV1() if nil != err { @@ -3088,6 +3106,9 @@ Retry: replacedInode.dirty = true inodeSliceToFlush = append(inodeSliceToFlush, replacedInode) + replacedInode.inodeHeadV1.ModificationTime = startTime + replacedInode.inodeHeadV1.StatusChangeTime = startTime + if nil == replacedInode.inodeHeadV1 { err = replacedInode.populateInodeHeadV1() if nil != err { From 41458014251e439de1d532ebb614a71b5c9733ba Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 5 Nov 2021 11:07:07 -0700 Subject: [PATCH 168/258] Completed DoUnlink() and DoRmDir() while cleaning up doRenameCommon() ...also got Delete of an Inode RPC working (TODO delete-on-close though) --- iclient/iclientpkg/fission.go | 568 +++++++++++++++++++++++++++------- iclient/iclientpkg/globals.go | 1 - iclient/iclientpkg/lease.go | 36 ++- imgr/imgrpkg/retry-rpc.go | 45 ++- 4 files changed, 532 insertions(+), 118 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 0fb677f9..c946bd8b 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -823,7 +823,16 @@ Retry: func (dummy *globalsStruct) DoUnlink(inHeader *fission.InHeader, unlinkIn *fission.UnlinkIn) (errno syscall.Errno) { var ( - startTime time.Time = time.Now() + deleteInodeTableEntryRequest *imgrpkg.DeleteInodeTableEntryRequestStruct + deleteInodeTableEntryResponse *imgrpkg.DeleteInodeTableEntryResponseStruct + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + dirInode *inodeStruct + err error + inodeLockRequest *inodeLockRequestStruct + ok bool + startTime time.Time = time.Now() + targetInode *inodeStruct ) logTracef("==> DoUnlink(inHeader: %+v, unlinkIn: %+v)", inHeader, unlinkIn) @@ -835,14 +844,149 @@ func (dummy *globalsStruct) DoUnlink(inHeader *fission.InHeader, unlinkIn *fissi globals.stats.DoUnlinkUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + dirInode = lookupInode(inHeader.NodeID) + if nil == dirInode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + if nil == dirInode.inodeHeadV1 { + err = dirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if dirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.ENOTDIR + return + } + + if dirInode.payload == nil { + err = dirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + directoryEntryValueV1AsValue, ok, err = dirInode.payload.GetByKey(string(unlinkIn.Name[:])) + if nil != err { + logFatalf("dirInode.payload.GetByKey(string(unlinkIn.Name[:])) failed: %v", err) + } + if !ok { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + inodeLockRequest.inodeNumber = directoryEntryValueV1.InodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + targetInode = lookupInode(directoryEntryValueV1.InodeNumber) + if nil == targetInode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + if nil == targetInode.inodeHeadV1 { + err = targetInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if targetInode.inodeHeadV1.InodeType == ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.EISDIR + return + } + + dirInode.dirty = true + + dirInode.inodeHeadV1.ModificationTime = startTime + dirInode.inodeHeadV1.StatusChangeTime = startTime + + ok, err = dirInode.payload.DeleteByKey(string(unlinkIn.Name[:])) + if nil != err { + logFatalf("dirInode.payload.DeleteByKey(string(unlinkIn.Name[:]) failed: %v", err) + } + if !ok { + logFatalf("dirInode.payload.DeleteByKey(string(unlinkIn.Name[:]) returned !ok") + } + + targetInode.dirty = true + + targetInode.inodeHeadV1.ModificationTime = startTime + targetInode.inodeHeadV1.StatusChangeTime = startTime + + delete(targetInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: dirInode.inodeNumber, + ParentDirEntryName: string(unlinkIn.Name[:]), + }) + + flushInodesInSlice([]*inodeStruct{dirInode, targetInode}) + + if len(targetInode.linkSet) == 0 { + inodeLockRequest.markForDelete(targetInode.inodeNumber) + + deleteInodeTableEntryRequest = &imgrpkg.DeleteInodeTableEntryRequestStruct{ + MountID: globals.mountID, + InodeNumber: targetInode.inodeNumber, + } + deleteInodeTableEntryResponse = &imgrpkg.DeleteInodeTableEntryResponseStruct{} + + err = rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) + if nil != err { + logFatalf("rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) failed: %v", err) + } + } + + inodeLockRequest.unlockAll() + + errno = 0 return } func (dummy *globalsStruct) DoRmDir(inHeader *fission.InHeader, rmDirIn *fission.RmDirIn) (errno syscall.Errno) { var ( - startTime time.Time = time.Now() + childDirInode *inodeStruct + childDirInodePayloadLen int + deleteInodeTableEntryRequest *imgrpkg.DeleteInodeTableEntryRequestStruct + deleteInodeTableEntryResponse *imgrpkg.DeleteInodeTableEntryResponseStruct + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + err error + inodeLockRequest *inodeLockRequestStruct + ok bool + parentDirInode *inodeStruct + startTime time.Time = time.Now() ) logTracef("==> DoRmDir(inHeader: %+v, rmDirIn: %+v)", inHeader, rmDirIn) @@ -854,8 +998,145 @@ func (dummy *globalsStruct) DoRmDir(inHeader *fission.InHeader, rmDirIn *fission globals.stats.DoRmDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + parentDirInode = lookupInode(inHeader.NodeID) + if nil == parentDirInode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + if nil == parentDirInode.inodeHeadV1 { + err = parentDirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if parentDirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.ENOTDIR + return + } + + if parentDirInode.payload == nil { + err = parentDirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + directoryEntryValueV1AsValue, ok, err = parentDirInode.payload.GetByKey(string(rmDirIn.Name[:])) + if nil != err { + logFatalf("parentDirInode.payload.GetByKey(string(rmDirIn.Name[:])) failed: %v", err) + } + if !ok { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) + if !ok { + logFatalf("directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) returned !ok") + } + + inodeLockRequest.inodeNumber = directoryEntryValueV1.InodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + childDirInode = lookupInode(directoryEntryValueV1.InodeNumber) + if nil == childDirInode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + if nil == childDirInode.inodeHeadV1 { + err = childDirInode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if childDirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.EISDIR + return + } + + if childDirInode.payload == nil { + err = childDirInode.oldPayload() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + childDirInodePayloadLen, err = childDirInode.payload.Len() + if nil != err { + logFatalf("childDirInode.payload.Len() failed: %v", err) + } + if childDirInodePayloadLen != 2 { + inodeLockRequest.unlockAll() + errno = syscall.ENOTEMPTY + return + } + + parentDirInode.dirty = true + + parentDirInode.inodeHeadV1.ModificationTime = startTime + parentDirInode.inodeHeadV1.StatusChangeTime = startTime + + delete(parentDirInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: childDirInode.inodeNumber, + ParentDirEntryName: "..", + }) + + ok, err = parentDirInode.payload.DeleteByKey(string(rmDirIn.Name[:])) + if nil != err { + logFatalf("parentDirInode.payload.DeleteByKey(oldName) failed: %v", err) + } + if !ok { + logFatalf("parentDirInode.payload.DeleteByKey(oldName) returned !ok") + } + + flushInodesInSlice([]*inodeStruct{parentDirInode}) + + inodeLockRequest.markForDelete(childDirInode.inodeNumber) + + deleteInodeTableEntryRequest = &imgrpkg.DeleteInodeTableEntryRequestStruct{ + MountID: globals.mountID, + InodeNumber: childDirInode.inodeNumber, + } + deleteInodeTableEntryResponse = &imgrpkg.DeleteInodeTableEntryResponseStruct{} + + err = rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) + if nil != err { + logFatalf("rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) failed: %v", err) + } + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -959,7 +1240,7 @@ Retry: } targetInode = lookupInode(linkIn.OldNodeID) - if nil == dirInode { + if nil == targetInode { inodeLockRequest.unlockAll() linkOut = nil errno = syscall.ENOENT @@ -2924,16 +3205,17 @@ func fixAttrSizes(attr *fission.Attr) { func doRenameCommon(oldDirInodeNumber uint64, oldName string, newDirInodeNumber uint64, newName string, startTime time.Time) (errno syscall.Errno) { var ( - directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct - directoryEntryValueV1AsValue sortedmap.Value - err error - inodeLockRequest *inodeLockRequestStruct - inodeSliceToFlush []*inodeStruct = make([]*inodeStruct, 0) - newDirInode *inodeStruct - ok bool - oldDirInode *inodeStruct - renamedInode *inodeStruct - replacedInode *inodeStruct + deleteInodeTableEntryRequest *imgrpkg.DeleteInodeTableEntryRequestStruct + deleteInodeTableEntryResponse *imgrpkg.DeleteInodeTableEntryResponseStruct + directoryEntryValueV1 *ilayout.DirectoryEntryValueV1Struct + directoryEntryValueV1AsValue sortedmap.Value + err error + inodeLockRequest *inodeLockRequestStruct + newDirInode *inodeStruct + ok bool + oldDirInode *inodeStruct + renamedInode *inodeStruct + replacedInode *inodeStruct ) Retry: @@ -2951,17 +3233,6 @@ Retry: errno = syscall.ENOENT return } - if oldDirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { - inodeLockRequest.unlockAll() - errno = syscall.ENOTDIR - return - } - - oldDirInode.dirty = true - inodeSliceToFlush = append(inodeSliceToFlush, oldDirInode) - - oldDirInode.inodeHeadV1.ModificationTime = startTime - oldDirInode.inodeHeadV1.StatusChangeTime = startTime if nil == oldDirInode.inodeHeadV1 { err = oldDirInode.populateInodeHeadV1() @@ -2972,6 +3243,12 @@ Retry: } } + if oldDirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.ENOTDIR + return + } + if oldDirInode.payload == nil { err = oldDirInode.oldPayload() if nil != err { @@ -3010,12 +3287,6 @@ Retry: goto Retry } - renamedInode.dirty = true - inodeSliceToFlush = append(inodeSliceToFlush, renamedInode) - - renamedInode.inodeHeadV1.ModificationTime = startTime - renamedInode.inodeHeadV1.StatusChangeTime = startTime - if nil == renamedInode.inodeHeadV1 { err = renamedInode.populateInodeHeadV1() if nil != err { @@ -3053,12 +3324,6 @@ Retry: return } - newDirInode.dirty = true - inodeSliceToFlush = append(inodeSliceToFlush, newDirInode) - - newDirInode.inodeHeadV1.ModificationTime = startTime - newDirInode.inodeHeadV1.StatusChangeTime = startTime - if nil == newDirInode.inodeHeadV1 { err = newDirInode.populateInodeHeadV1() if nil != err { @@ -3098,25 +3363,24 @@ Retry: } replacedInode = lookupInode(directoryEntryValueV1.InodeNumber) - if nil == renamedInode { + if nil == replacedInode { inodeLockRequest.unlockAll() goto Retry } - replacedInode.dirty = true - inodeSliceToFlush = append(inodeSliceToFlush, replacedInode) - - replacedInode.inodeHeadV1.ModificationTime = startTime - replacedInode.inodeHeadV1.StatusChangeTime = startTime - if nil == replacedInode.inodeHeadV1 { err = replacedInode.populateInodeHeadV1() if nil != err { inodeLockRequest.unlockAll() - errno = syscall.ENOENT - return + goto Retry } } + + if replacedInode.inodeHeadV1.InodeType == ilayout.InodeTypeDir { + inodeLockRequest.unlockAll() + errno = syscall.EISDIR + return + } } else { replacedInode = nil } @@ -3137,24 +3401,21 @@ Retry: } } - ok, err = oldDirInode.payload.DeleteByKey(oldName) - if nil != err { - logFatalf("oldDirInode.payload.DeleteByKey(newName) failed: %v", err) - } - if !ok { - logFatalf("oldDirInode.payload.DeleteByKey(newName) returned !ok") - } + renamedInode.dirty = true - delete(oldDirInode.linkSet, ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: renamedInode.inodeNumber, - ParentDirEntryName: "..", - }) + renamedInode.inodeHeadV1.ModificationTime = startTime + renamedInode.inodeHeadV1.StatusChangeTime = startTime delete(renamedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ ParentDirInodeNumber: oldDirInodeNumber, ParentDirEntryName: oldName, }) + renamedInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: newDirInodeNumber, + ParentDirEntryName: newName, + }] = struct{}{} + ok, err = renamedInode.payload.PatchByKey( "..", &ilayout.DirectoryEntryValueV1Struct{ @@ -3168,6 +3429,34 @@ Retry: logFatalf("renamedInode.payload.PatchByKey(\"..\",) returned !ok") } + oldDirInode.dirty = true + + oldDirInode.inodeHeadV1.ModificationTime = startTime + oldDirInode.inodeHeadV1.StatusChangeTime = startTime + + delete(oldDirInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: renamedInode.inodeNumber, + ParentDirEntryName: oldName, + }) + + ok, err = oldDirInode.payload.DeleteByKey(oldName) + if nil != err { + logFatalf("oldDirInode.payload.DeleteByKey(oldName) failed: %v", err) + } + if !ok { + logFatalf("oldDirInode.payload.DeleteByKey(oldName) returned !ok") + } + + newDirInode.dirty = true + + newDirInode.inodeHeadV1.ModificationTime = startTime + newDirInode.inodeHeadV1.StatusChangeTime = startTime + + newDirInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: renamedInode.inodeNumber, + ParentDirEntryName: "..", + }] = struct{}{} + ok, err = newDirInode.payload.Put( newName, &ilayout.DirectoryEntryValueV1Struct{ @@ -3180,70 +3469,141 @@ Retry: if !ok { logFatalf("newDirInode.payload.Put(newName,) returned !ok") } - - newDirInode.linkSet[ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: renamedInode.inodeNumber, - ParentDirEntryName: "..", - }] = struct{}{} - - renamedInode.linkSet[ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: newDirInodeNumber, - ParentDirEntryName: newName, - }] = struct{}{} } else { if replacedInode != nil { - ok, err = newDirInode.payload.DeleteByKey(newName) + replacedInode.dirty = true + + replacedInode.inodeHeadV1.ModificationTime = startTime + replacedInode.inodeHeadV1.StatusChangeTime = startTime + + delete(replacedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: newDirInodeNumber, + ParentDirEntryName: newName, + }) + + renamedInode.dirty = true + + renamedInode.inodeHeadV1.ModificationTime = startTime + renamedInode.inodeHeadV1.StatusChangeTime = startTime + + delete(renamedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: oldDirInodeNumber, + ParentDirEntryName: oldName, + }) + + renamedInode.linkSet[ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: newDirInodeNumber, + ParentDirEntryName: newName, + }] = struct{}{} + + oldDirInode.dirty = true + + oldDirInode.inodeHeadV1.ModificationTime = startTime + oldDirInode.inodeHeadV1.StatusChangeTime = startTime + + ok, err = oldDirInode.payload.DeleteByKey(oldName) if nil != err { - logFatalf("newDirInode.payload.DeleteByKey(newName) failed: %v", err) + logFatalf("oldDirInode.payload.DeleteByKey(oldName) failed: %v", err) } if !ok { - logFatalf("newDirInode.payload.DeleteByKey(newName) returned !ok") + logFatalf("oldDirInode.payload.DeleteByKey(oldName) returned !ok") } - delete(replacedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + newDirInode.dirty = true + + newDirInode.inodeHeadV1.ModificationTime = startTime + newDirInode.inodeHeadV1.StatusChangeTime = startTime + + ok, err = newDirInode.payload.PatchByKey( + newName, + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: renamedInode.inodeNumber, + InodeType: ilayout.InodeTypeDir, + }) + if nil != err { + logFatalf("newDirInode.payload.PatchByKey(newName,) failed: %v", err) + } + if !ok { + logFatalf("newDirInode.payload.PatchByKey(newName,) returned !ok") + } + } else { + renamedInode.dirty = true + + renamedInode.inodeHeadV1.ModificationTime = startTime + renamedInode.inodeHeadV1.StatusChangeTime = startTime + + delete(renamedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ + ParentDirInodeNumber: oldDirInodeNumber, + ParentDirEntryName: oldName, + }) + + renamedInode.linkSet[ilayout.InodeLinkTableEntryStruct{ ParentDirInodeNumber: newDirInodeNumber, ParentDirEntryName: newName, - }) + }] = struct{}{} + + oldDirInode.dirty = true + + oldDirInode.inodeHeadV1.ModificationTime = startTime + oldDirInode.inodeHeadV1.StatusChangeTime = startTime + + ok, err = oldDirInode.payload.DeleteByKey(oldName) + if nil != err { + logFatalf("oldDirInode.payload.DeleteByKey(oldName) failed: %v", err) + } + if !ok { + logFatalf("oldDirInode.payload.DeleteByKey(oldName) returned !ok") + } + + newDirInode.dirty = true + + newDirInode.inodeHeadV1.ModificationTime = startTime + newDirInode.inodeHeadV1.StatusChangeTime = startTime - if len(replacedInode.linkSet) == 0 { - logWarnf("TODO: we need to delete replacedInode") + ok, err = newDirInode.payload.Put( + newName, + &ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: renamedInode.inodeNumber, + InodeType: ilayout.InodeTypeDir, + }) + if nil != err { + logFatalf("newDirInode.payload.Put(newName,) failed: %v", err) + } + if !ok { + logFatalf("newDirInode.payload.Put(newName,) returned !ok") } } + } - ok, err = oldDirInode.payload.DeleteByKey(oldName) - if nil != err { - logFatalf("oldDirInode.payload.DeleteByKey(newName) failed: %v", err) + if replacedInode == nil { + if oldDirInodeNumber == newDirInodeNumber { + flushInodesInSlice([]*inodeStruct{renamedInode, oldDirInode}) + } else { + flushInodesInSlice([]*inodeStruct{renamedInode, oldDirInode, newDirInode}) } - if !ok { - logFatalf("oldDirInode.payload.DeleteByKey(newName) returned !ok") + } else { + if oldDirInodeNumber == newDirInodeNumber { + flushInodesInSlice([]*inodeStruct{replacedInode, renamedInode, oldDirInode}) + } else { + flushInodesInSlice([]*inodeStruct{replacedInode, renamedInode, oldDirInode, newDirInode}) } - delete(renamedInode.linkSet, ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: oldDirInodeNumber, - ParentDirEntryName: oldName, - }) + if len(replacedInode.linkSet) == 0 { + inodeLockRequest.markForDelete(replacedInode.inodeNumber) - ok, err = newDirInode.payload.Put( - newName, - &ilayout.DirectoryEntryValueV1Struct{ - InodeNumber: renamedInode.inodeNumber, - InodeType: renamedInode.inodeHeadV1.InodeType, - }) - if nil != err { - logFatalf("newDirInode.payload.Put(newName,) failed: %v", err) - } - if !ok { - logFatalf("newDirInode.payload.Put(newName,) returned !ok") - } + deleteInodeTableEntryRequest = &imgrpkg.DeleteInodeTableEntryRequestStruct{ + MountID: globals.mountID, + InodeNumber: replacedInode.inodeNumber, + } + deleteInodeTableEntryResponse = &imgrpkg.DeleteInodeTableEntryResponseStruct{} - renamedInode.linkSet[ilayout.InodeLinkTableEntryStruct{ - ParentDirInodeNumber: newDirInodeNumber, - ParentDirEntryName: newName, - }] = struct{}{} + err = rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) + if nil != err { + logFatalf("rpcDeleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) failed: %v", err) + } + } } - flushInodesInSlice(inodeSliceToFlush) - inodeLockRequest.unlockAll() errno = 0 diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 50732569..0215574d 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -99,7 +99,6 @@ type openHandleStruct struct { } type inodeStruct struct { - sync.WaitGroup // Signaled to indicate .markedForDelete == true triggered removal has completed inodeNumber uint64 // dirty bool // markedForDelete bool // If true, remove from globalsStruct.inodeTable upon last dereference diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index bc807a24..cc937687 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -30,6 +30,32 @@ func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { return } +// markForDelete is called to schedule an inode to be deleted from globals.inodeTable +// upon last dereference. Note that an exclusive lock must be held for the specified +// inodeNumber and that globals.Lock() must not be held. +// +func (inodeLockRequest *inodeLockRequestStruct) markForDelete(inodeNumber uint64) { + var ( + inodeHeldLock *inodeHeldLockStruct + ok bool + ) + + globals.Lock() + + inodeHeldLock, ok = inodeLockRequest.locksHeld[inodeNumber] + if !ok { + logFatalf("inodeLockRequest.locksHeld[inodeNumber] returned !ok") + } + + if !inodeHeldLock.exclusive { + logFatalf("inodeHeldLock.exclusive was false") + } + + inodeHeldLock.inode.markedForDelete = true + + globals.Unlock() +} + // addThisLock is called for an existing inodeLockRequestStruct with the inodeNumber and // exclusive fields set to specify the inode to be locked and whether or not the lock // should be exlusive. @@ -54,8 +80,6 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { ok bool ) -Retry: - globals.Lock() if inodeLockRequest.inodeNumber == 0 { @@ -68,12 +92,6 @@ Retry: inode, ok = globals.inodeTable[inodeLockRequest.inodeNumber] if ok { - if inode.markedForDelete { - globals.Unlock() - inode.Wait() - goto Retry - } - switch inode.leaseState { case inodeLeaseStateNone: case inodeLeaseStateSharedRequested: @@ -492,8 +510,6 @@ func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeReleased { logFatalf("received unexpected leaseResponse.LeaseResponseType: %v", leaseResponse.LeaseResponseType) } - - inode.Done() } inodeLockRequest.locksHeld = make(map[uint64]*inodeHeldLockStruct) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 3d9e5f19..57e2bbea 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -479,18 +479,57 @@ func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesReque func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { var ( - startTime time.Time = time.Now() + leaseRequest *leaseRequestStruct + mount *mountStruct + ok bool + startTime time.Time = time.Now() ) defer func() { globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO: Mark for deletion... and, if inodeOpenCount == 0, kick off delete of inode now + globals.Lock() + + mount, ok = globals.mountMap[deleteInodeTableEntryRequest.MountID] + if !ok { + globals.Unlock() + err = fmt.Errorf("%s %s", EUnknownMountID, deleteInodeTableEntryRequest.MountID) + return + } + + if mount.authTokenHasExpired() { + globals.Unlock() + err = fmt.Errorf("%s %s", EAuthTokenRejected, mount.authToken) + return + } + + leaseRequest, ok = mount.leaseRequestMap[deleteInodeTableEntryRequest.InodeNumber] + if !ok || (leaseRequestStateExclusiveGranted != leaseRequest.requestState) { + globals.Unlock() + err = fmt.Errorf("%s %016X", EMissingLease, deleteInodeTableEntryRequest.InodeNumber) + return + } + + // TODO: Need to actually clean up the Inode... but for now, just remove it + + ok, err = mount.volume.inodeTable.DeleteByKey(deleteInodeTableEntryRequest.InodeNumber) + if nil != err { + logFatalf("volume.inodeTable.DeleteByKey(n.InodeNumber) failed: %v", err) + } + if !ok { + logFatalf("volume.inodeTable.DeleteByKey(n.InodeNumber) returned !ok") + } + + globals.Unlock() - return fmt.Errorf(ETODO + " deleteInodeTableEntry") + err = nil + return } +// TODO - The thinking is that some "to-be-deleted-upon-last-close" logic will substitute +// for the InodeTable entry... and that some background garbage collector will do the work. + func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { var ( leaseRequest *leaseRequestStruct From b609f61eaf37195905f9f9a3be717b5de14e5564 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 5 Nov 2021 16:33:24 -0700 Subject: [PATCH 169/258] Finished Do{Set|Get|List|Remove}XAttr() ...Need to add support for XATTR_CREATE & XATTR_REPLACE in fission.SetXAttrIn.Flags... --- iclient/iclientpkg/fission.go | 264 ++++++++++++++++++++++++++++++++-- 1 file changed, 250 insertions(+), 14 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index c946bd8b..f12adc53 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1893,7 +1893,10 @@ func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission func (dummy *globalsStruct) DoSetXAttr(inHeader *fission.InHeader, setXAttrIn *fission.SetXAttrIn) (errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + startTime time.Time = time.Now() ) logTracef("==> DoSetXAttr(inHeader: %+v, setXAttrIn: %+v)", inHeader, setXAttrIn) @@ -1905,14 +1908,55 @@ func (dummy *globalsStruct) DoSetXAttr(inHeader *fission.InHeader, setXAttrIn *f globals.stats.DoSetXAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + inode.dirty = true + + inode.inodeHeadV1.ModificationTime = startTime + inode.inodeHeadV1.StatusChangeTime = startTime + + inode.streamMap[string(setXAttrIn.Name[:])] = setXAttrIn.Data + + flushInodesInSlice([]*inodeStruct{inode}) + + inodeLockRequest.unlockAll() + + errno = 0 return } func (dummy *globalsStruct) DoGetXAttr(inHeader *fission.InHeader, getXAttrIn *fission.GetXAttrIn) (getXAttrOut *fission.GetXAttrOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + ok bool + startTime time.Time = time.Now() + streamData []byte ) logTracef("==> DoGetXAttr(inHeader: %+v, getXAttrIn: %+v)", inHeader, getXAttrIn) @@ -1924,15 +1968,91 @@ func (dummy *globalsStruct) DoGetXAttr(inHeader *fission.InHeader, getXAttrIn *f globals.stats.DoGetXAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - getXAttrOut = nil - errno = syscall.ENOSYS + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + getXAttrOut = nil + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + getXAttrOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + streamData, ok = inode.streamMap[string(getXAttrIn.Name[:])] + if !ok { + inodeLockRequest.unlockAll() + getXAttrOut = nil + errno = syscall.ENODATA + return + } + + if getXAttrIn.Size == 0 { + getXAttrOut = &fission.GetXAttrOut{ + Size: uint32(len(streamData)), + Padding: 0, + Data: make([]byte, 0), + } + inodeLockRequest.unlockAll() + errno = 0 + return + } + + if getXAttrIn.Size < uint32(len(streamData)) { + inodeLockRequest.unlockAll() + getXAttrOut = nil + errno = syscall.ERANGE + return + } + + getXAttrOut = &fission.GetXAttrOut{ + Size: uint32(len(streamData)), + Padding: 0, + Data: make([]byte, len(streamData)), + } + _ = copy(getXAttrOut.Data, streamData) + + inodeLockRequest.unlockAll() + + errno = 0 return } func (dummy *globalsStruct) DoListXAttr(inHeader *fission.InHeader, listXAttrIn *fission.ListXAttrIn) (listXAttrOut *fission.ListXAttrOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + cumulativeStreamNameCount uint32 = 0 + cumulativeStreamNameSize uint32 = 0 + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + listXAttrOutName []byte + obtainExclusiveLock bool + startTime time.Time = time.Now() + streamName string ) logTracef("==> DoListXAttr(inHeader: %+v, listXAttrIn: %+v)", inHeader, listXAttrIn) @@ -1944,15 +2064,89 @@ func (dummy *globalsStruct) DoListXAttr(inHeader *fission.InHeader, listXAttrIn globals.stats.DoListXAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - listXAttrOut = nil - errno = syscall.ENOSYS + obtainExclusiveLock = false + +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = obtainExclusiveLock + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + listXAttrOut = nil + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + if obtainExclusiveLock { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + listXAttrOut = nil + errno = syscall.ENOENT + return + } + } else { + inodeLockRequest.unlockAll() + obtainExclusiveLock = true + goto Retry + } + } + + for streamName = range inode.streamMap { + cumulativeStreamNameCount++ + cumulativeStreamNameSize += uint32(len(streamName)) + 1 + } + + if listXAttrIn.Size == 0 { + inodeLockRequest.unlockAll() + listXAttrOut = &fission.ListXAttrOut{ + Size: cumulativeStreamNameSize, + Padding: 0, + Name: make([][]byte, 0), + } + errno = 0 + return + } + + if listXAttrIn.Size < cumulativeStreamNameSize { + inodeLockRequest.unlockAll() + listXAttrOut = nil + errno = syscall.ERANGE + return + } + + listXAttrOut = &fission.ListXAttrOut{ + Size: cumulativeStreamNameSize, + Padding: 0, + Name: make([][]byte, 0, cumulativeStreamNameCount), + } + + for streamName = range inode.streamMap { + listXAttrOutName = make([]byte, len(streamName)) + _ = copy(listXAttrOutName, streamName) + listXAttrOut.Name = append(listXAttrOut.Name, listXAttrOutName) + } + + inodeLockRequest.unlockAll() + + errno = 0 return } func (dummy *globalsStruct) DoRemoveXAttr(inHeader *fission.InHeader, removeXAttrIn *fission.RemoveXAttrIn) (errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + ok bool + startTime time.Time = time.Now() ) logTracef("==> DoRemoveXAttr(inHeader: %+v, removeXAttrIn: %+v)", inHeader, removeXAttrIn) @@ -1964,8 +2158,50 @@ func (dummy *globalsStruct) DoRemoveXAttr(inHeader *fission.InHeader, removeXAtt globals.stats.DoRemoveXAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + _, ok = inode.streamMap[string(removeXAttrIn.Name[:])] + if !ok { + inodeLockRequest.unlockAll() + errno = syscall.ENODATA + return + } + + inode.dirty = true + + inode.inodeHeadV1.ModificationTime = startTime + inode.inodeHeadV1.StatusChangeTime = startTime + + delete(inode.streamMap, string(removeXAttrIn.Name[:])) + + flushInodesInSlice([]*inodeStruct{inode}) + + inodeLockRequest.unlockAll() + + errno = 0 return } From 38156f3dbe22bd75902eb6ea0fa92c2ae349254e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 5 Nov 2021 16:58:50 -0700 Subject: [PATCH 170/258] Picked up newest package fission to get SetXAttrIn.Flags constants --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f04fbde2..4ac2b69d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 - github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066 + github.com/NVIDIA/fission v0.0.0-20211105235624-b9d1d59bb6c2 github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 github.com/ansel1/merry v1.6.1 github.com/creachadair/cityhash v0.1.0 diff --git a/go.sum b/go.sum index 6e3f5ee3..5710ebd6 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1Dnxq github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066 h1:K/8Os/A5UFB6mJiQaeGJtXjvb4sOLm6xj7ulQxqUACw= github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= +github.com/NVIDIA/fission v0.0.0-20211105235624-b9d1d59bb6c2 h1:aeeSFyX1qAyT7AgMJcAqj2f3mpoKJC6jvhqkl14qtr4= +github.com/NVIDIA/fission v0.0.0-20211105235624-b9d1d59bb6c2/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= From 95ceaad105e3f8ae9b658d826850bfe75e2e5f34 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 5 Nov 2021 17:10:54 -0700 Subject: [PATCH 171/258] Added support for SetXAttrIn{Create|Replace} values for SetXAttrIn.Flags --- iclient/iclientpkg/fission.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index f12adc53..86495b05 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1896,6 +1896,7 @@ func (dummy *globalsStruct) DoSetXAttr(inHeader *fission.InHeader, setXAttrIn *f err error inode *inodeStruct inodeLockRequest *inodeLockRequestStruct + ok bool startTime time.Time = time.Now() ) @@ -1933,6 +1934,29 @@ Retry: } } + _, ok = inode.streamMap[string(setXAttrIn.Name[:])] + + switch setXAttrIn.Flags { + case 0: + // Fall through + case fission.SetXAttrInCreate: + if ok { + inodeLockRequest.unlockAll() + errno = syscall.EEXIST + return + } + case fission.SetXAttrInReplace: + if !ok { + inodeLockRequest.unlockAll() + errno = syscall.ENODATA + return + } + default: + inodeLockRequest.unlockAll() + errno = syscall.ENOTSUP + return + } + inode.dirty = true inode.inodeHeadV1.ModificationTime = startTime From b2aa6f437263d7d09eeb865f6dce314501f91fcf Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 5 Nov 2021 18:05:20 -0700 Subject: [PATCH 172/258] Finished DoSetAttr() Note that truncate et. al. won't work until write support is added... --- iclient/iclientpkg/fission.go | 100 +++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 9 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 86495b05..fa0e0ebe 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -303,7 +303,11 @@ Retry: func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fission.SetAttrIn) (setAttrOut *fission.SetAttrOut, errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + inodeNumber uint64 + startTime time.Time = time.Now() ) logTracef("==> DoSetAttr(inHeader: %+v, setAttrIn: %+v)", inHeader, setAttrIn) @@ -315,9 +319,92 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis globals.stats.DoSetAttrUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - setAttrOut = nil - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + inode = lookupInode(inHeader.NodeID) + if nil == inode { + inodeLockRequest.unlockAll() + setAttrOut = nil + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + setAttrOut = nil + errno = syscall.ENOENT + return + } + } + + if (setAttrIn.Valid & fission.SetAttrInValidSize) != 0 { + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeFile { + inodeLockRequest.unlockAll() + setAttrOut = nil + errno = syscall.ENOENT + return + } + } + + inode.dirty = true + + inode.inodeHeadV1.StatusChangeTime = startTime + + if (setAttrIn.Valid & fission.SetAttrInValidMode) != 0 { + inode.inodeHeadV1.Mode = uint16(setAttrIn.Mode & uint32(syscall.S_IRWXU|syscall.S_IRWXG|syscall.S_IRWXO)) + } + + if (setAttrIn.Valid & fission.SetAttrInValidUID) != 0 { + inode.inodeHeadV1.UserID = uint64(setAttrIn.UID) + } + + if (setAttrIn.Valid & fission.SetAttrInValidGID) != 0 { + inode.inodeHeadV1.GroupID = uint64(setAttrIn.GID) + } + + // if (setAttrIn.Valid & fission.SetAttrInValidSize) != 0 { + // // TODO - need to adjust file size... + // } + + if (setAttrIn.Valid & fission.SetAttrInValidMTime) != 0 { + if (setAttrIn.Valid & fission.SetAttrInValidMTimeNow) != 0 { + inode.inodeHeadV1.ModificationTime = startTime + } else { + inode.inodeHeadV1.ModificationTime = time.Unix(int64(setAttrIn.MTimeSec), int64(setAttrIn.MTimeNSec)) + } + } + + inodeNumber = inode.inodeNumber + + flushInodesInSlice([]*inodeStruct{inode}) + + inodeLockRequest.unlockAll() + + setAttrOut = &fission.SetAttrOut{ + AttrValidSec: globals.fuseAttrValidDurationSec, + AttrValidNSec: globals.fuseAttrValidDurationNSec, + Dummy: 0, + // Attr to be filled in below + } + + err = doAttrFetch(inodeNumber, &setAttrOut.Attr) + + if nil == err { + errno = 0 + } else { + setAttrOut = nil + errno = syscall.EIO + } + return } @@ -3323,11 +3410,6 @@ func nsToUnixTime(ns uint64) (sec uint64, nsec uint32) { return } -func unixTimeToNs(sec uint64, nsec uint32) (ns uint64) { - ns = (sec * 1e9) + uint64(nsec) - return -} - func dirEntType(iLayoutInodeType uint8) (dirEntType uint32) { switch iLayoutInodeType { case ilayout.InodeTypeDir: From db9732affd7763cc2a7d01c89dd408ff6f0fcdc7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 6 Nov 2021 12:23:57 -0700 Subject: [PATCH 173/258] Added (*inodeStruct).{record|unmap}Extent() --- iclient/iclientpkg/inode.go | 335 +++++++++++++++++++++++++++++++++++- 1 file changed, 331 insertions(+), 4 deletions(-) diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 3ab3f9bc..ac5b49f3 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -133,10 +133,7 @@ func (inode *inodeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, ob layoutMapEntry, ok = inode.layoutMap[inode.putObjectNumber] if !ok { - layoutMapEntry = layoutMapEntryStruct{ - objectSize: 0, - bytesReferenced: 0, - } + logFatalf("inode.layoutMap[inode.putObjectNumber] returned !ok") } objectNumber = inode.putObjectNumber @@ -147,6 +144,9 @@ func (inode *inodeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, ob inode.layoutMap[inode.putObjectNumber] = layoutMapEntry + inode.superBlockInodeObjectSizeAdjustment += int64(len(nodeByteSlice)) + inode.superBlockInodeBytesReferencedAdjustment += int64(len(nodeByteSlice)) + inode.putObjectBuffer = append(inode.putObjectBuffer, nodeByteSlice...) err = nil @@ -651,6 +651,13 @@ func (inode *inodeStruct) ensurePutObjectIsActive() { inode.putObjectBuffer = make([]byte, 0) inode.superBlockInodeObjectCountAdjustment++ + + inode.layoutMap[inode.putObjectNumber] = layoutMapEntryStruct{ + objectSize: 0, + bytesReferenced: 0, + } + + inode.superBlockInodeObjectCountAdjustment++ } } @@ -784,3 +791,323 @@ func flushInodesInSlice(inodeSlice []*inodeStruct) { logFatalf("rpcPutInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesResponse) failed: %v", err) } } + +// recordExtent is called before appending to a fileInode's putObjectBuffer to update +// the ExtentMap to reference the about to be appended data. As it is possible that +// the extent being recorded overlaps with one or more existing extents, recordExtent() +// embeds a call to unmapExtent(). If the to be recorded extent is contiguous with an +// existing extent (both in terms of fileOffset and objectOffset), these will be +// combined. Thus, a sequentially written fileInode will have at most a single extent +// referencing each object. The fileInode's Size will not be adjusted as holes in the +// ExtentMap equate to "read as zero" and the fileInode's Size may have been established +// via a SetAttr (i.e. without actually writing data to extent the fileInode). +// +// The fileInode's layoutMap and, potentially, dereferencedObjectNumberArray will be +// updated to reflect the dereferenced extent. Similarly, the fileInode's pending +// updates for superBlockInode{BytesReferencedAdjustment|Object{Count|Size}} will +// be updated. +// +func (fileInode *inodeStruct) recordExtent(startingFileOffset uint64, length uint64) { + var ( + err error + extentMapEntryKeyV1 uint64 + extentMapEntryKeyV1AsKey sortedmap.Key + extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct + extentMapEntryValueV1AsValue sortedmap.Value + index int + layoutMapEntry layoutMapEntryStruct + ok bool + ) + + fileInode.ensurePutObjectIsActive() + + fileInode.unmapExtent(startingFileOffset, length) + + layoutMapEntry, ok = fileInode.layoutMap[fileInode.putObjectNumber] + if !ok { + logFatalf("fileInode.layoutMap[inode.putObjectNumber] returned !ok") + } + + layoutMapEntry.objectSize += length + layoutMapEntry.bytesReferenced += length + + fileInode.layoutMap[fileInode.putObjectNumber] = layoutMapEntry + + fileInode.superBlockInodeObjectSizeAdjustment += int64(length) + fileInode.superBlockInodeBytesReferencedAdjustment += int64(length) + + index, _, err = fileInode.payload.BisectLeft(startingFileOffset) + if nil != err { + logFatalf("fileInode.payload.BisectLeft(startingFileOffset) failed: %v", err) + } + + extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = fileInode.payload.GetByIndex(index) + if nil != err { + logFatalf("fileInode.payload.GetByIndex(index) failed: %v", err) + } + + if ok { + extentMapEntryKeyV1, ok = extentMapEntryKeyV1AsKey.(uint64) + if !ok { + logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") + } + extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) + if !ok { + logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") + } + + if (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) == startingFileOffset { + // The extent to be recorded starts right after an existing extent by fileOffset + + if extentMapEntryValueV1.ObjectNumber == fileInode.putObjectNumber { + // And the just prior existing extent was recorded to the current putObject + + if (extentMapEntryValueV1.ObjectOffset + extentMapEntryValueV1.Length) == uint64(len(fileInode.putObjectBuffer)) { + // And the extent to be recorded starts right after the just prior extent in the current putObject - so we can combine them + + extentMapEntryValueV1.Length += length + + ok, err = fileInode.payload.PatchByIndex(index, extentMapEntryValueV1) + if nil != err { + logFatalf("fileInode.payload.PatchByIndex(index, extentMapEntryValueV1) failed: %v", err) + } + if !ok { + logFatalf("fileInode.payload.PatchByIndex(index, extentMapEntryValueV1) returned !ok") + } + + // So we need not fall through to the un-optimized path + + return + } + } + } + } + + // Un-optimized path - just record the new extent + + extentMapEntryValueV1 = &ilayout.ExtentMapEntryValueV1Struct{ + Length: length, + ObjectNumber: fileInode.putObjectNumber, + ObjectOffset: uint64(len(fileInode.putObjectBuffer)), + } + + ok, err = fileInode.payload.Put(startingFileOffset, extentMapEntryValueV1) + if nil != err { + logFatalf("fileInode.payload.Put(startingFileOffset, extentMapEntryValueV1) failed: %v", err) + } + if !ok { + logFatalf("fileInode.payload.Put(startingFileOffset, extentMapEntryValueV1) returned !ok") + } +} + +// unmapExtent releases references for a range of bytes in a fileInode. If length +// is zero, the unmapped extent is assumed to be to the end of the fileInode (i.e. +// a truncate opereration). The fileInode's Size will not be adjusted as holes in +// the ExtentMap equate to "read as zero" and the fileInode's Size may have been +// established via SetAttr (i.e. without actually writing data to extend the fileInode). +// +// The fileInode's layoutMap and, potentially, dereferencedObjectNumberArray will be +// updated to reflect the dereferenced extent. Similarly, the fileInode's pending +// updates for superBlockInode{BytesReferencedAdjustment|Object{Count|Size}} will +// be updated. +// +func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint64) { + var ( + err error + extentLengthToTrim uint64 + extentMapEntryKeyV1 uint64 + extentMapEntryKeyV1AsKey sortedmap.Key + extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct + extentMapEntryValueV1AsValue sortedmap.Value + found bool + index int + layoutMapEntry layoutMapEntryStruct + ok bool + subsequentFileOffset uint64 + ) + + index, found, err = fileInode.payload.BisectLeft(startingFileOffset) + if nil != err { + logFatalf("fileInode.payload.BisectLeft(startingFileOffset) failed: %v", err) + } + + if !found { + // See if there is an extent just to the left of the extent to unmap + + extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = fileInode.payload.GetByIndex(index) + if nil != err { + logFatalf("fileInode.payload.GetByIndex(index) failed: %v", err) + } + + if ok { + // Potentially trim the extent just to the left of the extent to unmap + + extentMapEntryKeyV1, ok = extentMapEntryKeyV1AsKey.(uint64) + if !ok { + logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") + } + extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) + if !ok { + logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") + } + + if extentMapEntryKeyV1+extentMapEntryValueV1.Length > startingFileOffset { + extentLengthToTrim = (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) - startingFileOffset + + layoutMapEntry, ok = fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] + if !ok { + logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") + } + + if extentLengthToTrim > layoutMapEntry.bytesReferenced { + logFatalf("extentLengthToTrim > layoutMapEntry.bytesReferenced") + } else if (extentLengthToTrim == layoutMapEntry.bytesReferenced) && (extentMapEntryValueV1.ObjectNumber != fileInode.putObjectNumber) { + delete(fileInode.layoutMap, extentMapEntryValueV1.ObjectNumber) + + fileInode.superBlockInodeObjectCountAdjustment-- + fileInode.superBlockInodeObjectSizeAdjustment -= int64(layoutMapEntry.objectSize) + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) + + fileInode.dereferencedObjectNumberArray = append(fileInode.dereferencedObjectNumberArray, extentMapEntryValueV1.ObjectNumber) + } else { + layoutMapEntry.bytesReferenced -= extentLengthToTrim + + fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry + + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) + } + + ok, err = fileInode.payload.PatchByIndex(index, extentMapEntryValueV1) + if nil != err { + logFatalf("fileInode.payload.GetByIndex(index) failed: %v", err) + } + if !ok { + logFatalf("fileInode.payload.GetByIndex(index) returned !ok") + } + } + + // Adjust index to start at the next extent that might overlap with the extent to unmap + + index++ + } + } + + // Now delete or trim existing extents that overlap with the extent to unmap + + subsequentFileOffset = startingFileOffset + length + + for { + extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = fileInode.payload.GetByIndex(index) + if nil != err { + logFatalf("fileInode.payload.GetByIndex(index) failed: %v", err) + } + + if !ok { + // We reached the end of the ExtentMap so we are done + + return + } + + extentMapEntryKeyV1, ok = extentMapEntryKeyV1AsKey.(uint64) + if !ok { + logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") + } + + if subsequentFileOffset <= extentMapEntryKeyV1 { + // We reached an extent that starts after the extent to unmap so we are done + + return + } + + extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) + if !ok { + logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") + } + + if (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) <= subsequentFileOffset { + // Trim this extent on the left + + extentLengthToTrim = subsequentFileOffset - extentMapEntryKeyV1 + + layoutMapEntry, ok = fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] + if !ok { + logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") + } + + if extentLengthToTrim > layoutMapEntry.bytesReferenced { + logFatalf("extentLengthToTrim > layoutMapEntry.bytesReferenced") + } else if (extentLengthToTrim == layoutMapEntry.bytesReferenced) && (extentMapEntryValueV1.ObjectNumber != fileInode.putObjectNumber) { + delete(fileInode.layoutMap, extentMapEntryValueV1.ObjectNumber) + + fileInode.superBlockInodeObjectCountAdjustment-- + fileInode.superBlockInodeObjectSizeAdjustment -= int64(layoutMapEntry.objectSize) + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) + + fileInode.dereferencedObjectNumberArray = append(fileInode.dereferencedObjectNumberArray, extentMapEntryValueV1.ObjectNumber) + } else { + layoutMapEntry.bytesReferenced -= extentLengthToTrim + + fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry + + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) + } + + ok, err = fileInode.payload.DeleteByIndex(index) + if nil != err { + logFatalf("fileInode.payload.DeleteByIndex(index) failed: %v", err) + } + if !ok { + logFatalf("fileInode.payload.DeleteByIndex(index) returned !ok") + } + + extentMapEntryValueV1.Length -= extentLengthToTrim + + ok, err = fileInode.payload.Put(subsequentFileOffset, extentMapEntryValueV1) + if nil != err { + logFatalf("fileInode.payload.Put(subsequentFileOffset, extentMapEntryValueV1) failed: %v", err) + } + if !ok { + logFatalf("fileInode.payload.Put(subsequentFileOffset, extentMapEntryValueV1) returned !ok") + } + + // We know the next loop would find the trimmed extent that starts after the extnet to unmap so we are done + + return + } + + // This extent to be totally unmapped + + layoutMapEntry, ok = fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] + if !ok { + logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") + } + + if extentMapEntryValueV1.Length > layoutMapEntry.bytesReferenced { + logFatalf("extentMapEntryValueV1.Length > layoutMapEntry.bytesReferenced") + } else if (extentMapEntryValueV1.Length == layoutMapEntry.bytesReferenced) && (extentMapEntryValueV1.ObjectNumber != fileInode.putObjectNumber) { + delete(fileInode.layoutMap, extentMapEntryValueV1.ObjectNumber) + + fileInode.superBlockInodeObjectCountAdjustment-- + fileInode.superBlockInodeObjectSizeAdjustment -= int64(layoutMapEntry.objectSize) + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentMapEntryValueV1.Length) + + fileInode.dereferencedObjectNumberArray = append(fileInode.dereferencedObjectNumberArray, extentMapEntryValueV1.ObjectNumber) + } else { + layoutMapEntry.bytesReferenced -= extentMapEntryValueV1.Length + + fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry + + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentMapEntryValueV1.Length) + } + + ok, err = fileInode.payload.DeleteByIndex(index) + if nil != err { + logFatalf("fileInode.payload.DeleteByIndex(index) failed: %v", err) + } + if !ok { + logFatalf("fileInode.payload.DeleteByIndex(index) returned !ok") + } + + // Now loop back to fetch the next existing extent + } +} From 488282ea1218bdee2a73bc45047a11b123cb201a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 6 Nov 2021 18:08:40 -0700 Subject: [PATCH 174/258] Got DoRead() & DoWrite() working from/to the write-back cache (i.e. no flushing yet) --- iclient/iclientpkg/fission.go | 106 +++++++++++++++++++++++++++++----- iclient/iclientpkg/globals.go | 3 + iclient/iclientpkg/inode.go | 32 +++++++--- 3 files changed, 118 insertions(+), 23 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index fa0e0ebe..677ffbac 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -371,9 +371,19 @@ Retry: inode.inodeHeadV1.GroupID = uint64(setAttrIn.GID) } - // if (setAttrIn.Valid & fission.SetAttrInValidSize) != 0 { - // // TODO - need to adjust file size... - // } + if (setAttrIn.Valid & fission.SetAttrInValidSize) != 0 { + if setAttrIn.Size < inode.inodeHeadV1.Size { + if inode.payload == nil { + err = inode.oldPayload() + if nil != err { + logFatalf("inode.oldPayload() failed: %v", err) + } + } + + inode.unmapExtent(setAttrIn.Size, 0) + } + inode.inodeHeadV1.Size = setAttrIn.Size + } if (setAttrIn.Valid & fission.SetAttrInValidMTime) != 0 { if (setAttrIn.Valid & fission.SetAttrInValidMTimeNow) != 0 { @@ -1457,7 +1467,16 @@ Retry: return } - // TODO: Need to truncate file if openIn.Flags contains fission.FOpenRequestTRUNC + if (openIn.Flags & fission.FOpenRequestTRUNC) == fission.FOpenRequestTRUNC { + if inode.payload == nil { + err = inode.oldPayload() + if nil != err { + logFatalf("inode.oldPayload() failed: %v", err) + } + } + + inode.unmapExtent(0, 0) + } adjustInodeTableEntryOpenCountRequest = &imgrpkg.AdjustInodeTableEntryOpenCountRequestStruct{ MountID: globals.mountID, @@ -1522,6 +1541,9 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R defer func() { globals.stats.DoReadUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + if (errno == 0) && (readOut != nil) { + globals.stats.DoReadBytes.Add(uint64(len(readOut.Data))) + } }() obtainExclusiveLock = false @@ -1535,7 +1557,7 @@ Retry: goto Retry } - openHandle = lookupOpenHandleByInodeNumber(readIn.FH) + openHandle = lookupOpenHandleByFissionFH(readIn.FH) if nil == openHandle { inodeLockRequest.unlockAll() readOut = nil @@ -1638,7 +1660,7 @@ Retry: readPlan = make([]*ilayout.ExtentMapEntryValueV1Struct, 0, 2*(extentMapEntryIndexV1Max-extentMapEntryIndexV1Min)) - for extentMapEntryIndexV1 = extentMapEntryIndexV1Min; extentMapEntryIndexV1 < extentMapEntryIndexV1Max; extentMapEntryIndexV1++ { + for extentMapEntryIndexV1 = extentMapEntryIndexV1Min; extentMapEntryIndexV1 <= extentMapEntryIndexV1Max; extentMapEntryIndexV1++ { extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, _, err = inode.payload.GetByIndex(extentMapEntryIndexV1) if nil != err { logFatal(err) @@ -1712,14 +1734,15 @@ Retry: for _, readPlanEntry = range readPlan { switch readPlanEntry.ObjectNumber { case 0: - readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length, readPlanEntry.Length)...) + readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length)...) case inode.putObjectNumber: + // Note that if putObject is inactive, case 0: will hae already matched readPlanEntry.ObjectNumber readOut.Data = append(readOut.Data, inode.putObjectBuffer[readPlanEntry.ObjectOffset:(readPlanEntry.ObjectOffset+readPlanEntry.Length)]...) default: // TODO - need to actually read from the cache (in a coherent/cooperative way) // TODO - need to handle case where extent crosses cache line boundary // TODO - but for now, we will simply append zeroes - readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length, readPlanEntry.Length)...) // TODO + readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length)...) // TODO } } @@ -1734,6 +1757,9 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission err error inode *inodeStruct inodeLockRequest *inodeLockRequestStruct + newSize uint64 + offset uint64 + oldSize uint64 openHandle *openHandleStruct startTime time.Time = time.Now() ) @@ -1745,8 +1771,17 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission defer func() { globals.stats.DoWriteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + if (errno == 0) && (writeOut != nil) { + globals.stats.DoWriteBytes.Add(uint64(writeOut.Size)) + } }() + if uint64(writeIn.Size) != uint64(len(writeIn.Data)) { + writeOut = nil + errno = syscall.EIO + return + } + Retry: inodeLockRequest = newLockRequest() inodeLockRequest.inodeNumber = inHeader.NodeID @@ -1756,7 +1791,7 @@ Retry: goto Retry } - openHandle = lookupOpenHandleByInodeNumber(writeIn.FH) + openHandle = lookupOpenHandleByFissionFH(writeIn.FH) if nil == openHandle { inodeLockRequest.unlockAll() writeOut = nil @@ -1811,16 +1846,49 @@ Retry: logFatalf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } - // TODO - need to patch ExtentMap - // TODO - need to append to inode.putObjectBuffer - // TODO - need to detect combinable ExtentMap entries + if inode.payload == nil { + err = inode.oldPayload() + if nil != err { + logFatalf("inode.oldPayload() failed: %v", err) + } + } + + oldSize = inode.inodeHeadV1.Size + + if openHandle.fissionFlagsAppend { + offset = oldSize + newSize = oldSize + uint64(len(writeIn.Data)) + } else { + offset = writeIn.Offset + newSize = writeIn.Offset + uint64(len(writeIn.Data)) + if newSize < oldSize { + newSize = oldSize + } + } + + inode.dirty = true + + inode.inodeHeadV1.ModificationTime = startTime + + inode.inodeHeadV1.Size = newSize + + inode.recordExtent(offset, uint64(len(writeIn.Data))) + + inode.putObjectBuffer = append(inode.putObjectBuffer, writeIn.Data...) + // TODO - need to trigger a flush if len(payload.putObjectBuffer) >= globals.config.FileFlushTriggerSize // TODO - need to (possibly) trigger new timer after globals.config.FileFlushTriggerDuration + // TODO - for now, we could just flush every write (but DoRead() will not read flushed data yet) + // flushInodesInSlice([]*inodeStruct{inode}) inodeLockRequest.unlockAll() - writeOut = nil // TODO - errno = syscall.ENOSYS + writeOut = &fission.WriteOut{ + Size: writeIn.Size, + Padding: 0, + } + + errno = 0 return } @@ -2934,9 +3002,15 @@ Retry: logFatalf("fileInode.inodeHeadV1.InodeType != ilayout.InodeTypeFile") } - // TODO: Need to truncate file (i.e. assume createIn.Flags contains fission.FOpenRequestTRUNC) - } else { // dirInode.payload.GetByKey(string(createIn.Name[:])) returned !ok + if fileInode.payload == nil { + err = fileInode.oldPayload() + if nil != err { + logFatalf("fileInode.oldPayload() failed: %v", err) + } + } + fileInode.unmapExtent(0, 0) + } else { // dirInode.payload.GetByKey(string(createIn.Name[:])) returned !ok fileInodeNumber = fetchNonce() fileInode = &inodeStruct{ diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 0215574d..eb77d89e 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -215,6 +215,9 @@ type statsStruct struct { DoReadDirPlusUsecs bucketstats.BucketLog2Round // (*globalsStruct)DoReadDirPlus() DoRename2Usecs bucketstats.BucketLog2Round // (*globalsStruct)DoRename2() DoLSeekUsecs bucketstats.BucketLog2Round // (*globalsStruct)DoLSeek() + + DoReadBytes bucketstats.BucketLog2Round // (*globalsStruct)DoRead() + DoWriteBytes bucketstats.BucketLog2Round // (*globalsStruct)DoWrite() } type globalsStruct struct { diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index ac5b49f3..05c748f7 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -577,17 +577,35 @@ func (inode *inodeStruct) convertInodeHeadV1LayoutToLayoutMap() { func (inode *inodeStruct) convertLayoutMapToInodeHeadV1Layout() { var ( - ilayoutInodeHeadV1LayoutIndex uint64 = 0 - layoutMapEntry layoutMapEntryStruct - objectNumber uint64 + layoutMapEntry layoutMapEntryStruct + layoutMapEntryToDeleteList []uint64 + objectNumber uint64 ) - inode.inodeHeadV1.Layout = make([]ilayout.InodeHeadLayoutEntryV1Struct, len(inode.layoutMap)) + inode.inodeHeadV1.Layout = make([]ilayout.InodeHeadLayoutEntryV1Struct, 0, len(inode.layoutMap)) for objectNumber, layoutMapEntry = range inode.layoutMap { - inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectNumber = objectNumber - inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].ObjectSize = layoutMapEntry.objectSize - inode.inodeHeadV1.Layout[ilayoutInodeHeadV1LayoutIndex].BytesReferenced = layoutMapEntry.bytesReferenced + if layoutMapEntry.objectSize == 0 { + if layoutMapEntry.bytesReferenced != 0 { + logFatalf("(layoutMapEntry.objectSize == 0) && (layoutMapEntry.bytesReferenced != 0)") + } + + layoutMapEntryToDeleteList = append(layoutMapEntryToDeleteList, objectNumber) + } else { + if layoutMapEntry.bytesReferenced == 0 { + logFatalf("(layoutMapEntry.objectSize != 0) && (layoutMapEntry.bytesReferenced == 0)") + } + + inode.inodeHeadV1.Layout = append(inode.inodeHeadV1.Layout, ilayout.InodeHeadLayoutEntryV1Struct{ + ObjectNumber: objectNumber, + ObjectSize: layoutMapEntry.objectSize, + BytesReferenced: layoutMapEntry.bytesReferenced, + }) + } + } + + for _, objectNumber = range layoutMapEntryToDeleteList { + delete(inode.layoutMap, objectNumber) } } From f404c00b07e2215dc4f049939883686309569137 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 7 Nov 2021 12:33:30 -0800 Subject: [PATCH 175/258] Picked up fission API change allowing specification of both maxRead and maxWrite --- go.mod | 2 +- go.sum | 2 ++ iclient/dev.conf | 1 + iclient/iclient.conf | 1 + iclient/iclientpkg/api.go | 1 + iclient/iclientpkg/fission.go | 1 + iclient/iclientpkg/globals.go | 6 ++++++ 7 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4ac2b69d..33adc312 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 - github.com/NVIDIA/fission v0.0.0-20211105235624-b9d1d59bb6c2 + github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 github.com/ansel1/merry v1.6.1 github.com/creachadair/cityhash v0.1.0 diff --git a/go.sum b/go.sum index 5710ebd6..99f600b6 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066 h1:K/8Os/A5UFB6mJiQ github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= github.com/NVIDIA/fission v0.0.0-20211105235624-b9d1d59bb6c2 h1:aeeSFyX1qAyT7AgMJcAqj2f3mpoKJC6jvhqkl14qtr4= github.com/NVIDIA/fission v0.0.0-20211105235624-b9d1d59bb6c2/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= +github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b h1:iJ0ZiDsn950iZV10cAAxltZxmJleVa5Cfza3BS4WJ24= +github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/iclient/dev.conf b/iclient/dev.conf index 11b20fbf..2b1f2fc6 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -8,6 +8,7 @@ FUSEBlockSize: 512 FUSEAllowOther: true FUSEMaxBackground: 1000 FUSECongestionThreshhold: 0 +FUSEMaxRead: 131076 FUSEMaxWrite: 131076 FUSEEntryValidDuration: 250ms FUSEAttrValidDuration: 250ms diff --git a/iclient/iclient.conf b/iclient/iclient.conf index e28731c7..874415a4 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -8,6 +8,7 @@ FUSEBlockSize: 512 FUSEAllowOther: true FUSEMaxBackground: 1000 FUSECongestionThreshhold: 0 +FUSEMaxRead: 131076 FUSEMaxWrite: 131076 FUSEEntryValidDuration: 250ms FUSEAttrValidDuration: 250ms diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index adcd8cae..b6ab2d36 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -14,6 +14,7 @@ // FUSEAllowOther: true // FUSEMaxBackground: 1000 // FUSECongestionThreshhold: 0 +// FUSEMaxRead: 131076 // FUSEMaxWrite: 131076 // FUSEEntryValidDuration: 250ms // FUSEAttrValidDuration: 250ms diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 677ffbac..8ea5773a 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -44,6 +44,7 @@ func performMountFUSE() (err error) { globals.config.VolumeName, globals.config.MountPointDirPath, fuseSubtype, + globals.config.FUSEMaxRead, globals.config.FUSEMaxWrite, fuseDefaultPermissions, globals.config.FUSEAllowOther, diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index eb77d89e..a9834e3c 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -29,6 +29,7 @@ type configStruct struct { FUSEAllowOther bool FUSEMaxBackground uint16 FUSECongestionThreshhold uint16 + FUSEMaxRead uint32 FUSEMaxWrite uint32 FUSEEntryValidDuration time.Duration FUSEAttrValidDuration time.Duration @@ -295,6 +296,10 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err if nil != err { logFatal(err) } + globals.config.FUSEMaxRead, err = confMap.FetchOptionValueUint32("ICLIENT", "FUSEMaxRead") + if nil != err { + logFatal(err) + } globals.config.FUSEMaxWrite, err = confMap.FetchOptionValueUint32("ICLIENT", "FUSEMaxWrite") if nil != err { logFatal(err) @@ -512,6 +517,7 @@ func uninitializeGlobals() (err error) { globals.config.FUSEAllowOther = false globals.config.FUSEMaxBackground = 0 globals.config.FUSECongestionThreshhold = 0 + globals.config.FUSEMaxRead = 0 globals.config.FUSEMaxWrite = 0 globals.config.FUSEEntryValidDuration = time.Duration(0) globals.config.FUSEAttrValidDuration = time.Duration(0) From 1b081e4b21bac7a91c9078b7c666b9ecbd5eabac Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 8 Nov 2021 10:06:18 -0800 Subject: [PATCH 176/258] Finished ReadCache... and, thus, DoRead() is now complete --- iclient/iclientpkg/fission.go | 192 +++++++++++++++++++++++++++++----- iclient/iclientpkg/globals.go | 78 +++++++------- iclient/iclientpkg/inode.go | 13 ++- 3 files changed, 216 insertions(+), 67 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 8ea5773a..94131e3e 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1511,24 +1511,32 @@ Retry: func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.ReadIn) (readOut *fission.ReadOut, errno syscall.Errno) { var ( - curOffset uint64 - err error - extentMapEntryIndexV1 int - extentMapEntryIndexV1Max int // Entry entry at or just after where readIn.Offset+readIn.Size may reside - extentMapEntryIndexV1Min int // First entry at or just before where readIn.Offset may reside - extentMapEntryKeyV1 uint64 - extentMapEntryKeyV1AsKey sortedmap.Key - extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct - extentMapEntryValueV1AsValue sortedmap.Value - inode *inodeStruct - inodeLockRequest *inodeLockRequestStruct - obtainExclusiveLock bool - ok bool - openHandle *openHandleStruct - readPlan []*ilayout.ExtentMapEntryValueV1Struct - readPlanEntry *ilayout.ExtentMapEntryValueV1Struct // If .ObjectNumber == 0, .ObjectOffset is ignored... .Length is the number of zero fill bytes - remainingSize uint64 - startTime time.Time = time.Now() + curOffset uint64 + err error + extentMapEntryIndexV1 int + extentMapEntryIndexV1Max int // Entry entry at or just after where readIn.Offset+readIn.Size may reside + extentMapEntryIndexV1Min int // First entry at or just before where readIn.Offset may reside + extentMapEntryKeyV1 uint64 + extentMapEntryKeyV1AsKey sortedmap.Key + extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct + extentMapEntryValueV1AsValue sortedmap.Value + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + obtainExclusiveLock bool + ok bool + openHandle *openHandleStruct + readCacheKey readCacheKeyStruct + readCacheLine *readCacheLineStruct + readCacheLineBuf []byte + readCacheLineBufLengthAvailableToConsume uint64 + readCacheLineOffset uint64 + readCacheLineToEvict *readCacheLineStruct + readCacheLineToEvictListElement *list.Element + readCacheLineWG *sync.WaitGroup + readPlan []*ilayout.ExtentMapEntryValueV1Struct + readPlanEntry *ilayout.ExtentMapEntryValueV1Struct // If .ObjectNumber == 0, .ObjectOffset is ignored... .Length is the number of zero fill bytes + remainingSize uint64 + startTime time.Time = time.Now() ) logTracef("==> DoRead(inHeader: %+v, readIn: %+v)", inHeader, readIn) @@ -1732,18 +1740,154 @@ Retry: }) } + // Now process readPlan + for _, readPlanEntry = range readPlan { switch readPlanEntry.ObjectNumber { case 0: readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length)...) case inode.putObjectNumber: - // Note that if putObject is inactive, case 0: will hae already matched readPlanEntry.ObjectNumber + // Note that if putObject is inactive, case 0: will have already matched readPlanEntry.ObjectNumber readOut.Data = append(readOut.Data, inode.putObjectBuffer[readPlanEntry.ObjectOffset:(readPlanEntry.ObjectOffset+readPlanEntry.Length)]...) default: - // TODO - need to actually read from the cache (in a coherent/cooperative way) - // TODO - need to handle case where extent crosses cache line boundary - // TODO - but for now, we will simply append zeroes - readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length)...) // TODO + for readPlanEntry.Length > 0 { + readCacheKey = readCacheKeyStruct{ + objectNumber: readPlanEntry.ObjectNumber, + lineNumber: readPlanEntry.ObjectOffset / globals.config.ReadCacheLineSize, + } + readCacheLineOffset = readPlanEntry.ObjectOffset - (readCacheKey.lineNumber * globals.config.ReadCacheLineSize) + + globals.Lock() + + readCacheLine, ok = globals.readCacheMap[readCacheKey] + + if ok { + // readCacheLine is in globals.readCacheMap but may be being filled + + globals.readCacheLRU.MoveToBack(readCacheLine.listElement) + + if readCacheLine.wg == nil { + // readCacheLine is already filled... + + readCacheLineBuf = readCacheLine.buf + + globals.Unlock() + } else { + // readCacheLine is being filled... so just wait for it + + readCacheLineWG = readCacheLine.wg + + globals.Unlock() + + readCacheLineWG.Wait() + + // If readCacheLine fill failed... we must exit + + globals.Lock() + readCacheLineBuf = readCacheLine.buf + globals.Unlock() + + if nil == readCacheLineBuf { + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.EIO + return + } + } + } else { + // readCacheLine is absent from globals.readCacheMap... so put it there and fill it + + readCacheLineWG = &sync.WaitGroup{} + + readCacheLine = &readCacheLineStruct{ + wg: readCacheLineWG, + key: readCacheKey, + buf: nil, + } + + readCacheLine.wg.Add(1) + + readCacheLine.listElement = globals.readCacheLRU.PushBack(readCacheLine) + + // Need to evict LRU'd readCacheLine if globals.config.ReadCacheLineCountMax is exceeded + + for globals.config.ReadCacheLineCountMax < uint64(globals.readCacheLRU.Len()) { + readCacheLineToEvictListElement = globals.readCacheLRU.Front() + readCacheLineToEvict, ok = readCacheLineToEvictListElement.Value.(*readCacheLineStruct) + if !ok { + logFatalf("readCacheLineToEvictListElement.Value.(*readCacheLineStruct) returned !ok") + } + + delete(globals.readCacheMap, readCacheLineToEvict.key) + _ = globals.readCacheLRU.Remove(readCacheLineToEvict.listElement) + } + + globals.Unlock() + + readCacheLineBuf, err = objectGETRange( + readCacheLine.key.objectNumber, + readCacheLine.key.lineNumber&globals.config.ReadCacheLineSize, + globals.config.ReadCacheLineSize) + + if nil != err { + // readCacheLine fill failed... so tell others and exit + + globals.Lock() + + delete(globals.readCacheMap, readCacheKey) + _ = globals.readCacheLRU.Remove(readCacheLine.listElement) + + readCacheLine.wg = nil + + readCacheLineWG.Done() + + inodeLockRequest.unlockAll() + + readOut = nil + errno = syscall.EIO + return + } + + // readCacheLine fill succeeded... so tell others and continue + + globals.Lock() + + readCacheLine.wg = nil + readCacheLine.buf = readCacheLineBuf + + readCacheLineWG.Done() + + globals.Unlock() + } + + // If we make it here, we have a non-nil readCacheLineBuf matching readCacheKey + + if readCacheLineOffset >= uint64(len(readCacheLineBuf)) { + // readCacheLineBuf unexpectedly too short... we must exit + + inodeLockRequest.unlockAll() + readOut = nil + errno = syscall.EIO + return + } + + readCacheLineBufLengthAvailableToConsume = uint64(len(readCacheLineBuf)) - readCacheLineOffset + + if readPlanEntry.Length > readCacheLineBufLengthAvailableToConsume { + // Consume tail of readCacheLineBuf starting at readCacheLineOffset and continue looping + + readOut.Data = append(readOut.Data, readCacheLineBuf[readCacheLineOffset:]...) + + readPlanEntry.Length -= readCacheLineBufLengthAvailableToConsume + readPlanEntry.ObjectOffset += readCacheLineBufLengthAvailableToConsume + } else { + // Consume only the portion of readCacheLineBuf needed and trigger loop exit + + readOut.Data = append(readOut.Data, readCacheLineBuf[readCacheLineOffset:(readCacheLineOffset+readPlanEntry.Length)]...) + + readPlanEntry.Length = 0 + } + } } } @@ -1880,7 +2024,7 @@ Retry: // TODO - need to trigger a flush if len(payload.putObjectBuffer) >= globals.config.FileFlushTriggerSize // TODO - need to (possibly) trigger new timer after globals.config.FileFlushTriggerDuration // TODO - for now, we could just flush every write (but DoRead() will not read flushed data yet) - // flushInodesInSlice([]*inodeStruct{inode}) + flushInodesInSlice([]*inodeStruct{inode}) inodeLockRequest.unlockAll() diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index a9834e3c..f706eaac 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -104,7 +104,7 @@ type inodeStruct struct { dirty bool // markedForDelete bool // If true, remove from globalsStruct.inodeTable upon last dereference leaseState inodeLeaseStateType // - listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU + listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive}LeaseLRU heldList *list.List // List of granted inodeHeldLockStruct's requestList *list.List // List of pending inodeLockRequestStruct's inodeHeadV1 *ilayout.InodeHeadV1Struct // @@ -147,10 +147,10 @@ type readCacheKeyStruct struct { } type readCacheLineStruct struct { - sync.WaitGroup // Used by those needing to block while a prior accessor reads in .buf - key readCacheKeyStruct // - listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive|LeaseLRU - buf []byte // == nil if being read in + wg *sync.WaitGroup // If != nil, used by those needing to block while a prior accessor reads in .buf + key readCacheKeyStruct // + listElement *list.Element // Maintains position in globalsStruct.readCacheLRU + buf []byte // == nil if being read in } type statsStruct struct { @@ -222,39 +222,39 @@ type statsStruct struct { } type globalsStruct struct { - sync.Mutex // - config configStruct // - fuseEntryValidDurationSec uint64 // - fuseEntryValidDurationNSec uint32 // - fuseAttrValidDurationSec uint64 // - fuseAttrValidDurationNSec uint32 // - logFile *os.File // == nil if config.LogFilePath == "" - retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" - httpClient *http.Client // - swiftRetryDelay []swiftRetryDelayElementStruct // - swiftAuthInString string // - swiftAuthWaitGroup *sync.WaitGroup // != nil if updateAuthTokenAndStorageURL() is active - swiftAuthToken string // - swiftStorageURL string // - retryRPCClientConfig *retryrpc.ClientConfig // - retryRPCClient *retryrpc.Client // - mountID string // - fissionErrChan chan error // - nonceWaitGroup *sync.WaitGroup // != nil if rpcFetchNonceRange() already underway - nextNonce uint64 // - noncesRemaining uint64 // - readCacheMap map[readCacheKeyStruct]readCacheLineStruct // - readCacheLRU *list.List // LRU-ordered list of readCacheLineStruct.listElement's - inodeTable map[uint64]*inodeStruct // - inodePayloadCache sortedmap.BPlusTreeCache // - openHandleMapByInodeNumber map[uint64]*openHandleStruct // Key == openHandleStruct.inodeNumber - openHandleMapByFissionFH map[uint64]*openHandleStruct // Key == openHandleStruct.fissionFH - sharedLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted - exclusiveLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted - httpServer *http.Server // - httpServerWG sync.WaitGroup // - stats *statsStruct // - fissionVolume fission.Volume // + sync.Mutex // + config configStruct // + fuseEntryValidDurationSec uint64 // + fuseEntryValidDurationNSec uint32 // + fuseAttrValidDurationSec uint64 // + fuseAttrValidDurationNSec uint32 // + logFile *os.File // == nil if config.LogFilePath == "" + retryRPCCACertPEM []byte // == nil if config.RetryRPCCACertFilePath == "" + httpClient *http.Client // + swiftRetryDelay []swiftRetryDelayElementStruct // + swiftAuthInString string // + swiftAuthWaitGroup *sync.WaitGroup // != nil if updateAuthTokenAndStorageURL() is active + swiftAuthToken string // + swiftStorageURL string // + retryRPCClientConfig *retryrpc.ClientConfig // + retryRPCClient *retryrpc.Client // + mountID string // + fissionErrChan chan error // + nonceWaitGroup *sync.WaitGroup // != nil if rpcFetchNonceRange() already underway + nextNonce uint64 // + noncesRemaining uint64 // + readCacheMap map[readCacheKeyStruct]*readCacheLineStruct // + readCacheLRU *list.List // LRU-ordered list of readCacheLineStruct.listElement's + inodeTable map[uint64]*inodeStruct // + inodePayloadCache sortedmap.BPlusTreeCache // + openHandleMapByInodeNumber map[uint64]*openHandleStruct // Key == openHandleStruct.inodeNumber + openHandleMapByFissionFH map[uint64]*openHandleStruct // Key == openHandleStruct.fissionFH + sharedLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted + exclusiveLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted + httpServer *http.Server // + httpServerWG sync.WaitGroup // + stats *statsStruct // + fissionVolume fission.Volume // } var globals globalsStruct @@ -492,7 +492,7 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err globals.nextNonce = 0 globals.noncesRemaining = 0 - globals.readCacheMap = make(map[readCacheKeyStruct]readCacheLineStruct) + globals.readCacheMap = make(map[readCacheKeyStruct]*readCacheLineStruct) globals.readCacheLRU = list.New() globals.inodeTable = make(map[uint64]*inodeStruct) diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 05c748f7..e24bb5c0 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -920,9 +920,10 @@ func (fileInode *inodeStruct) recordExtent(startingFileOffset uint64, length uin // unmapExtent releases references for a range of bytes in a fileInode. If length // is zero, the unmapped extent is assumed to be to the end of the fileInode (i.e. -// a truncate opereration). The fileInode's Size will not be adjusted as holes in -// the ExtentMap equate to "read as zero" and the fileInode's Size may have been -// established via SetAttr (i.e. without actually writing data to extend the fileInode). +// a truncate opereration) as indicated by the fileInode's current Size. The +// fileInode's Size will not be adjusted as holes in the ExtentMap equate to +// "read as zero" and the fileInode's Size may have been established via SetAttr +// (i.e. without actually writing data to extend the fileInode). // // The fileInode's layoutMap and, potentially, dereferencedObjectNumberArray will be // updated to reflect the dereferenced extent. Similarly, the fileInode's pending @@ -944,6 +945,10 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint subsequentFileOffset uint64 ) + if length == 0 { + length = fileInode.inodeHeadV1.Size - startingFileOffset + } + index, found, err = fileInode.payload.BisectLeft(startingFileOffset) if nil != err { logFatalf("fileInode.payload.BisectLeft(startingFileOffset) failed: %v", err) @@ -1042,7 +1047,7 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") } - if (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) <= subsequentFileOffset { + if (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) > subsequentFileOffset { // Trim this extent on the left extentLengthToTrim = subsequentFileOffset - extentMapEntryKeyV1 From 985df10253772414a9b39f5710a7d829a5e5cb3e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 13 Nov 2021 11:53:07 -0800 Subject: [PATCH 177/258] [WIP] coding up the FileInode async flush logic... ...need to figure out how to handle timer pop case (i.e. we don't hold an ExclusiveLock !!!) --- go.mod | 2 +- go.sum | 4 +++ iclient/iclientpkg/fission.go | 6 +++- iclient/iclientpkg/globals.go | 21 +++++++----- iclient/iclientpkg/inode.go | 62 +++++++++++++++++++++++++++++++++++ iclient/iclientpkg/lease.go | 1 + 6 files changed, 85 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 33adc312..dad94ac5 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 + golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e ) require ( diff --git a/go.sum b/go.sum index 99f600b6..1d589ab2 100644 --- a/go.sum +++ b/go.sum @@ -630,6 +630,10 @@ golang.org/x/sys v0.0.0-20211022215931-8e5104632af7 h1:e2q1CMOFXDvurT2sa2yhJAkuA golang.org/x/sys v0.0.0-20211022215931-8e5104632af7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42 h1:G2DDmludOQZoWbpCr7OKDxnl478ZBGMcOhrv+ooX/Q4= +golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e h1:zeJt6jBtVDK23XK9QXcmG0FvO0elikp0dYZQZOeL1y0= +golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 94131e3e..e4863567 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -596,6 +596,7 @@ Retry: superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, dereferencedObjectNumberArray: make([]uint64, 0), + flusherTrigger: nil, putObjectNumber: 0, putObjectBuffer: nil, } @@ -798,6 +799,7 @@ Retry: superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, dereferencedObjectNumberArray: make([]uint64, 0), + flusherTrigger: nil, putObjectNumber: 0, putObjectBuffer: nil, } @@ -1998,6 +2000,8 @@ Retry: } } + // TODO - if this write would exceed globals.config.FileFlushTriggerSize, flush first + oldSize = inode.inodeHeadV1.Size if openHandle.fissionFlagsAppend { @@ -2021,7 +2025,6 @@ Retry: inode.putObjectBuffer = append(inode.putObjectBuffer, writeIn.Data...) - // TODO - need to trigger a flush if len(payload.putObjectBuffer) >= globals.config.FileFlushTriggerSize // TODO - need to (possibly) trigger new timer after globals.config.FileFlushTriggerDuration // TODO - for now, we could just flush every write (but DoRead() will not read flushed data yet) flushInodesInSlice([]*inodeStruct{inode}) @@ -3191,6 +3194,7 @@ Retry: superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, dereferencedObjectNumberArray: make([]uint64, 0), + flusherTrigger: nil, putObjectNumber: 0, putObjectBuffer: nil, } diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index f706eaac..7476861e 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -113,15 +113,18 @@ type inodeStruct struct { layoutMap map[uint64]layoutMapEntryStruct // For DirInode & FileInode: Map form of .inodeHeadV1.Layout; key == ilayout.InodeHeadLayoutEntryV1Struct.ObjectNumber payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} // For FileInode: ExtentMap B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} - superBlockInodeObjectCountAdjustment int64 // - superBlockInodeObjectSizeAdjustment int64 // - superBlockInodeBytesReferencedAdjustment int64 // - dereferencedObjectNumberArray []uint64 // - putObjectNumber uint64 // ObjectNumber to PUT during flush - putObjectBuffer []byte // PUT content to send to .putObjectNumber'd Object: - // For DirInode: marshaled .payload & ilayout.InodeHeadV1Struct - // For FileInode: file extents, marshaled .payload, & ilayout.InodeHeadV1Struct - // For SymLinkInode: marshaled ilayout.InodeHeadV1Struct + superBlockInodeObjectCountAdjustment int64 // + superBlockInodeObjectSizeAdjustment int64 // + superBlockInodeBytesReferencedAdjustment int64 // + dereferencedObjectNumberArray []uint64 // + flusherWG sync.WaitGroup // For FileInode, marked Done() when a flush operation has completed + flusherTrigger chan struct{} // For FileInode, used to immediately trigger a flush operation + // If nil, no flusher has been launched yet (& .flusherWG.Done() will immediately return) + putObjectNumber uint64 // ObjectNumber to PUT during flush + putObjectBuffer []byte // PUT content to send to .putObjectNumber'd Object: + // For DirInode: marshaled .payload &å ilayout.InodeHeadV1Struct + // For FileInode: file extents then marshaled .payload & ilayout.InodeHeadV1Struct + // For SymLinkInode: marshaled ilayout.InodeHeadV1Struct } type inodeHeldLockStruct struct { diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index e24bb5c0..be6b679a 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -8,6 +8,7 @@ import ( "fmt" "log" "sync" + "time" "github.com/NVIDIA/sortedmap" @@ -1134,3 +1135,64 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint // Now loop back to fetch the next existing extent } } + +func (fileInode *inodeStruct) launchFlusher(withTimeout bool) { + var ( + timer *time.Timer + ) + + if withTimeout { + timer = time.NewTimer(globals.config.FileFlushTriggerDuration) + } else { + timer = nil + } + + fileInode.flusherWG.Add(1) + fileInode.flusherTrigger = make(chan struct{}, 1) + + go fileInode.gorFlusher(timer) +} + +func (fileInode *inodeStruct) forceFlusher() { + if fileInode.flusherTrigger == nil { + fileInode.launchFlusher(false) + } + + fileInode.flusherTrigger <- struct{}{} + + fileInode.flusherWG.Wait() + + select { + case <-fileInode.flusherTrigger: + // Our write to .flusherTrigger arrived after gorFlusher's timer (if any) had expired + default: + // Our write to .flusherTrigger arrived before gorFlusher's timer (if any) had expired + } + + // Either way, .flusherTrigger is now drained... so drop it + + close(fileInode.flusherTrigger) + fileInode.flusherTrigger = nil +} + +func (fileInode *inodeStruct) gorFlusher(timer *time.Timer) { + if timer == nil { + <-fileInode.flusherTrigger + } else { + select { + case <-timer.C: + // Our timer has expired + case <-fileInode.flusherTrigger: + // We need to cancel our timer safely + + if !timer.Stop() { + <-timer.C + } + } + } + + // TODO - finally, actually perform the flush + // TODO - note that if the timer popped, we need to get the lock first... somehow avoiding deadlock... + + fileInode.flusherWG.Done() +} diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index cc937687..a27f09bd 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -131,6 +131,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, dereferencedObjectNumberArray: make([]uint64, 0), + flusherTrigger: nil, putObjectNumber: 0, putObjectBuffer: nil, } From 867b04fe6dd0fda9aaabd44a48c6a5e7860c3421 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 15 Nov 2021 17:29:11 -0800 Subject: [PATCH 178/258] Finished DoWrite() --- iclient/iclientpkg/fission.go | 97 +++++++++++++++++++++---- iclient/iclientpkg/globals.go | 23 +++--- iclient/iclientpkg/impl.go | 8 --- iclient/iclientpkg/inode.go | 131 ++++++++++++++++++---------------- iclient/iclientpkg/lease.go | 2 +- 5 files changed, 169 insertions(+), 92 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index e4863567..a489263a 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -596,9 +596,9 @@ Retry: superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, dereferencedObjectNumberArray: make([]uint64, 0), - flusherTrigger: nil, putObjectNumber: 0, putObjectBuffer: nil, + fileFlusher: nil, } symLinkInode.linkSet[ilayout.InodeLinkTableEntryStruct{ @@ -799,9 +799,9 @@ Retry: superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, dereferencedObjectNumberArray: make([]uint64, 0), - flusherTrigger: nil, putObjectNumber: 0, putObjectBuffer: nil, + fileFlusher: nil, } childDirInode.linkSet[ilayout.InodeLinkTableEntryStruct{ @@ -2000,7 +2000,27 @@ Retry: } } - // TODO - if this write would exceed globals.config.FileFlushTriggerSize, flush first + if inode.dirty { + // A prior call to DoWrite() put something in inode.putObjectBuffer... + // + // Pre-flush if this write would cause inode.putObjectBuffer to exceed globals.config.FileFlushTriggerSize + + if (uint64(len(inode.putObjectBuffer)) + uint64(len(writeIn.Data))) > globals.config.FileFlushTriggerSize { + flushInodesInSlice([]*inodeStruct{inode}) + + inode.dirty = true + + inode.ensurePutObjectIsActive() + + inode.launchFlusher() + } + } else { + inode.dirty = true + + inode.ensurePutObjectIsActive() + + inode.launchFlusher() + } oldSize = inode.inodeHeadV1.Size @@ -2015,8 +2035,6 @@ Retry: } } - inode.dirty = true - inode.inodeHeadV1.ModificationTime = startTime inode.inodeHeadV1.Size = newSize @@ -2025,10 +2043,6 @@ Retry: inode.putObjectBuffer = append(inode.putObjectBuffer, writeIn.Data...) - // TODO - need to (possibly) trigger new timer after globals.config.FileFlushTriggerDuration - // TODO - for now, we could just flush every write (but DoRead() will not read flushed data yet) - flushInodesInSlice([]*inodeStruct{inode}) - inodeLockRequest.unlockAll() writeOut = &fission.WriteOut{ @@ -2534,7 +2548,11 @@ Retry: func (dummy *globalsStruct) DoFlush(inHeader *fission.InHeader, flushIn *fission.FlushIn) (errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoFlush(inHeader: %+v, flushIn: %+v)", inHeader, flushIn) @@ -2546,8 +2564,61 @@ func (dummy *globalsStruct) DoFlush(inHeader *fission.InHeader, flushIn *fission globals.stats.DoFlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + openHandle = lookupOpenHandleByFissionFH(flushIn.FH) + if nil == openHandle { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + if openHandle.inodeNumber != inHeader.NodeID { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + if !openHandle.fissionFlagsWrite { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + + inode = lookupInode(openHandle.inodeNumber) + if nil == inode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeFile { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + + if inode.dirty { + flushInodesInSlice([]*inodeStruct{inode}) + } + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -3194,9 +3265,9 @@ Retry: superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, dereferencedObjectNumberArray: make([]uint64, 0), - flusherTrigger: nil, putObjectNumber: 0, putObjectBuffer: nil, + fileFlusher: nil, } fileInode.linkSet[ilayout.InodeLinkTableEntryStruct{ diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 7476861e..6e9e95ce 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -99,6 +99,13 @@ type openHandleStruct struct { fissionFlagsWrite bool // As supplied in fission.{CreateIn|OpenIn|OpenDirIn}.Flags [true if (.Flags & (syscall.O_ACCMODE) is one of syscall.{O_RDWR|O_WRONLY}] } +type fileInodeFlusherStruct struct { + inode *inodeStruct + inodeNumber uint64 // Having a copy avoids needing to accessing a potentially ExclusivelyLock'd inodeStruct + timer *time.Timer // Initiated with time.NewTimer(globals.config.FileFlushTriggerDuration) + cancelChan chan struct{} // Buffered chan to tell (*fileInodeFlusherStruct).gorFlusher() to exit now w/out flushing +} + type inodeStruct struct { inodeNumber uint64 // dirty bool // @@ -113,18 +120,16 @@ type inodeStruct struct { layoutMap map[uint64]layoutMapEntryStruct // For DirInode & FileInode: Map form of .inodeHeadV1.Layout; key == ilayout.InodeHeadLayoutEntryV1Struct.ObjectNumber payload sortedmap.BPlusTree // For DirInode: Directory B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} // For FileInode: ExtentMap B+Tree from .inodeHeadV1.PayloadObjec{Number|Offset|Length} - superBlockInodeObjectCountAdjustment int64 // - superBlockInodeObjectSizeAdjustment int64 // - superBlockInodeBytesReferencedAdjustment int64 // - dereferencedObjectNumberArray []uint64 // - flusherWG sync.WaitGroup // For FileInode, marked Done() when a flush operation has completed - flusherTrigger chan struct{} // For FileInode, used to immediately trigger a flush operation - // If nil, no flusher has been launched yet (& .flusherWG.Done() will immediately return) - putObjectNumber uint64 // ObjectNumber to PUT during flush - putObjectBuffer []byte // PUT content to send to .putObjectNumber'd Object: + superBlockInodeObjectCountAdjustment int64 // + superBlockInodeObjectSizeAdjustment int64 // + superBlockInodeBytesReferencedAdjustment int64 // + dereferencedObjectNumberArray []uint64 // + putObjectNumber uint64 // ObjectNumber to PUT during flush + putObjectBuffer []byte // PUT content to send to .putObjectNumber'd Object: // For DirInode: marshaled .payload &å ilayout.InodeHeadV1Struct // For FileInode: file extents then marshaled .payload & ilayout.InodeHeadV1Struct // For SymLinkInode: marshaled ilayout.InodeHeadV1Struct + fileFlusher *fileInodeFlusherStruct // If non-nil, FileInode data in .putObjectBuffer time-triggered flush tracked by a (*fileInodeFlusherStruct).gorFlusher() } type inodeHeldLockStruct struct { diff --git a/iclient/iclientpkg/impl.go b/iclient/iclientpkg/impl.go index 3b1e58fd..600e0558 100644 --- a/iclient/iclientpkg/impl.go +++ b/iclient/iclientpkg/impl.go @@ -23,15 +23,11 @@ func start(confMap conf.ConfMap, fissionErrChan chan error) (err error) { return } - // TODO - err = performMountFUSE() if nil != err { return } - // TODO - err = startHTTPServer() if nil != err { return @@ -46,15 +42,11 @@ func stop() (err error) { return } - // TODO - err = performUnmountFUSE() if nil != err { return } - // TODO - err = stopLeaseHandler() if nil != err { return diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index be6b679a..328cdee6 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -680,6 +680,70 @@ func (inode *inodeStruct) ensurePutObjectIsActive() { } } +func (fileInode *inodeStruct) launchFlusher() { + fileInode.fileFlusher = &fileInodeFlusherStruct{ + inode: fileInode, + inodeNumber: fileInode.inodeNumber, + timer: time.NewTimer(globals.config.FileFlushTriggerDuration), + cancelChan: make(chan struct{}, 1), + } + + go fileInode.fileFlusher.gor() +} + +func (fileFlusher *fileInodeFlusherStruct) gor() { + var ( + inodeLockRequest *inodeLockRequestStruct + ) + + select { + case <-fileFlusher.cancelChan: + // We need to cancel our timer safely + + if !fileFlusher.timer.Stop() { + <-fileFlusher.timer.C + } + case <-fileFlusher.timer.C: + // Our timer has expired - so we likely need to do a flush... but we need an Exclusive Lock to do so + // + // But note that "our" fileFlusher.putObjectNumber may have already been flushed... + + Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = fileFlusher.inodeNumber + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + goto Retry + } + + if fileFlusher == fileFlusher.inode.fileFlusher { + // We now know that fileFlusher.inode.dirty == true and + // that we need to flush "our" fileFlusher.putObjectNumber + + flushInodesInSlice([]*inodeStruct{fileFlusher.inode}) + } + + inodeLockRequest.unlockAll() + } +} + +func (fileFlusher *fileInodeFlusherStruct) cancel() { + // Prevent (*fileInodeFlusherStruct).gor() from case where it's .timer had already expired + // and it is awaiting the ExclusiveLock before checking fileFlusher.inode.fileFlusher + + fileFlusher.inode.fileFlusher = nil + + // Now send "cancel" request to (*fileInodeFlusherStruct).gor() + // + // Note that it may never actually read the "cancel" request, but since .cancelChan + // is buffered, and we close it here, the garbage collector will soon reclaim it + + fileFlusher.cancelChan <- struct{}{} + + close(fileFlusher.cancelChan) +} + // flush marshals the current state of an assumed to be dirty inode into its .putObjectBuffer. // This .putObjectBuffer is then passed to objectPUT() following which the inode is marked clean. // @@ -712,6 +776,12 @@ func (inode *inodeStruct) flush() (inodeHeadLength uint64) { } } + if inode.inodeHeadV1.InodeType == ilayout.InodeTypeFile { + if inode.fileFlusher != nil { + inode.fileFlusher.cancel() + } + } + inode.convertLinkSetToInodeHeadV1LinkTable() inode.convertStreamMapToInodeHeadV1StreamTable() inode.convertLayoutMapToInodeHeadV1Layout() @@ -1135,64 +1205,3 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint // Now loop back to fetch the next existing extent } } - -func (fileInode *inodeStruct) launchFlusher(withTimeout bool) { - var ( - timer *time.Timer - ) - - if withTimeout { - timer = time.NewTimer(globals.config.FileFlushTriggerDuration) - } else { - timer = nil - } - - fileInode.flusherWG.Add(1) - fileInode.flusherTrigger = make(chan struct{}, 1) - - go fileInode.gorFlusher(timer) -} - -func (fileInode *inodeStruct) forceFlusher() { - if fileInode.flusherTrigger == nil { - fileInode.launchFlusher(false) - } - - fileInode.flusherTrigger <- struct{}{} - - fileInode.flusherWG.Wait() - - select { - case <-fileInode.flusherTrigger: - // Our write to .flusherTrigger arrived after gorFlusher's timer (if any) had expired - default: - // Our write to .flusherTrigger arrived before gorFlusher's timer (if any) had expired - } - - // Either way, .flusherTrigger is now drained... so drop it - - close(fileInode.flusherTrigger) - fileInode.flusherTrigger = nil -} - -func (fileInode *inodeStruct) gorFlusher(timer *time.Timer) { - if timer == nil { - <-fileInode.flusherTrigger - } else { - select { - case <-timer.C: - // Our timer has expired - case <-fileInode.flusherTrigger: - // We need to cancel our timer safely - - if !timer.Stop() { - <-timer.C - } - } - } - - // TODO - finally, actually perform the flush - // TODO - note that if the timer popped, we need to get the lock first... somehow avoiding deadlock... - - fileInode.flusherWG.Done() -} diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index a27f09bd..ed64d83b 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -131,9 +131,9 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { superBlockInodeObjectSizeAdjustment: 0, superBlockInodeBytesReferencedAdjustment: 0, dereferencedObjectNumberArray: make([]uint64, 0), - flusherTrigger: nil, putObjectNumber: 0, putObjectBuffer: nil, + fileFlusher: nil, } globals.inodeTable[inodeLockRequest.inodeNumber] = inode From 942ec619407570ecce89028ba15ba07588947c2a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 24 Nov 2021 12:11:36 -0800 Subject: [PATCH 179/258] Added -commonName option to icert ...needed to ensure a non-RootCA cert doesn't look like a self-signed cert (i.e. Issuer: and Subject: names being identical would indicate this) --- Dockerfile | 4 ++-- icert/main.go | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 219be4cf..e98f9db0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,8 +104,8 @@ RUN make $MakeTarget FROM base as imgr COPY --from=build /clone/icert/icert ./ -RUN ./icert -ca -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 -RUN ./icert -ed25519 -caCert caCert.pem -caKey caKey.pem -ttl 3560 -cert cert.pem -key key.pem -dns imgr +RUN ./icert -ca -ed25519 -caCert caCert.pem -caKey caKey.pem -commonName RootCA -ttl 3560 +RUN ./icert -ed25519 -caCert caCert.pem -caKey caKey.pem -commonName imgr -ttl 3560 -cert cert.pem -key key.pem -dns imgr COPY --from=build /clone/imgr/imgr ./ COPY --from=build /clone/imgr/imgr.conf ./ diff --git a/icert/main.go b/icert/main.go index 0806766a..1592eb36 100644 --- a/icert/main.go +++ b/icert/main.go @@ -14,6 +14,8 @@ // path to CA Certificate's PrivateKey // -cert string // path to Endpoint Certificate +// -commonName string +// generated Certificate's Subject.CommonName // -country value // generated Certificate's Subject.Country // -dns value @@ -87,6 +89,8 @@ func main() { generateKeyAlgorithmEd25519Flag = flag.Bool(icertpkg.GenerateKeyAlgorithmEd25519, false, "generate key via Ed25519") generateKeyAlgorithmRSAFlag = flag.Bool(icertpkg.GenerateKeyAlgorithmRSA, false, "generate key via RSA") + commonNameFlag = flag.String("commonName", "", "generated Certificate's Subject.CommonName") + organizationFlag stringSlice countryFlag stringSlice provinceFlag stringSlice @@ -130,6 +134,8 @@ func main() { fmt.Printf("generateKeyAlgorithmEd25519Flag: %v\n", *generateKeyAlgorithmEd25519Flag) fmt.Printf(" generateKeyAlgorithmRSAFlag: %v\n", *generateKeyAlgorithmRSAFlag) fmt.Println() + fmt.Printf(" commonNameFlag: \"%v\"\n", *commonNameFlag) + fmt.Println() fmt.Printf(" organizationFlag: %v\n", organizationFlag) fmt.Printf(" countryFlag: %v\n", countryFlag) fmt.Printf(" provinceFlag: %v\n", provinceFlag) @@ -196,6 +202,7 @@ func main() { } subject = pkix.Name{ + CommonName: *commonNameFlag, Organization: organizationFlag, Country: countryFlag, Province: provinceFlag, From e1e5392642a41f4120e6b9e9beb9f7496f66978b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 24 Nov 2021 14:29:37 -0800 Subject: [PATCH 180/258] Removed the problematic --no-cache option in Dockerfile/docker-compose.yml comments --- Dockerfile | 6 ------ docker-compose.yml | 7 +------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index e98f9db0..29094a29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ # --target {base|dev|build|imgr|iclient} \ # [--build-arg GolangVersion=] \ # [--build-arg MakeTarget={|all|ci|minimal}] \ -# [--no-cache] \ # [-t [:]] . # # Notes: @@ -31,11 +30,6 @@ # 1) identifies Makefile target(s) to build (following make clean) # 2) defaults to blank (equivalent to "all") # 3) only used in --target build -# --no-cache: -# 1) tells Docker to ignore cached images that might be stale -# 2) useful due to Docker not understanding changes to build-args -# 3) useful due to Docker not understanding changes to context dir -# 4) if running a sequence of builds, consider instead docker builder prune # -t: # 1) provides a name REPOSITORY:TAG for the built image # 2) if no tag is specified, TAG will be "latest" diff --git a/docker-compose.yml b/docker-compose.yml index dc0c0bf7..bba03b9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,8 @@ # To build the `development` service # -# docker compose build [--no-cache] +# docker compose build # # Notes: -# --no-cache: -# 1) tells Docker to ignore cached images that might be stale -# 2) useful due to Docker not understanding changes to build-args -# 3) useful due to Docker not understanding changes to context dir -# 4) finer grained alternative to previously running docker builder prune # Images built will be named "proxyfs_:latest" # Use docker tag SOURCE_IMAGE[:TAG] TARGET_IMAGE[:TAG] to create an alias # From 13cee642e50c0137bef11750c811e74d518291b2 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 1 Dec 2021 17:15:30 -0800 Subject: [PATCH 181/258] [WIP] iclient/iclientpkg/lease.go needs work to handle blocked/unblocked lock requests --- Dockerfile | 4 + go.mod | 22 ++++- go.sum | 53 ++++++++++ iclient/dev.conf | 2 + iclient/iclient.conf | 2 + iclient/iclientpkg/api.go | 2 + iclient/iclientpkg/fission.go | 62 +++++++++++- iclient/iclientpkg/globals.go | 25 ++++- iclient/iclientpkg/impl.go | 10 -- iclient/iclientpkg/inode.go | 1 + iclient/iclientpkg/lease.go | 175 ++++++++++++++++++++++++++++------ 11 files changed, 311 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index 29094a29..ccef303e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -86,6 +86,10 @@ WORKDIR /tmp RUN wget -nv ${GolangURL} RUN tar -C /usr/local -xzf $GolangBasename ENV PATH $PATH:/usr/local/go/bin +RUN git clone https://github.com/go-delve/delve +WORKDIR /tmp/delve +RUN go build github.com/go-delve/delve/cmd/dlv +RUN cp dlv /usr/local/go/bin/. VOLUME /src WORKDIR /src diff --git a/go.mod b/go.mod index dad94ac5..5989fb71 100644 --- a/go.mod +++ b/go.mod @@ -19,48 +19,62 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 go.etcd.io/etcd v3.3.25+incompatible - golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f + golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 ) require ( github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cilium/ebpf v0.7.0 // indirect github.com/coreos/bbolt v1.3.6 // indirect github.com/coreos/etcd v3.3.25+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/cosiner/argv v0.1.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/derekparker/trie v0.0.0-20200317170641-1fdf38b7b0e9 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-delve/delve v1.7.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-dap v0.6.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect - github.com/kr/pretty v0.2.0 // indirect + github.com/kr/pretty v0.2.1 // indirect github.com/magiconair/properties v1.8.5 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect github.com/ory/go-acc v0.2.6 // indirect github.com/ory/viper v1.7.5 // indirect github.com/pborman/uuid v1.2.1 // indirect github.com/pelletier/go-toml v1.9.4 // indirect + github.com/peterh/liner v1.2.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/common v0.30.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect @@ -70,9 +84,11 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.starlark.net v0.0.0-20211013185944-b0039bd2cfe3 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.19.0 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect diff --git a/go.sum b/go.sum index 1d589ab2..a9aaa463 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -86,7 +87,10 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -98,12 +102,19 @@ github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg= +github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/derekparker/trie v0.0.0-20200317170641-1fdf38b7b0e9 h1:G765iDCq7bP5opdrPkXk+4V3yfkgV9iGFuheWZ/X/zY= +github.com/derekparker/trie v0.0.0-20200317170641-1fdf38b7b0e9/go.mod h1:D6ICZm05D9VN1n/8iOtBxLpXtoGp6HDFUJ1RNVieOSE= github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= @@ -119,11 +130,14 @@ github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-delve/delve v1.7.3 h1:5I8KjqwKIz6I7JQ4QA+eY5PRB0+INs9h7wEDRzFnuHQ= +github.com/go-delve/delve v1.7.3/go.mod h1:mBVf2XSFxRX8Y8AuGPhANnsxuZAnTFRM4l8KeaO5pm8= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -192,6 +206,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-dap v0.6.0 h1:Y1RHGUtv3R8y6sXq2dtGRMYrFB2hSqyFVws7jucrzX4= +github.com/google/go-dap v0.6.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -241,6 +257,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -276,6 +294,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -284,8 +304,15 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -324,6 +351,8 @@ github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bA github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterh/liner v1.2.1 h1:O4BlKaq/LWu6VRWmol4ByWfzx6MfXc5Op5HETyIy5yg= +github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -358,13 +387,18 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -386,6 +420,7 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -395,6 +430,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -422,6 +458,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= @@ -437,6 +474,9 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.starlark.net v0.0.0-20200821142938-949cc6f4b097/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= +go.starlark.net v0.0.0-20211013185944-b0039bd2cfe3 h1:oBcONsksxvpeodDrLjiMDaKHXKAVVfAydhe/792CE/o= +go.starlark.net v0.0.0-20211013185944-b0039bd2cfe3/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -451,6 +491,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -497,6 +540,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -540,6 +584,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -620,6 +666,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0NHHxO3Z0g7Gus2O1kk= golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= @@ -634,6 +682,8 @@ golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42 h1:G2DDmludOQZoWbpCr7OKDxnl4 golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e h1:zeJt6jBtVDK23XK9QXcmG0FvO0elikp0dYZQZOeL1y0= golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -705,6 +755,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736 h1:cw6nUxdoEN5iEIWYD8aAsTZ8iYjLVNiHAb7xz/80WO4= +golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -833,6 +885,7 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= diff --git a/iclient/dev.conf b/iclient/dev.conf index 2b1f2fc6..715f0cb5 100644 --- a/iclient/dev.conf +++ b/iclient/dev.conf @@ -36,6 +36,8 @@ ReadCacheLineSize: 1048576 ReadCacheLineCountMax: 1024 FileFlushTriggerSize: 10485760 FileFlushTriggerDuration: 10s +InodeLockRetryDelay: 10ms +InodeLockRetryDelayVariance: 50 LogFilePath: LogToConsole: true TraceEnabled: false diff --git a/iclient/iclient.conf b/iclient/iclient.conf index 874415a4..c697a3a2 100644 --- a/iclient/iclient.conf +++ b/iclient/iclient.conf @@ -36,6 +36,8 @@ ReadCacheLineSize: 1048576 ReadCacheLineCountMax: 1024 FileFlushTriggerSize: 10485760 FileFlushTriggerDuration: 10s +InodeLockRetryDelay: 10ms +InodeLockRetryDelayVariance: 50 LogFilePath: LogToConsole: true TraceEnabled: false diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go index b6ab2d36..d2bf786c 100644 --- a/iclient/iclientpkg/api.go +++ b/iclient/iclientpkg/api.go @@ -42,6 +42,8 @@ // ReadCacheLineCountMax: 1024 # 1GiB (with ReadCacheLineSize of 1MiB) // FileFlushTriggerSize: 10485760 # [10MiB] Amount of written data before metadata is appended and flush is triggered // FileFlushTriggerDuration: 10s # Amount of time before unwritten data and its metadata flush is triggered +// InodeLockRetryDelay: 10ms +// InodeLockRetryDelayVariance: 50 # Percentage (1-100) of InodeLockRetryDelay // LogFilePath: iclient.log // LogToConsole: true // TraceEnabled: false diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index a489263a..3b3935e6 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -101,6 +101,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -240,6 +241,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -326,6 +328,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -445,6 +448,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -519,6 +523,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -610,6 +615,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -722,6 +728,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -850,6 +857,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -950,6 +958,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -1003,6 +1012,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -1104,6 +1114,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -1157,6 +1168,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -1283,6 +1295,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -1336,6 +1349,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -1436,6 +1450,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -1565,6 +1580,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -1935,6 +1951,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2138,6 +2155,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2232,6 +2250,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2317,6 +2336,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2413,6 +2433,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2505,6 +2526,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2570,6 +2592,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2685,6 +2708,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2799,6 +2823,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -2962,6 +2987,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -3137,6 +3163,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -3207,7 +3234,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { - inodeLockRequest.unlockAll() + performInodeLockRetryDelay() goto Retry } @@ -3279,6 +3306,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -3509,6 +3537,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -3763,6 +3792,7 @@ Retry: inodeLockRequest.exclusive = obtainExclusiveLock inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -3853,15 +3883,25 @@ func doRenameCommon(oldDirInodeNumber uint64, oldName string, newDirInodeNumber renamedInode *inodeStruct replacedInode *inodeStruct ) + logInfof("UNDO: entered doRenameCommon(oldDirInodeNumber: %v, oldName: %s, newDirInodeNumber: %v, newName: %s)", oldDirInodeNumber, oldName, newDirInodeNumber, newName) + defer func() { + logTracef("UNDO: leaving doRenameCommon() with errno: %v", errno) + }() Retry: + logTracef("UNDO: Reached doRenameCommon() Point A") inodeLockRequest = newLockRequest() inodeLockRequest.inodeNumber = oldDirInodeNumber inodeLockRequest.exclusive = true + logTracef("UNDO: Reached doRenameCommon() Point A.1") inodeLockRequest.addThisLock() + logTracef("UNDO: Reached doRenameCommon() Point A.2") if len(inodeLockRequest.locksHeld) == 0 { + logTracef("UNDO: Reached doRenameCommon() Point A.3") + performInodeLockRetryDelay() goto Retry } + logTracef("UNDO: Reached doRenameCommon() Point A.4") oldDirInode = lookupInode(oldDirInodeNumber) if nil == oldDirInode { @@ -3869,6 +3909,7 @@ Retry: errno = syscall.ENOENT return } + logTracef("UNDO: Reached doRenameCommon() Point A.5") if nil == oldDirInode.inodeHeadV1 { err = oldDirInode.populateInodeHeadV1() @@ -3878,12 +3919,14 @@ Retry: return } } + logTracef("UNDO: Reached doRenameCommon() Point A.6") if oldDirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { inodeLockRequest.unlockAll() errno = syscall.ENOTDIR return } + logTracef("UNDO: Reached doRenameCommon() Point A.7") if oldDirInode.payload == nil { err = oldDirInode.oldPayload() @@ -3893,6 +3936,7 @@ Retry: return } } + logTracef("UNDO: Reached doRenameCommon() Point A.8") directoryEntryValueV1AsValue, ok, err = oldDirInode.payload.GetByKey(oldName) if nil != err { @@ -3903,6 +3947,7 @@ Retry: errno = syscall.ENOENT return } + logTracef("UNDO: Reached doRenameCommon() Point A.9") directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) if !ok { @@ -3913,15 +3958,17 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { - inodeLockRequest.unlockAll() + performInodeLockRetryDelay() goto Retry } + logTracef("UNDO: Reached doRenameCommon() Point A.A") renamedInode = lookupInode(directoryEntryValueV1.InodeNumber) if nil == renamedInode { inodeLockRequest.unlockAll() goto Retry } + logTracef("UNDO: Reached doRenameCommon() Point A.B") if nil == renamedInode.inodeHeadV1 { err = renamedInode.populateInodeHeadV1() @@ -3932,6 +3979,7 @@ Retry: } } + logTracef("UNDO: Reached doRenameCommon() Point B") if oldDirInodeNumber == newDirInodeNumber { newDirInode = oldDirInode @@ -3945,6 +3993,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } @@ -3978,6 +4027,7 @@ Retry: } } } + logTracef("UNDO: Reached doRenameCommon() Point C") directoryEntryValueV1AsValue, ok, err = newDirInode.payload.GetByKey(newName) if nil != err { @@ -3994,7 +4044,7 @@ Retry: inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { - inodeLockRequest.unlockAll() + performInodeLockRetryDelay() goto Retry } @@ -4020,8 +4070,10 @@ Retry: } else { replacedInode = nil } + logTracef("UNDO: Reached doRenameCommon() Point D") if renamedInode.inodeHeadV1.InodeType == ilayout.InodeTypeDir { + logTracef("UNDO: Reached doRenameCommon() Point E") if replacedInode != nil { inodeLockRequest.unlockAll() errno = syscall.EISDIR @@ -4106,7 +4158,9 @@ Retry: logFatalf("newDirInode.payload.Put(newName,) returned !ok") } } else { + logTracef("UNDO: Reached doRenameCommon() Point F") if replacedInode != nil { + logTracef("UNDO: Reached doRenameCommon() Point G") replacedInode.dirty = true replacedInode.inodeHeadV1.ModificationTime = startTime @@ -4163,6 +4217,7 @@ Retry: logFatalf("newDirInode.payload.PatchByKey(newName,) returned !ok") } } else { + logTracef("UNDO: Reached doRenameCommon() Point H") renamedInode.dirty = true renamedInode.inodeHeadV1.ModificationTime = startTime @@ -4210,6 +4265,7 @@ Retry: } } } + logTracef("UNDO: Reached doRenameCommon() Point I") if replacedInode == nil { if oldDirInodeNumber == newDirInodeNumber { diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index 6e9e95ce..be56f42a 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -57,6 +57,10 @@ type configStruct struct { ReadCacheLineCountMax uint64 FileFlushTriggerSize uint64 FileFlushTriggerDuration time.Duration + InodeLockRetryLimit uint64 + InodeLockRetryDelay time.Duration + InodeLockRetryDelayVariance uint8 + InodeLockRetryExponentialBackoff float64 LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled LogToConsole bool TraceEnabled bool @@ -371,10 +375,10 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err } globals.config.SwiftRetryDelayVariance, err = confMap.FetchOptionValueUint8("ICLIENT", "SwiftRetryDelayVariance") if nil == err { - if 0 == globals.config.SwiftRetryDelayVariance { + if globals.config.SwiftRetryDelayVariance == 0 { logFatalf("[Agent]SwiftRetryDelayVariance must be > 0") } - if 100 < globals.config.SwiftRetryDelayVariance { + if globals.config.SwiftRetryDelayVariance > 100 { logFatalf("[Agent]SwiftRetryDelayVariance (%v) must be <= 100", globals.config.SwiftRetryDelayVariance) } } else { @@ -448,6 +452,21 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err if nil != err { logFatal(err) } + globals.config.InodeLockRetryDelay, err = confMap.FetchOptionValueDuration("ICLIENT", "InodeLockRetryDelay") + if nil != err { + logFatal(err) + } + globals.config.InodeLockRetryDelayVariance, err = confMap.FetchOptionValueUint8("ICLIENT", "InodeLockRetryDelayVariance") + if nil == err { + if globals.config.InodeLockRetryDelayVariance == 0 { + logFatalf("[Agent]InodeLockRetryDelayVariance must be > 0") + } + if globals.config.InodeLockRetryDelayVariance > 100 { + logFatalf("[Agent]InodeLockRetryDelayVariance (%v) must be <= 100", globals.config.InodeLockRetryDelayVariance) + } + } else { + logFatal(err) + } globals.config.LogFilePath, err = confMap.FetchOptionValueString("ICLIENT", "LogFilePath") if nil != err { err = confMap.VerifyOptionValueIsEmpty("ICLIENT", "LogFilePath") @@ -553,6 +572,8 @@ func uninitializeGlobals() (err error) { globals.config.ReadCacheLineCountMax = 0 globals.config.FileFlushTriggerSize = 0 globals.config.FileFlushTriggerDuration = time.Duration(0) + globals.config.InodeLockRetryDelay = time.Duration(0) + globals.config.InodeLockRetryDelayVariance = 0 globals.config.LogFilePath = "" globals.config.LogToConsole = false globals.config.TraceEnabled = false diff --git a/iclient/iclientpkg/impl.go b/iclient/iclientpkg/impl.go index 600e0558..2e136de3 100644 --- a/iclient/iclientpkg/impl.go +++ b/iclient/iclientpkg/impl.go @@ -18,11 +18,6 @@ func start(confMap conf.ConfMap, fissionErrChan chan error) (err error) { return } - err = startLeaseHandler() - if nil != err { - return - } - err = performMountFUSE() if nil != err { return @@ -47,11 +42,6 @@ func stop() (err error) { return } - err = stopLeaseHandler() - if nil != err { - return - } - err = stopRPCHandler() if nil != err { return diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 328cdee6..0ef9ba98 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -714,6 +714,7 @@ func (fileFlusher *fileInodeFlusherStruct) gor() { inodeLockRequest.exclusive = true inodeLockRequest.addThisLock() if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() goto Retry } diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index ed64d83b..7fdbfc13 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -5,18 +5,12 @@ package iclientpkg import ( "container/list" + "math/rand" + "time" "github.com/NVIDIA/proxyfs/imgr/imgrpkg" ) -func startLeaseHandler() (err error) { - return nil // TODO -} - -func stopLeaseHandler() (err error) { - return nil // TODO -} - // newLockRequest is called to create and initialize an inodeLockRequestStruct. // func newLockRequest() (inodeLockRequest *inodeLockRequestStruct) { @@ -56,6 +50,29 @@ func (inodeLockRequest *inodeLockRequestStruct) markForDelete(inodeNumber uint64 globals.Unlock() } +// performInodeLockRetryDelay simply delays the current goroutine for InodeLockRetryDelay +// interval +/- InodeLockRetryDelayVariance (interpreted as a percentage). +// +// It is expected that a caller to addThisLock(), when noticing locksHeld map is empty, +// will call performInodeLockRetryDelay() before re-attempting a lock sequence. +// +func performInodeLockRetryDelay() { + var ( + delay time.Duration + variance int64 + ) + + variance = rand.Int63n(int64(globals.config.InodeLockRetryDelay) & int64(globals.config.InodeLockRetryDelayVariance) / int64(100)) + + if (variance % 2) == 0 { + delay = time.Duration(int64(globals.config.InodeLockRetryDelay) + variance) + } else { + delay = time.Duration(int64(globals.config.InodeLockRetryDelay) - variance) + } + + time.Sleep(delay) +} + // addThisLock is called for an existing inodeLockRequestStruct with the inodeNumber and // exclusive fields set to specify the inode to be locked and whether or not the lock // should be exlusive. @@ -70,15 +87,20 @@ func (inodeLockRequest *inodeLockRequestStruct) markForDelete(inodeNumber uint64 // requested lock but also the implicit releasing of any locks previously in the locksHeld map // at the time of the call. // +// It is expected that the caller, when noticing locksHeld map is empty, will call +// performInodeLockRetryDelay() before re-attempting the lock sequence. +// func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { var ( - err error - inode *inodeStruct - inodeHeldLock *inodeHeldLockStruct - leaseRequest *imgrpkg.LeaseRequestStruct - leaseResponse *imgrpkg.LeaseResponseStruct - ok bool + blockedInodeLockRequest *inodeLockRequestStruct + err error + inode *inodeStruct + inodeHeldLock *inodeHeldLockStruct + leaseRequest *imgrpkg.LeaseRequestStruct + leaseResponse *imgrpkg.LeaseResponseStruct + ok bool ) + logTracef("UNDO: entered addThisLock() with .inodeNumber: %v .exclusive: %v", inodeLockRequest.inodeNumber, inodeLockRequest.exclusive) globals.Lock() @@ -140,8 +162,21 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } if inode.requestList.Len() != 0 { - // At lease one other inodeLockRequestStruct is blocked, so this one must block + logTracef("UNDO: ok... addThisLock() found that we cannot immediately grant the lock as others are blocked...") + // At lease one other inodeLockRequestStruct is blocked, so this one must either block or, to avoid deadlock, release other held locks and retry + + if len(inodeLockRequest.locksHeld) != 0 { + // We must avoid deadlock by releasing other held locks and return + logTracef("UNDO: ok... addThisLock() needs to avoid deadlock, so we must release our other locks and trigger a retry") + + globals.Unlock() + inodeLockRequest.unlockAll() + + return + } + + logTracef("UNDO: ok... addThisLock() will simply block this request") inodeLockRequest.Add(1) inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) @@ -156,6 +191,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { if inode.heldList.Len() != 0 { if inodeLockRequest.exclusive { // Lock is held, so this exclusive inodeLockRequestStruct must block + logTracef("UNDO: ok... addThisLock() needs to block on .inodeNunber: %v .exclusive: TRUEv while inode.heldList.Len() == %v", inodeLockRequest.inodeNumber, inode.heldList.Len()) inodeLockRequest.Add(1) @@ -380,9 +416,34 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + logInfof("UNDO: @ CheckRequestList: loop in addThisLock()") + + CheckRequestList: if inode.requestList.Front() != nil { - logFatalf("TODO: for now, we don't handle multiple shared lock requests queued up") + blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) + + if !blockedInodeLockRequest.exclusive { + // We can also unblock blockedInodeLockRequest + + _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) + blockedInodeLockRequest.listElement = nil + + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: blockedInodeLockRequest, + exclusive: false, + } + + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + + blockedInodeLockRequest.Done() + + // Now go back and check for more + + goto CheckRequestList + } } globals.Unlock() @@ -460,27 +521,83 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { // func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { var ( - err error - inode *inodeStruct - inodeHeldLock *inodeHeldLockStruct - leaseRequest *imgrpkg.LeaseRequestStruct - leaseResponse *imgrpkg.LeaseResponseStruct - releaseList []*inodeStruct = make([]*inodeStruct, 0) + blockedInodeLockRequest *inodeLockRequestStruct + err error + inode *inodeStruct + inodeHeldLock *inodeHeldLockStruct + leaseRequest *imgrpkg.LeaseRequestStruct + leaseResponse *imgrpkg.LeaseResponseStruct + releaseList []*inodeStruct = make([]*inodeStruct, 0) ) globals.Lock() for _, inodeHeldLock = range inodeLockRequest.locksHeld { - _ = inodeHeldLock.inode.heldList.Remove(inodeHeldLock.listElement) - if inodeHeldLock.inode.requestList.Len() == 0 { - if inodeHeldLock.inode.heldList.Len() == 0 { - if inodeHeldLock.inode.markedForDelete { - releaseList = append(releaseList, inodeHeldLock.inode) - delete(globals.inodeTable, inodeHeldLock.inode.inodeNumber) + inode = inodeHeldLock.inode + logTracef("UNDO: entered unlockAll()... found an inodeHeldLock with .inode..inodeNumber: %v .exclusive: %v", inodeHeldLock.inode.inodeNumber, inodeHeldLock.exclusive) + + _ = inode.heldList.Remove(inodeHeldLock.listElement) + + if inode.requestList.Len() == 0 { + if inode.heldList.Len() == 0 { + if inode.markedForDelete { + releaseList = append(releaseList, inode) + delete(globals.inodeTable, inode.inodeNumber) } } } else { - logFatalf("TODO: for now, we don't handle blocked inodeLockRequestStruct's") + if inodeHeldLock.exclusive { + logInfof("UNDO: @ We can unblock at least one blockedInodeLockRequest") + // We can unblock at least one blockedInodeLockRequest + + blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) + + _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) + blockedInodeLockRequest.listElement = nil + + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: blockedInodeLockRequest, + exclusive: blockedInodeLockRequest.exclusive, + } + + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + + blockedInodeLockRequest.Done() + + if !blockedInodeLockRequest.exclusive { + // We can also unblock following blockedInodeLockRequest's that are !.exclusive + + CheckRequestList: + + if inode.requestList.Front() != nil { + blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) + + if !blockedInodeLockRequest.exclusive { + // We can also unblock next blockedInodeLockRequest + + _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) + blockedInodeLockRequest.listElement = nil + + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: blockedInodeLockRequest, + exclusive: false, + } + + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + + blockedInodeLockRequest.Done() + + // Now go back and check for more + + goto CheckRequestList + } + } + } + } } } From ba63a3cf193dab2b4036535c1a15b937b159dc1b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 2 Dec 2021 16:49:22 -0800 Subject: [PATCH 182/258] Added 1st unit test in iclientpkg (lease_test.go just invokes test{Setup|Teardown}() for now) --- iclient/iclientpkg/api_test.go | 12 - iclient/iclientpkg/lease_test.go | 43 ++++ iclient/iclientpkg/utils_test.go | 366 +++++++++++++++++++++++++++++++ 3 files changed, 409 insertions(+), 12 deletions(-) delete mode 100644 iclient/iclientpkg/api_test.go create mode 100644 iclient/iclientpkg/lease_test.go create mode 100644 iclient/iclientpkg/utils_test.go diff --git a/iclient/iclientpkg/api_test.go b/iclient/iclientpkg/api_test.go deleted file mode 100644 index 5cb23770..00000000 --- a/iclient/iclientpkg/api_test.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package iclientpkg - -import ( - "testing" -) - -func TestAPI(t *testing.T) { - t.Logf("TODO") -} diff --git a/iclient/iclientpkg/lease_test.go b/iclient/iclientpkg/lease_test.go new file mode 100644 index 00000000..5cbfb591 --- /dev/null +++ b/iclient/iclientpkg/lease_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +import ( + "testing" +) + +func TestLocks(t *testing.T) { + var ( + err error + fissionErrChan chan error + ) + + testSetup(t) + + fissionErrChan = make(chan error, 1) + + err = initializeGlobals(testGlobals.confMap, fissionErrChan) + if nil != err { + t.Fatalf("initializeGlobals(testGlobals.confMap, fissionErrChan) failed: %v", err) + } + + err = startRPCHandler() + if nil != err { + t.Fatalf("startRPCHandler() failed: %v", err) + } + + t.Logf("TODO... actually test locks in func TestLocks()") + + err = stopRPCHandler() + if nil != err { + t.Fatalf("stopRPCHandler() failed: %v", err) + } + + err = uninitializeGlobals() + if nil != err { + t.Fatalf("uninitializeGlobals() failed: %v", err) + } + + testTeardown(t) +} diff --git a/iclient/iclientpkg/utils_test.go b/iclient/iclientpkg/utils_test.go new file mode 100644 index 00000000..996db82a --- /dev/null +++ b/iclient/iclientpkg/utils_test.go @@ -0,0 +1,366 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +import ( + "crypto/x509/pkix" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/icert/icertpkg" + "github.com/NVIDIA/proxyfs/imgr/imgrpkg" + "github.com/NVIDIA/proxyfs/iswift/iswiftpkg" +) + +const ( + testIPAddr = "dev" + testRetryRPCPort = 32356 + testMgrHTTPServerPort = 15346 + testClientHTTPServerPort = 15347 + testSwiftProxyTCPPort = 8080 + testSwiftAuthUser = "test" + testSwiftAuthKey = "test" + testAccount = "AUTH_test" + testContainer = "con" + testVolume = "testvol" + testRPCDeadlineIO = "60s" + testRPCKeepAlivePeriod = "60s" +) + +type testGlobalsStruct struct { + tempDir string + mountPointDir string + caCertFile string + caKeyFile string + caCertPEMBlock []byte + caKeyPEMBlock []byte + endpointCertFile string + endpointKeyFile string + endpointCertPEMBlock []byte + endpointKeyPEMBlock []byte + confMap conf.ConfMap + imgrHTTPServerURL string + authURL string + authToken string + accountURL string + containerURL string +} + +var testGlobals *testGlobalsStruct + +func testSetup(t *testing.T) { + var ( + authRequestHeaders http.Header + authResponseHeaders http.Header + confStrings []string + err error + expectedAccountURL string + postAndPutVolumePayload string + putAccountRequestHeaders http.Header + putContainerRequestHeaders http.Header + tempDir string + ) + + tempDir, err = ioutil.TempDir("", "iclientpkg_test") + if nil != err { + t.Fatalf("ioutil.TempDir(\"\", \"iclientpkg_test\") failed: %v", err) + } + + testGlobals = &testGlobalsStruct{ + tempDir: tempDir, + mountPointDir: tempDir + "/mnt", + caCertFile: tempDir + "/caCertFile", + caKeyFile: tempDir + "/caKeyFile", + endpointCertFile: tempDir + "/endpoingCertFile", + endpointKeyFile: tempDir + "/endpointKeyFile", + imgrHTTPServerURL: fmt.Sprintf("http://%s:%d", testIPAddr, testMgrHTTPServerPort), + authURL: fmt.Sprintf("http://%s:%d/auth/v1.0", testIPAddr, testSwiftProxyTCPPort), + } + + testGlobals.caCertPEMBlock, testGlobals.caKeyPEMBlock, err = icertpkg.GenCACert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization CA"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + time.Hour, + testGlobals.caCertFile, + testGlobals.caKeyFile) + if nil != err { + t.Fatalf("icertpkg.GenCACert() failed: %v", err) + } + + testGlobals.endpointCertPEMBlock, testGlobals.endpointKeyPEMBlock, err = icertpkg.GenEndpointCert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization Endpoint"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + []string{testIPAddr}, + []net.IP{}, + time.Hour, + testGlobals.caCertPEMBlock, + testGlobals.caKeyPEMBlock, + testGlobals.endpointCertFile, + testGlobals.endpointKeyFile) + if nil != err { + t.Fatalf("icertpkg.GenEndpointCert() failed: %v", err) + } + + err = os.MkdirAll(testGlobals.mountPointDir, os.ModePerm) + if nil != err { + t.Fatalf("os.MkdirAll() failed: %v", err) + } + + confStrings = []string{ + "ICLIENT.VolumeName=" + testVolume, + "ICLIENT.MountPointDirPath=" + testGlobals.mountPointDir, + "ICLIENT.FUSEBlockSize=512", + "ICLIENT.FUSEAllowOther=true", + "ICLIENT.FUSEMaxBackground=1000", + "ICLIENT.FUSECongestionThreshhold=0", + "ICLIENT.FUSEMaxRead=131076", + "ICLIENT.FUSEMaxWrite=131076", + "ICLIENT.FUSEEntryValidDuration=250ms", + "ICLIENT.FUSEAttrValidDuration=250ms", + "ICLIENT.AuthPlugInPath=../../iauth/iauth-swift/iauth-swift.so", + "ICLIENT.AuthPlugInEnvName=", + "ICLIENT.AuthPlugInEnvValue=" + fmt.Sprintf("{\"AuthURL\":\"http://dev:%d/auth/v1.0\"\\u002C\"AuthUser\":\"%s\"\\u002C\"AuthKey\":\"%s\"\\u002C\"Account\":\"%s\"\\u002C\"Container\":\"%s\"}", testSwiftProxyTCPPort, testSwiftAuthUser, testSwiftAuthKey, testAccount, testContainer), + "ICLIENT.SwiftTimeout=10m", + "ICLIENT.SwiftRetryLimit=4", + "ICLIENT.SwiftRetryDelay=100ms", + "ICLIENT.SwiftRetryDelayVariance=25", + "ICLIENT.SwiftRetryExponentialBackoff=1.4", + "ICLIENT.SwiftConnectionPoolSize=128", + "ICLIENT.RetryRPCPublicIPAddr=dev", + "ICLIENT.RetryRPCPort=32356", + "ICLIENT.RetryRPCDeadlineIO=60s", + "ICLIENT.RetryRPCKeepAlivePeriod=60s", + "ICLIENT.RetryRPCCACertFilePath=" + testGlobals.caCertFile, + "ICLIENT.MaxSharedLeases=500", + "ICLIENT.MaxExclusiveLeases=100", + "ICLIENT.InodePayloadEvictLowLimit=100000", + "ICLIENT.InodePayloadEvictHighLimit=100010", + "ICLIENT.DirInodeMaxKeysPerBPlusTreePage=1024", + "ICLIENT.FileInodeMaxKeysPerBPlusTreePage=2048", + "ICLIENT.ReadCacheLineSize=1048576", + "ICLIENT.ReadCacheLineCountMax=1024", + "ICLIENT.FileFlushTriggerSize=10485760", + "ICLIENT.FileFlushTriggerDuration=10s", + "ICLIENT.InodeLockRetryDelay=10ms", + "ICLIENT.InodeLockRetryDelayVariance=50", + "ICLIENT.LogFilePath:", + "ICLIENT.LogToConsole=true", + "ICLIENT.TraceEnabled=false", + "ICLIENT.FUSELogEnabled=false", + "ICLIENT.HTTPServerIPAddr=" + testIPAddr, + "ICLIENT.HTTPServerPort=" + fmt.Sprintf("%d", testClientHTTPServerPort), + + "IMGR.PublicIPAddr=" + testIPAddr, + "IMGR.PrivateIPAddr=" + testIPAddr, + "IMGR.RetryRPCPort=" + fmt.Sprintf("%d", testRetryRPCPort), + "IMGR.HTTPServerPort=" + fmt.Sprintf("%d", testMgrHTTPServerPort), + + "IMGR.RetryRPCTTLCompleted=10m", + "IMGR.RetryRPCAckTrim=100ms", + "IMGR.RetryRPCDeadlineIO=60s", + "IMGR.RetryRPCKeepAlivePeriod=60s", + + "IMGR.RetryRPCCertFilePath=" + testGlobals.endpointCertFile, + "IMGR.RetryRPCKeyFilePath=" + testGlobals.endpointKeyFile, + + "IMGR.CheckPointInterval=10s", + + "IMGR.AuthTokenCheckInterval=1m", + + "IMGR.FetchNonceRangeToReturn=100", + + "IMGR.MinLeaseDuration=250ms", + "IMGR.LeaseInterruptInterval=250ms", + "IMGR.LeaseInterruptLimit=5", + "IMGR.LeaseEvictLowLimit=100000", + "IMGR.LeaseEvictHighLimit=100010", + + "IMGR.SwiftRetryDelay=100ms", + "IMGR.SwiftRetryExpBackoff=2", + "IMGR.SwiftRetryLimit=4", + + "IMGR.SwiftTimeout=10m", + "IMGR.SwiftConnectionPoolSize=128", + + "IMGR.ParallelObjectDeleteMax=10", + + "IMGR.InodeTableCacheEvictLowLimit=10000", + "IMGR.InodeTableCacheEvictHighLimit=10010", + + "IMGR.InodeTableMaxInodesPerBPlusTreePage=2048", + "IMGR.RootDirMaxDirEntriesPerBPlusTreePage=1024", + + "IMGR.LogFilePath=", + "IMGR.LogToConsole=true", + "IMGR.TraceEnabled=false", + + "ISWIFT.SwiftProxyIPAddr=" + testIPAddr, + "ISWIFT.SwiftProxyTCPPort=" + fmt.Sprintf("%d", testSwiftProxyTCPPort), + + "ISWIFT.MaxAccountNameLength=256", + "ISWIFT.MaxContainerNameLength=256", + "ISWIFT.MaxObjectNameLength=1024", + "ISWIFT.AccountListingLimit=10000", + "ISWIFT.ContainerListingLimit=10000", + } + + testGlobals.confMap, err = conf.MakeConfMapFromStrings(confStrings) + if nil != err { + t.Fatalf("conf.MakeConfMapFromStrings(confStrings) failed: %v", err) + } + + err = iswiftpkg.Start(testGlobals.confMap) + if nil != err { + t.Fatalf("iswifpkg.Start(testGlobals.confMap) failed: %v", err) + } + + authRequestHeaders = make(http.Header) + + authRequestHeaders["X-Auth-User"] = []string{testSwiftAuthUser} + authRequestHeaders["X-Auth-Key"] = []string{testSwiftAuthKey} + + authResponseHeaders, _, err = testDoHTTPRequest("GET", testGlobals.authURL, authRequestHeaders, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.authURL, authRequestHeaders, nil) failed: %v", err) + } + + testGlobals.authToken = authResponseHeaders.Get("X-Auth-Token") + testGlobals.accountURL = authResponseHeaders.Get("X-Storage-Url") + + expectedAccountURL = fmt.Sprintf("http://%s:%d/v1/%s", testIPAddr, testSwiftProxyTCPPort, testAccount) + + if expectedAccountURL != testGlobals.accountURL { + t.Fatalf("expectedAccountURL: %s but X-Storage-Url: %s", expectedAccountURL, testGlobals.accountURL) + } + + testGlobals.containerURL = testGlobals.accountURL + "/" + testContainer + + putAccountRequestHeaders = make(http.Header) + + putAccountRequestHeaders["X-Auth-Token"] = []string{testGlobals.authToken} + + _, _, err = testDoHTTPRequest("PUT", testGlobals.accountURL, putAccountRequestHeaders, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.accountURL, putAccountRequestHeaders, nil) failed: %v", err) + } + + putContainerRequestHeaders = make(http.Header) + + putContainerRequestHeaders["X-Auth-Token"] = []string{testGlobals.authToken} + + _, _, err = testDoHTTPRequest("PUT", testGlobals.containerURL, putContainerRequestHeaders, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.storageURL, putContainerRequestHeaders, nil) failed: %v", err) + } + + err = imgrpkg.Start(testGlobals.confMap) + if nil != err { + t.Fatalf("imgrpkg.Start(testGlobals.confMap) failed: %v", err) + } + + postAndPutVolumePayload = fmt.Sprintf("{\"StorageURL\":\"%s\",\"AuthToken\":\"%s\"}", testGlobals.containerURL, testGlobals.authToken) + + _, _, err = testDoHTTPRequest("POST", testGlobals.imgrHTTPServerURL+"/volume", nil, strings.NewReader(postAndPutVolumePayload)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"POST\", testGlobals.imgrHTTPServerURL+\"/volume\", nil, strings.NewReader(postAndPutVolumePayload)) failed: %v", err) + } + + _, _, err = testDoHTTPRequest("PUT", testGlobals.imgrHTTPServerURL+"/volume/"+testVolume, nil, strings.NewReader(postAndPutVolumePayload)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.imgrHTTPServerURL+\"/volume/\"+testVolume, nil, strings.NewReader(postAndPutVolumePayload)) failed: %v", err) + } +} + +func testTeardown(t *testing.T) { + var ( + err error + ) + + err = imgrpkg.Stop() + if nil != err { + t.Fatalf("imgrpkg.Stop() failed: %v", err) + } + + err = iswiftpkg.Stop() + if nil != err { + t.Fatalf("iswiftpkg.Stop() failed: %v", err) + } + + err = os.RemoveAll(testGlobals.tempDir) + if nil != err { + t.Fatalf("os.RemoveAll(testGlobals.tempDir) failed: %v", err) + } + + testGlobals = nil +} + +func testDoHTTPRequest(method string, url string, requestHeaders http.Header, requestBody io.Reader) (responseHeaders http.Header, responseBody []byte, err error) { + var ( + headerKey string + headerValues []string + httpRequest *http.Request + httpResponse *http.Response + ) + + httpRequest, err = http.NewRequest(method, url, requestBody) + if nil != err { + err = fmt.Errorf("http.NewRequest(\"%s\", \"%s\", nil) failed: %v", method, url, err) + return + } + + if nil != requestHeaders { + for headerKey, headerValues = range requestHeaders { + httpRequest.Header[headerKey] = headerValues + } + } + + httpResponse, err = http.DefaultClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("http.Do(httpRequest) failed: %v", err) + return + } + + responseBody, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) + return + } + + if (200 > httpResponse.StatusCode) || (299 < httpResponse.StatusCode) { + err = fmt.Errorf("httpResponse.StatusCode unexpected: %s", httpResponse.Status) + return + } + + responseHeaders = httpResponse.Header + + err = nil + return +} From c4e0ed0dc43b6d35c785d8d4906d2fa1dfa049e7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 2 Dec 2021 18:00:35 -0800 Subject: [PATCH 183/258] Decoupled Swift Auth plug-in from pkg version... so that make cover will work --- Makefile | 6 +++--- go.mod | 4 ++-- go.sum | 4 ++++ iauth/iauth-swift/iauth-swift.go | 4 +--- iclient/iclientpkg/utils_test.go | 2 +- imgr/imgrpkg/utils_test.go | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 3dadbd9b..9888920d 100644 --- a/Makefile +++ b/Makefile @@ -27,15 +27,15 @@ gobindirs = \ imgr \ iswift -godirsforci = $(gopkgdirs) $(goplugindirs) $(gobindirs); +godirsforci = $(gopkgdirs) $(gobindirs) godirpathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(godirsforci)) generatedfiles := \ coverage.coverprofile -all: version fmt pre-generate generate test build +all: version fmt pre-generate generate build test -ci: version fmt pre-generate generate test cover build +ci: version fmt pre-generate generate build test cover minimal: version pre-generate generate build diff --git a/go.mod b/go.mod index 5989fb71..a13d762f 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/mitchellh/mapstructure v1.4.2 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/ory/go-acc v0.2.6 // indirect github.com/ory/viper v1.7.5 // indirect github.com/pborman/uuid v1.2.1 // indirect @@ -97,7 +97,7 @@ require ( google.golang.org/grpc v1.40.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/ini.v1 v1.63.2 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/go.sum b/go.sum index a9aaa463..d79bd5c8 100644 --- a/go.sum +++ b/go.sum @@ -328,6 +328,8 @@ github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -863,6 +865,8 @@ gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/iauth/iauth-swift/iauth-swift.go b/iauth/iauth-swift/iauth-swift.go index e856d44b..ca4654e4 100644 --- a/iauth/iauth-swift/iauth-swift.go +++ b/iauth/iauth-swift/iauth-swift.go @@ -8,8 +8,6 @@ import ( "fmt" "net/http" "strings" - - "github.com/NVIDIA/proxyfs/version" ) type authInStruct struct { @@ -50,7 +48,7 @@ func PerformAuth(authInJSON string) (authToken string, storageURL string, err er authRequest.Header.Add("X-Auth-User", authIn.AuthUser) authRequest.Header.Add("X-Auth-Key", authIn.AuthKey) - authRequest.Header.Add("User-Agent", "iauth-swift "+version.ProxyFSVersion) + authRequest.Header.Add("User-Agent", "iauth-swift") authResponse, err = http.DefaultClient.Do(authRequest) if nil != err { diff --git a/iclient/iclientpkg/utils_test.go b/iclient/iclientpkg/utils_test.go index 996db82a..15c97c19 100644 --- a/iclient/iclientpkg/utils_test.go +++ b/iclient/iclientpkg/utils_test.go @@ -234,7 +234,7 @@ func testSetup(t *testing.T) { err = iswiftpkg.Start(testGlobals.confMap) if nil != err { - t.Fatalf("iswifpkg.Start(testGlobals.confMap) failed: %v", err) + t.Fatalf("iswiftpkg.Start(testGlobals.confMap) failed: %v", err) } authRequestHeaders = make(http.Header) diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index 6265488f..be19ca90 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -180,7 +180,7 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { err = iswiftpkg.Start(testGlobals.confMap) if nil != err { - t.Fatalf("iswifpkg.Start(testGlobals.confMap) failed: %v", err) + t.Fatalf("iswiftpkg.Start(testGlobals.confMap) failed: %v", err) } err = testDoAuth() From e4130780ce5bd747ae02be9fde5649f6dec3d2d8 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 2 Dec 2021 18:27:11 -0800 Subject: [PATCH 184/258] An attempt to avoid ports unavailable in test runner for GitHub --- iclient/iclientpkg/utils_test.go | 10 +++++----- imgr/imgrpkg/utils_test.go | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/iclient/iclientpkg/utils_test.go b/iclient/iclientpkg/utils_test.go index 15c97c19..c47ddd88 100644 --- a/iclient/iclientpkg/utils_test.go +++ b/iclient/iclientpkg/utils_test.go @@ -23,10 +23,10 @@ import ( const ( testIPAddr = "dev" - testRetryRPCPort = 32356 - testMgrHTTPServerPort = 15346 - testClientHTTPServerPort = 15347 - testSwiftProxyTCPPort = 8080 + testRetryRPCPort = 9101 + testMgrHTTPServerPort = 9102 + testClientHTTPServerPort = 9103 + testSwiftProxyTCPPort = 9104 testSwiftAuthUser = "test" testSwiftAuthKey = "test" testAccount = "AUTH_test" @@ -150,7 +150,7 @@ func testSetup(t *testing.T) { "ICLIENT.SwiftRetryExponentialBackoff=1.4", "ICLIENT.SwiftConnectionPoolSize=128", "ICLIENT.RetryRPCPublicIPAddr=dev", - "ICLIENT.RetryRPCPort=32356", + "ICLIENT.RetryRPCPort=" + fmt.Sprintf("%d", testRetryRPCPort), "ICLIENT.RetryRPCDeadlineIO=60s", "ICLIENT.RetryRPCKeepAlivePeriod=60s", "ICLIENT.RetryRPCCACertFilePath=" + testGlobals.caCertFile, diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index be19ca90..43bc1da5 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -22,9 +22,9 @@ import ( const ( testIPAddr = "127.0.0.1" - testRetryRPCPort = 32356 - testHTTPServerPort = 15346 - testSwiftProxyTCPPort = 8080 + testRetryRPCPort = 9201 + testHTTPServerPort = 9202 + testSwiftProxyTCPPort = 9203 testSwiftAuthUser = "test" testSwiftAuthKey = "test" testContainer = "testContainer" From a323ef87392b4dec2ce714aa7a34827c2d95f59d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 2 Dec 2021 18:36:48 -0800 Subject: [PATCH 185/258] Port #s weren't the problem... I was using the "dev" domain name for iclientpkg's tests --- iclient/iclientpkg/utils_test.go | 18 +++++++++--------- imgr/imgrpkg/utils_test.go | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/iclient/iclientpkg/utils_test.go b/iclient/iclientpkg/utils_test.go index c47ddd88..bda85152 100644 --- a/iclient/iclientpkg/utils_test.go +++ b/iclient/iclientpkg/utils_test.go @@ -22,11 +22,11 @@ import ( ) const ( - testIPAddr = "dev" - testRetryRPCPort = 9101 - testMgrHTTPServerPort = 9102 - testClientHTTPServerPort = 9103 - testSwiftProxyTCPPort = 9104 + testIPAddr = "127.0.0.1" // Don't use IPv6... the code doesn't properly "join" this with :port #s + testRetryRPCPort = 32356 + testMgrHTTPServerPort = 15346 + testClientHTTPServerPort = 15347 + testSwiftProxyTCPPort = 8080 testSwiftAuthUser = "test" testSwiftAuthKey = "test" testAccount = "AUTH_test" @@ -113,8 +113,8 @@ func testSetup(t *testing.T) { StreetAddress: []string{}, PostalCode: []string{}, }, - []string{testIPAddr}, - []net.IP{}, + []string{}, + []net.IP{net.ParseIP(testIPAddr)}, time.Hour, testGlobals.caCertPEMBlock, testGlobals.caKeyPEMBlock, @@ -142,14 +142,14 @@ func testSetup(t *testing.T) { "ICLIENT.FUSEAttrValidDuration=250ms", "ICLIENT.AuthPlugInPath=../../iauth/iauth-swift/iauth-swift.so", "ICLIENT.AuthPlugInEnvName=", - "ICLIENT.AuthPlugInEnvValue=" + fmt.Sprintf("{\"AuthURL\":\"http://dev:%d/auth/v1.0\"\\u002C\"AuthUser\":\"%s\"\\u002C\"AuthKey\":\"%s\"\\u002C\"Account\":\"%s\"\\u002C\"Container\":\"%s\"}", testSwiftProxyTCPPort, testSwiftAuthUser, testSwiftAuthKey, testAccount, testContainer), + "ICLIENT.AuthPlugInEnvValue=" + fmt.Sprintf("{\"AuthURL\":\"http://%s:%d/auth/v1.0\"\\u002C\"AuthUser\":\"%s\"\\u002C\"AuthKey\":\"%s\"\\u002C\"Account\":\"%s\"\\u002C\"Container\":\"%s\"}", testIPAddr, testSwiftProxyTCPPort, testSwiftAuthUser, testSwiftAuthKey, testAccount, testContainer), "ICLIENT.SwiftTimeout=10m", "ICLIENT.SwiftRetryLimit=4", "ICLIENT.SwiftRetryDelay=100ms", "ICLIENT.SwiftRetryDelayVariance=25", "ICLIENT.SwiftRetryExponentialBackoff=1.4", "ICLIENT.SwiftConnectionPoolSize=128", - "ICLIENT.RetryRPCPublicIPAddr=dev", + "ICLIENT.RetryRPCPublicIPAddr=" + testIPAddr, "ICLIENT.RetryRPCPort=" + fmt.Sprintf("%d", testRetryRPCPort), "ICLIENT.RetryRPCDeadlineIO=60s", "ICLIENT.RetryRPCKeepAlivePeriod=60s", diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index 43bc1da5..b40c6373 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -21,10 +21,10 @@ import ( ) const ( - testIPAddr = "127.0.0.1" - testRetryRPCPort = 9201 - testHTTPServerPort = 9202 - testSwiftProxyTCPPort = 9203 + testIPAddr = "127.0.0.1" // Don't use IPv6... the code doesn't properly "join" this with :port #s + testRetryRPCPort = 32356 + testHTTPServerPort = 15346 + testSwiftProxyTCPPort = 8080 testSwiftAuthUser = "test" testSwiftAuthKey = "test" testContainer = "testContainer" From 728315d070a1ededd68b6d57fc0ac7997c5a9047 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 3 Dec 2021 12:24:23 -0800 Subject: [PATCH 186/258] Test basic non-blocking success addThisLock() paths --- iclient/iclientpkg/lease_test.go | 105 ++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 3 deletions(-) diff --git a/iclient/iclientpkg/lease_test.go b/iclient/iclientpkg/lease_test.go index 5cbfb591..14f34ada 100644 --- a/iclient/iclientpkg/lease_test.go +++ b/iclient/iclientpkg/lease_test.go @@ -7,10 +7,20 @@ import ( "testing" ) +const ( + testInodeLockInodeNumberA = uint64(1) + testInodeLockInodeNumberB = uint64(2) +) + func TestLocks(t *testing.T) { var ( - err error - fissionErrChan chan error + err error + fissionErrChan chan error + inodeLockRequestA *inodeLockRequestStruct + inodeLockRequestB *inodeLockRequestStruct + // inodeLockRequestC *inodeLockRequestStruct + inodeHeldLock *inodeHeldLockStruct + ok bool ) testSetup(t) @@ -27,7 +37,96 @@ func TestLocks(t *testing.T) { t.Fatalf("startRPCHandler() failed: %v", err) } - t.Logf("TODO... actually test locks in func TestLocks()") + t.Logf("TODO... actually test locks in func TestLocks() below") + + inodeLockRequestA = newLockRequest() + inodeLockRequestA.inodeNumber = testInodeLockInodeNumberA + inodeLockRequestA.exclusive = false + inodeLockRequestA.addThisLock() + if len(inodeLockRequestA.locksHeld) != 1 { + t.Fatalf("len(inodeLockRequestA.locksHeld) (%v) should have been == 1", len(inodeLockRequestA.locksHeld)) + } + inodeHeldLock, ok = inodeLockRequestA.locksHeld[testInodeLockInodeNumberA] + if !ok { + t.Fatalf("inodeLockRequestA.locksHeld[testInodeLockInodeNumberA] returned !ok") + } + if inodeHeldLock.inode.inodeNumber != testInodeLockInodeNumberA { + t.Fatalf("inodeHeldLock.inode.inodeNumber (%v) != testInodeLockInodeNumberA (%v)", inodeHeldLock.inode.inodeNumber, testInodeLockInodeNumberA) + } + if inodeHeldLock.exclusive { + t.Fatalf("inodeHeldLock.exclusive should have been false") + } + + inodeLockRequestA.inodeNumber = testInodeLockInodeNumberB + inodeLockRequestA.exclusive = true + inodeLockRequestA.addThisLock() + if len(inodeLockRequestA.locksHeld) != 2 { + t.Fatalf("len(inodeLockRequestA.locksHeld) (%v) should have been == 2", len(inodeLockRequestA.locksHeld)) + } + inodeHeldLock, ok = inodeLockRequestA.locksHeld[testInodeLockInodeNumberA] + if !ok { + t.Fatalf("inodeLockRequestA.locksHeld[testInodeLockInodeNumberA] returned !ok") + } + if inodeHeldLock.inode.inodeNumber != testInodeLockInodeNumberA { + t.Fatalf("inodeHeldLock.inode.inodeNumber (%v) != testInodeLockInodeNumberA (%v)", inodeHeldLock.inode.inodeNumber, testInodeLockInodeNumberA) + } + if inodeHeldLock.exclusive { + t.Fatalf("inodeHeldLock.exclusive should have been false") + } + inodeHeldLock, ok = inodeLockRequestA.locksHeld[testInodeLockInodeNumberB] + if !ok { + t.Fatalf("inodeLockRequestA.locksHeld[testInodeLockInodeNumberB] returned !ok") + } + if inodeHeldLock.inode.inodeNumber != testInodeLockInodeNumberB { + t.Fatalf("inodeHeldLock.inode.inodeNumber (%v) != testInodeLockInodeNumberB (%v)", inodeHeldLock.inode.inodeNumber, testInodeLockInodeNumberB) + } + if !inodeHeldLock.exclusive { + t.Fatalf("inodeHeldLock.exclusive should have been true") + } + + inodeLockRequestB = newLockRequest() + inodeLockRequestB.inodeNumber = testInodeLockInodeNumberA + inodeLockRequestB.exclusive = false + inodeLockRequestB.addThisLock() + if len(inodeLockRequestB.locksHeld) != 1 { + t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 1", len(inodeLockRequestB.locksHeld)) + } + inodeHeldLock, ok = inodeLockRequestB.locksHeld[testInodeLockInodeNumberA] + if !ok { + t.Fatalf("inodeLockRequestB.locksHeld[testInodeLockInodeNumberA] returned !ok") + } + if inodeHeldLock.inode.inodeNumber != testInodeLockInodeNumberA { + t.Fatalf("inodeHeldLock.inode.inodeNumber (%v) != testInodeLockInodeNumberA (%v)", inodeHeldLock.inode.inodeNumber, testInodeLockInodeNumberA) + } + if inodeHeldLock.exclusive { + t.Fatalf("inodeHeldLock.exclusive should have been false") + } + + // inodeLockRequestC = newLockRequest() + // inodeLockRequestC.inodeNumber = testInodeLockInodeNumberB + // inodeLockRequestC.exclusive = false + // inodeLockRequestC.addThisLock() + // if len(inodeLockRequestC.locksHeld) != 0 { + // t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) + // } + + // inodeLockRequestC.exclusive = true + // inodeLockRequestC.addThisLock() + // if len(inodeLockRequestC.locksHeld) != 0 { + // t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) + // } + + inodeLockRequestA.unlockAll() + if len(inodeLockRequestA.locksHeld) != 0 { + t.Fatalf("len(inodeLockRequestA.locksHeld) (%v) should have been == 0", len(inodeLockRequestA.locksHeld)) + } + + inodeLockRequestB.unlockAll() + if len(inodeLockRequestB.locksHeld) != 0 { + t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) + } + + t.Logf("TODO... actually test locks in func TestLocks() above") err = stopRPCHandler() if nil != err { From 8c4fe4e1ecc032b98cfb1027126882d1ba7572bd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 3 Dec 2021 15:25:35 -0800 Subject: [PATCH 187/258] Flushed out the remaining iclientpkg lock sequences... disabling those that current fail --- iclient/iclientpkg/lease_test.go | 262 +++++++++++++++++++++++++++++-- 1 file changed, 246 insertions(+), 16 deletions(-) diff --git a/iclient/iclientpkg/lease_test.go b/iclient/iclientpkg/lease_test.go index 14f34ada..db118550 100644 --- a/iclient/iclientpkg/lease_test.go +++ b/iclient/iclientpkg/lease_test.go @@ -4,23 +4,106 @@ package iclientpkg import ( + "fmt" + "runtime" + "sync" "testing" + "time" ) const ( testInodeLockInodeNumberA = uint64(1) testInodeLockInodeNumberB = uint64(2) + testInodeLockInodeNumberC = uint64(3) + + testChildLockOperationDelay = time.Duration(100 * time.Millisecond) ) +type testChildLockStruct struct { + sync.WaitGroup // Signaled to tell testChildLock goroutine to release their lock and exit + exclusive bool + lockHeld bool + err error +} + +func testChildLockStart(exclusive bool) (testChildLock *testChildLockStruct) { + testChildLock = &testChildLockStruct{ + exclusive: exclusive, + lockHeld: false, + err: nil, + } + + testChildLock.Add(1) + + go testChildLock.gor() + + return +} + +func (testChildLock *testChildLockStruct) gor() { + var ( + inodeHeldLock *inodeHeldLockStruct + inodeLockRequest *inodeLockRequestStruct + ok bool + ) + + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = testInodeLockInodeNumberA + inodeLockRequest.exclusive = testChildLock.exclusive + inodeLockRequest.addThisLock() + + if len(inodeLockRequest.locksHeld) != 1 { + testChildLock.err = fmt.Errorf("len(inodeLockRequest.locksHeld) (%v) should have been == 1", len(inodeLockRequest.locksHeld)) + runtime.Goexit() + } + inodeHeldLock, ok = inodeLockRequest.locksHeld[testInodeLockInodeNumberA] + if !ok { + testChildLock.err = fmt.Errorf("inodeLockRequest.locksHeld[testInodeLockInodeNumberA] returned !ok") + runtime.Goexit() + } + if inodeHeldLock.inode.inodeNumber != testInodeLockInodeNumberA { + testChildLock.err = fmt.Errorf("inodeHeldLock.inode.inodeNumber (%v) != testInodeLockInodeNumberA (%v)", inodeHeldLock.inode.inodeNumber, testInodeLockInodeNumberA) + runtime.Goexit() + } + if inodeHeldLock.exclusive { + testChildLock.err = fmt.Errorf("inodeHeldLock.exclusive should have been false") + runtime.Goexit() + } + + testChildLock.lockHeld = true + + testChildLock.Wait() + + inodeLockRequest.unlockAll() + if len(inodeLockRequest.locksHeld) != 0 { + testChildLock.err = fmt.Errorf("len(inodeLockRequest.locksHeld) (%v) should have been == 0", len(inodeLockRequest.locksHeld)) + runtime.Goexit() + } + + testChildLock.lockHeld = false +} + +func (testChildLock *testChildLockStruct) delayAndCheckForError(t *testing.T) { + time.Sleep(testChildLockOperationDelay) + + if testChildLock.err != nil { + t.Fatal(testChildLock.err) + } +} + +func (testChildLock *testChildLockStruct) finish() { + testChildLock.Done() +} + func TestLocks(t *testing.T) { var ( + childLock [7]*testChildLockStruct err error fissionErrChan chan error + inodeHeldLock *inodeHeldLockStruct inodeLockRequestA *inodeLockRequestStruct inodeLockRequestB *inodeLockRequestStruct - // inodeLockRequestC *inodeLockRequestStruct - inodeHeldLock *inodeHeldLockStruct - ok bool + ok bool ) testSetup(t) @@ -37,7 +120,7 @@ func TestLocks(t *testing.T) { t.Fatalf("startRPCHandler() failed: %v", err) } - t.Logf("TODO... actually test locks in func TestLocks() below") + t.Logf("Verify we can get a shared lock on currently unlocked testInodeLockInodeNumberA") inodeLockRequestA = newLockRequest() inodeLockRequestA.inodeNumber = testInodeLockInodeNumberA @@ -57,6 +140,8 @@ func TestLocks(t *testing.T) { t.Fatalf("inodeHeldLock.exclusive should have been false") } + t.Logf("Verify we can add in an exclusive lock on currently unlocked testInodeLockInodeNumberB") + inodeLockRequestA.inodeNumber = testInodeLockInodeNumberB inodeLockRequestA.exclusive = true inodeLockRequestA.addThisLock() @@ -84,6 +169,8 @@ func TestLocks(t *testing.T) { t.Fatalf("inodeHeldLock.exclusive should have been true") } + t.Logf("Verify we can get a shared lock on currently shared locked testInodeLockInodeNumberA") + inodeLockRequestB = newLockRequest() inodeLockRequestB.inodeNumber = testInodeLockInodeNumberA inodeLockRequestB.exclusive = false @@ -102,31 +189,174 @@ func TestLocks(t *testing.T) { t.Fatalf("inodeHeldLock.exclusive should have been false") } - // inodeLockRequestC = newLockRequest() - // inodeLockRequestC.inodeNumber = testInodeLockInodeNumberB - // inodeLockRequestC.exclusive = false - // inodeLockRequestC.addThisLock() - // if len(inodeLockRequestC.locksHeld) != 0 { + inodeLockRequestB.unlockAll() + if len(inodeLockRequestB.locksHeld) != 0 { + t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) + } + + t.Logf("TODO: Verify attempting an exclusive lock on a currently shared locked testInodeLockInodeNumberA while holding a shared lock on testInodeLockInodeNumberC fails releasing the shared lock on testInodeLockInodeNumberC") + + // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberC + // inodeLockRequestB.exclusive = false + // inodeLockRequestB.addThisLock() + // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberA + // inodeLockRequestB.exclusive = true + // inodeLockRequestB.addThisLock() + // if len(inodeLockRequestB.locksHeld) != 0 { // t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) // } - // inodeLockRequestC.exclusive = true - // inodeLockRequestC.addThisLock() - // if len(inodeLockRequestC.locksHeld) != 0 { + t.Logf("TODO: Verify attempting a shared lock on a currently exclusively locked testInodeLockInodeNumberB while holding a shared lock on testInodeLockInodeNumberC fails releasing the shared lock on testInodeLockInodeNumberC") + + // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberC + // inodeLockRequestB.exclusive = false + // inodeLockRequestB.addThisLock() + // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberB + // inodeLockRequestB.exclusive = false + // inodeLockRequestB.addThisLock() + // if len(inodeLockRequestB.locksHeld) != 0 { // t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) // } + t.Logf("Verify we can release the shared lock on testInodeLockInodeNumberB and the exclusive lock on testInodeLockInodeNumberB") + inodeLockRequestA.unlockAll() if len(inodeLockRequestA.locksHeld) != 0 { t.Fatalf("len(inodeLockRequestA.locksHeld) (%v) should have been == 0", len(inodeLockRequestA.locksHeld)) } - inodeLockRequestB.unlockAll() - if len(inodeLockRequestB.locksHeld) != 0 { - t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) + // Perform a sequence of lock requests that all contend on testInodeLockInodeNumberA. + // + // The sequence will be: + // + // 0: Shared - this should be granted since testInodeLockInodeNumberA was previously unlocked + // 1: Shared - this should be granted since testInodeLockInodeNumberA is currently share locked + // 2: Exclusive - this should block since testInodeLockInodeNumberA is currently share locked + // - this should unblock when both 0 & 1 are unlocked + // 3: Shared - this should block since there is already a prior exclusive lock request + // - this should unblock when 2 is unlocked + // 4: Shared - this should block since there is already a prior exclusive lock request + // - this should unblock when 2 is unlocked + // 5: Exclusive - this should block since testInodeLockInodeNumberA is currently share locked + // - this should unblock when both 3 & 4 are unlocked + // 6: Exclusive - this should block since testInodeLockInodeNumberA is currently share locked + // - this should unblock when 5 is unlocked + + childLock[0] = testChildLockStart(false) + childLock[0].delayAndCheckForError(t) + childLock[1] = testChildLockStart(false) + childLock[1].delayAndCheckForError(t) + childLock[2] = testChildLockStart(true) + childLock[2].delayAndCheckForError(t) + childLock[3] = testChildLockStart(false) + childLock[3].delayAndCheckForError(t) + childLock[4] = testChildLockStart(false) + childLock[4].delayAndCheckForError(t) + childLock[5] = testChildLockStart(true) + childLock[5].delayAndCheckForError(t) + childLock[6] = testChildLockStart(true) + childLock[0].delayAndCheckForError(t) + + if !childLock[0].lockHeld || + !childLock[1].lockHeld || + childLock[2].lockHeld || + childLock[3].lockHeld || + childLock[4].lockHeld || + childLock[5].lockHeld || + childLock[6].lockHeld { + t.Fatalf("Initial sequence should have had testInodeLockInodeNumberA only held by 0 & 1") + } + + childLock[0].finish() + childLock[0].delayAndCheckForError(t) + + if childLock[0].lockHeld || + !childLock[1].lockHeld || + childLock[2].lockHeld || + childLock[3].lockHeld || + childLock[4].lockHeld || + childLock[5].lockHeld || + childLock[6].lockHeld { + t.Fatalf("After 0 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 1") } - t.Logf("TODO... actually test locks in func TestLocks() above") + childLock[1].finish() + childLock[1].delayAndCheckForError(t) + + // if childLock[0].lockHeld || + // childLock[1].lockHeld || + // !childLock[2].lockHeld || + // childLock[3].lockHeld || + // childLock[4].lockHeld || + // childLock[5].lockHeld || + // childLock[6].lockHeld { + // t.Fatalf("After 1 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 2") + // } + + childLock[2].finish() + childLock[2].delayAndCheckForError(t) + + // if childLock[0].lockHeld || + // childLock[1].lockHeld || + // childLock[2].lockHeld || + // !childLock[3].lockHeld || + // !childLock[4].lockHeld || + // childLock[5].lockHeld || + // childLock[6].lockHeld { + // t.Fatalf("After 2 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 3 & 4") + // } + + childLock[3].finish() + childLock[3].delayAndCheckForError(t) + + // if childLock[0].lockHeld || + // childLock[1].lockHeld || + // childLock[2].lockHeld || + // childLock[3].lockHeld || + // !childLock[4].lockHeld || + // childLock[5].lockHeld || + // childLock[6].lockHeld { + // t.Fatalf("After 3 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 4") + // } + + childLock[4].finish() + childLock[4].delayAndCheckForError(t) + + // if childLock[0].lockHeld || + // childLock[1].lockHeld || + // childLock[2].lockHeld || + // childLock[3].lockHeld || + // childLock[4].lockHeld || + // !childLock[5].lockHeld || + // childLock[6].lockHeld { + // t.Fatalf("After 4 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 5") + // } + + childLock[5].finish() + childLock[5].delayAndCheckForError(t) + + // if childLock[0].lockHeld || + // childLock[1].lockHeld || + // childLock[2].lockHeld || + // childLock[3].lockHeld || + // childLock[4].lockHeld || + // childLock[5].lockHeld || + // !childLock[6].lockHeld { + // t.Fatalf("After 5 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 6") + // } + + childLock[6].finish() + childLock[6].delayAndCheckForError(t) + + // if childLock[0].lockHeld || + // childLock[1].lockHeld || + // childLock[2].lockHeld || + // childLock[3].lockHeld || + // childLock[4].lockHeld || + // childLock[5].lockHeld || + // childLock[6].lockHeld { + // t.Fatalf("After 6 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by none") + // } err = stopRPCHandler() if nil != err { From d7a9e0059b1819049021e561dcf215494299e30d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 3 Dec 2021 17:02:48 -0800 Subject: [PATCH 188/258] Cleaned up t.Logf() (TODO) calls in iclientpkg/lease_test.go --- iclient/iclientpkg/lease_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iclient/iclientpkg/lease_test.go b/iclient/iclientpkg/lease_test.go index db118550..40379934 100644 --- a/iclient/iclientpkg/lease_test.go +++ b/iclient/iclientpkg/lease_test.go @@ -120,7 +120,7 @@ func TestLocks(t *testing.T) { t.Fatalf("startRPCHandler() failed: %v", err) } - t.Logf("Verify we can get a shared lock on currently unlocked testInodeLockInodeNumberA") + // Verify we can get a shared lock on currently unlocked testInodeLockInodeNumberA inodeLockRequestA = newLockRequest() inodeLockRequestA.inodeNumber = testInodeLockInodeNumberA @@ -140,7 +140,7 @@ func TestLocks(t *testing.T) { t.Fatalf("inodeHeldLock.exclusive should have been false") } - t.Logf("Verify we can add in an exclusive lock on currently unlocked testInodeLockInodeNumberB") + // Verify we can add in an exclusive lock on currently unlocked testInodeLockInodeNumberB inodeLockRequestA.inodeNumber = testInodeLockInodeNumberB inodeLockRequestA.exclusive = true @@ -169,7 +169,7 @@ func TestLocks(t *testing.T) { t.Fatalf("inodeHeldLock.exclusive should have been true") } - t.Logf("Verify we can get a shared lock on currently shared locked testInodeLockInodeNumberA") + // Verify we can get a shared lock on currently shared locked testInodeLockInodeNumberA inodeLockRequestB = newLockRequest() inodeLockRequestB.inodeNumber = testInodeLockInodeNumberA @@ -194,7 +194,7 @@ func TestLocks(t *testing.T) { t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) } - t.Logf("TODO: Verify attempting an exclusive lock on a currently shared locked testInodeLockInodeNumberA while holding a shared lock on testInodeLockInodeNumberC fails releasing the shared lock on testInodeLockInodeNumberC") + // Verify attempting an exclusive lock on a currently shared locked testInodeLockInodeNumberA while holding a shared lock on testInodeLockInodeNumberC fails releasing the shared lock on testInodeLockInodeNumberC // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberC // inodeLockRequestB.exclusive = false @@ -206,7 +206,7 @@ func TestLocks(t *testing.T) { // t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) // } - t.Logf("TODO: Verify attempting a shared lock on a currently exclusively locked testInodeLockInodeNumberB while holding a shared lock on testInodeLockInodeNumberC fails releasing the shared lock on testInodeLockInodeNumberC") + // Verify attempting a shared lock on a currently exclusively locked testInodeLockInodeNumberB while holding a shared lock on testInodeLockInodeNumberC fails releasing the shared lock on testInodeLockInodeNumberC // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberC // inodeLockRequestB.exclusive = false @@ -218,7 +218,7 @@ func TestLocks(t *testing.T) { // t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) // } - t.Logf("Verify we can release the shared lock on testInodeLockInodeNumberB and the exclusive lock on testInodeLockInodeNumberB") + // Verify we can release the shared lock on testInodeLockInodeNumberB and the exclusive lock on testInodeLockInodeNumberB inodeLockRequestA.unlockAll() if len(inodeLockRequestA.locksHeld) != 0 { From 30f790c9cbbe2713217ee802d66941f7c3747370 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 6 Dec 2021 16:30:49 -0800 Subject: [PATCH 189/258] Fixed most of the remaining iclientpkg lock semantics ...need to complete the deadlock avoidance case... --- iclient/iclientpkg/fission.go | 24 ----- iclient/iclientpkg/lease.go | 168 +++++++++++++++++++++++-------- iclient/iclientpkg/lease_test.go | 112 ++++++++++----------- 3 files changed, 181 insertions(+), 123 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 3b3935e6..809a6b0a 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -3883,25 +3883,16 @@ func doRenameCommon(oldDirInodeNumber uint64, oldName string, newDirInodeNumber renamedInode *inodeStruct replacedInode *inodeStruct ) - logInfof("UNDO: entered doRenameCommon(oldDirInodeNumber: %v, oldName: %s, newDirInodeNumber: %v, newName: %s)", oldDirInodeNumber, oldName, newDirInodeNumber, newName) - defer func() { - logTracef("UNDO: leaving doRenameCommon() with errno: %v", errno) - }() Retry: - logTracef("UNDO: Reached doRenameCommon() Point A") inodeLockRequest = newLockRequest() inodeLockRequest.inodeNumber = oldDirInodeNumber inodeLockRequest.exclusive = true - logTracef("UNDO: Reached doRenameCommon() Point A.1") inodeLockRequest.addThisLock() - logTracef("UNDO: Reached doRenameCommon() Point A.2") if len(inodeLockRequest.locksHeld) == 0 { - logTracef("UNDO: Reached doRenameCommon() Point A.3") performInodeLockRetryDelay() goto Retry } - logTracef("UNDO: Reached doRenameCommon() Point A.4") oldDirInode = lookupInode(oldDirInodeNumber) if nil == oldDirInode { @@ -3909,7 +3900,6 @@ Retry: errno = syscall.ENOENT return } - logTracef("UNDO: Reached doRenameCommon() Point A.5") if nil == oldDirInode.inodeHeadV1 { err = oldDirInode.populateInodeHeadV1() @@ -3919,14 +3909,12 @@ Retry: return } } - logTracef("UNDO: Reached doRenameCommon() Point A.6") if oldDirInode.inodeHeadV1.InodeType != ilayout.InodeTypeDir { inodeLockRequest.unlockAll() errno = syscall.ENOTDIR return } - logTracef("UNDO: Reached doRenameCommon() Point A.7") if oldDirInode.payload == nil { err = oldDirInode.oldPayload() @@ -3936,7 +3924,6 @@ Retry: return } } - logTracef("UNDO: Reached doRenameCommon() Point A.8") directoryEntryValueV1AsValue, ok, err = oldDirInode.payload.GetByKey(oldName) if nil != err { @@ -3947,7 +3934,6 @@ Retry: errno = syscall.ENOENT return } - logTracef("UNDO: Reached doRenameCommon() Point A.9") directoryEntryValueV1, ok = directoryEntryValueV1AsValue.(*ilayout.DirectoryEntryValueV1Struct) if !ok { @@ -3961,14 +3947,12 @@ Retry: performInodeLockRetryDelay() goto Retry } - logTracef("UNDO: Reached doRenameCommon() Point A.A") renamedInode = lookupInode(directoryEntryValueV1.InodeNumber) if nil == renamedInode { inodeLockRequest.unlockAll() goto Retry } - logTracef("UNDO: Reached doRenameCommon() Point A.B") if nil == renamedInode.inodeHeadV1 { err = renamedInode.populateInodeHeadV1() @@ -3979,7 +3963,6 @@ Retry: } } - logTracef("UNDO: Reached doRenameCommon() Point B") if oldDirInodeNumber == newDirInodeNumber { newDirInode = oldDirInode @@ -4027,7 +4010,6 @@ Retry: } } } - logTracef("UNDO: Reached doRenameCommon() Point C") directoryEntryValueV1AsValue, ok, err = newDirInode.payload.GetByKey(newName) if nil != err { @@ -4070,10 +4052,8 @@ Retry: } else { replacedInode = nil } - logTracef("UNDO: Reached doRenameCommon() Point D") if renamedInode.inodeHeadV1.InodeType == ilayout.InodeTypeDir { - logTracef("UNDO: Reached doRenameCommon() Point E") if replacedInode != nil { inodeLockRequest.unlockAll() errno = syscall.EISDIR @@ -4158,9 +4138,7 @@ Retry: logFatalf("newDirInode.payload.Put(newName,) returned !ok") } } else { - logTracef("UNDO: Reached doRenameCommon() Point F") if replacedInode != nil { - logTracef("UNDO: Reached doRenameCommon() Point G") replacedInode.dirty = true replacedInode.inodeHeadV1.ModificationTime = startTime @@ -4217,7 +4195,6 @@ Retry: logFatalf("newDirInode.payload.PatchByKey(newName,) returned !ok") } } else { - logTracef("UNDO: Reached doRenameCommon() Point H") renamedInode.dirty = true renamedInode.inodeHeadV1.ModificationTime = startTime @@ -4265,7 +4242,6 @@ Retry: } } } - logTracef("UNDO: Reached doRenameCommon() Point I") if replacedInode == nil { if oldDirInodeNumber == newDirInodeNumber { diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index 7fdbfc13..be261cd1 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -100,7 +100,6 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { leaseResponse *imgrpkg.LeaseResponseStruct ok bool ) - logTracef("UNDO: entered addThisLock() with .inodeNumber: %v .exclusive: %v", inodeLockRequest.inodeNumber, inodeLockRequest.exclusive) globals.Lock() @@ -162,12 +161,10 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { } if inode.requestList.Len() != 0 { - logTracef("UNDO: ok... addThisLock() found that we cannot immediately grant the lock as others are blocked...") // At lease one other inodeLockRequestStruct is blocked, so this one must either block or, to avoid deadlock, release other held locks and retry if len(inodeLockRequest.locksHeld) != 0 { // We must avoid deadlock by releasing other held locks and return - logTracef("UNDO: ok... addThisLock() needs to avoid deadlock, so we must release our other locks and trigger a retry") globals.Unlock() @@ -176,7 +173,6 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { return } - logTracef("UNDO: ok... addThisLock() will simply block this request") inodeLockRequest.Add(1) inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) @@ -191,7 +187,6 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { if inode.heldList.Len() != 0 { if inodeLockRequest.exclusive { // Lock is held, so this exclusive inodeLockRequestStruct must block - logTracef("UNDO: ok... addThisLock() needs to block on .inodeNunber: %v .exclusive: TRUEv while inode.heldList.Len() == %v", inodeLockRequest.inodeNumber, inode.heldList.Len()) inodeLockRequest.Add(1) @@ -372,7 +367,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { default: logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } - } else { + } else { // !inodeLockRequest.exclusive switch inode.leaseState { case inodeLeaseStateNone: inodeLockRequest.listElement = inode.requestList.PushFront(inodeLockRequest) @@ -416,7 +411,6 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock - logInfof("UNDO: @ CheckRequestList: loop in addThisLock()") CheckRequestList: @@ -534,67 +528,155 @@ func (inodeLockRequest *inodeLockRequestStruct) unlockAll() { for _, inodeHeldLock = range inodeLockRequest.locksHeld { inode = inodeHeldLock.inode - logTracef("UNDO: entered unlockAll()... found an inodeHeldLock with .inode..inodeNumber: %v .exclusive: %v", inodeHeldLock.inode.inodeNumber, inodeHeldLock.exclusive) _ = inode.heldList.Remove(inodeHeldLock.listElement) - if inode.requestList.Len() == 0 { - if inode.heldList.Len() == 0 { + if inode.heldList.Len() == 0 { + // This was the last lock holder + + if inode.requestList.Len() == 0 { + // No pending lock requests exist... should we delete inode? + if inode.markedForDelete { releaseList = append(releaseList, inode) delete(globals.inodeTable, inode.inodeNumber) } - } - } else { - if inodeHeldLock.exclusive { - logInfof("UNDO: @ We can unblock at least one blockedInodeLockRequest") - // We can unblock at least one blockedInodeLockRequest + } else { + // We can unblock one or more pending lock requests blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) - _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) - blockedInodeLockRequest.listElement = nil + if blockedInodeLockRequest.exclusive { + switch inode.leaseState { + case inodeLeaseStateSharedGranted: + inode.leaseState = inodeLeaseStateSharedPromoting - inodeHeldLock = &inodeHeldLockStruct{ - inode: inode, - inodeLockRequest: blockedInodeLockRequest, - exclusive: blockedInodeLockRequest.exclusive, - } + globals.Unlock() + + leaseRequest = &imgrpkg.LeaseRequestStruct{ + MountID: globals.mountID, + InodeNumber: blockedInodeLockRequest.inodeNumber, + LeaseRequestType: imgrpkg.LeaseRequestTypePromote, + } + leaseResponse = &imgrpkg.LeaseResponseStruct{} - inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) - blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + err = rpcLease(leaseRequest, leaseResponse) + if nil != err { + logFatal(err) + } - blockedInodeLockRequest.Done() + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypePromoted { + logFatalf("TODO: for now, we don't handle a Lease Request actually failing") + } - if !blockedInodeLockRequest.exclusive { - // We can also unblock following blockedInodeLockRequest's that are !.exclusive + globals.Lock() - CheckRequestList: + inode.leaseState = inodeLeaseStateExclusiveGranted - if inode.requestList.Front() != nil { - blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) + _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) + blockedInodeLockRequest.listElement = nil - if !blockedInodeLockRequest.exclusive { - // We can also unblock next blockedInodeLockRequest + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: blockedInodeLockRequest, + exclusive: true, + } - _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) - blockedInodeLockRequest.listElement = nil + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + + blockedInodeLockRequest.Done() + case inodeLeaseStateSharedReleasing: + logFatalf("TODO: for now, we don't handle a Shared Lease releasing") + case inodeLeaseStateSharedExpired: + logFatalf("TODO: for now, we don't handle a Shared Lease expiring") + case inodeLeaseStateExclusiveGranted: + _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) + blockedInodeLockRequest.listElement = nil + + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: blockedInodeLockRequest, + exclusive: true, + } + + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + + blockedInodeLockRequest.Done() + case inodeLeaseStateExclusiveDemoting: + logFatalf("TODO: for now, we don't handle an Exclusive Lease demoting") + case inodeLeaseStateExclusiveReleasing: + logFatalf("TODO: for now, we don't handle an Exclusive Lease releasing") + case inodeLeaseStateExclusiveExpired: + logFatalf("TODO: for now, we don't handle an Exclusive Lease expiring") + default: + logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) + } + } else { // !blockedInodeLockRequest.exclusive + switch inode.leaseState { + case inodeLeaseStateSharedGranted: - inodeHeldLock = &inodeHeldLockStruct{ - inode: inode, - inodeLockRequest: blockedInodeLockRequest, - exclusive: false, + GrantBlockedInodeLockRequestWhileHoldingSharedLease: + + _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) + blockedInodeLockRequest.listElement = nil + + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: blockedInodeLockRequest, + exclusive: false, + } + + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + + blockedInodeLockRequest.Done() + + if inode.requestList.Len() == 0 { + blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) + + if !blockedInodeLockRequest.exclusive { + goto GrantBlockedInodeLockRequestWhileHoldingSharedLease } + } + case inodeLeaseStateSharedReleasing: + logFatalf("TODO: for now, we don't handle a Shared Lease releasing") + case inodeLeaseStateSharedExpired: + logFatalf("TODO: for now, we don't handle a Shared Lease expiring") + case inodeLeaseStateExclusiveGranted: + + GrantBlockedInodeLockRequestWhileHoldingExclusiveLease: + + _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) + blockedInodeLockRequest.listElement = nil - inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) - blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: blockedInodeLockRequest, + exclusive: false, + } + + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock - blockedInodeLockRequest.Done() + blockedInodeLockRequest.Done() - // Now go back and check for more + if inode.requestList.Len() != 0 { + blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) - goto CheckRequestList + if !blockedInodeLockRequest.exclusive { + goto GrantBlockedInodeLockRequestWhileHoldingExclusiveLease + } } + case inodeLeaseStateExclusiveDemoting: + logFatalf("TODO: for now, we don't handle an Exclusive Lease demoting") + case inodeLeaseStateExclusiveReleasing: + logFatalf("TODO: for now, we don't handle an Exclusive Lease releasing") + case inodeLeaseStateExclusiveExpired: + logFatalf("TODO: for now, we don't handle an Exclusive Lease expiring") + default: + logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } } } diff --git a/iclient/iclientpkg/lease_test.go b/iclient/iclientpkg/lease_test.go index 40379934..40840a90 100644 --- a/iclient/iclientpkg/lease_test.go +++ b/iclient/iclientpkg/lease_test.go @@ -65,8 +65,8 @@ func (testChildLock *testChildLockStruct) gor() { testChildLock.err = fmt.Errorf("inodeHeldLock.inode.inodeNumber (%v) != testInodeLockInodeNumberA (%v)", inodeHeldLock.inode.inodeNumber, testInodeLockInodeNumberA) runtime.Goexit() } - if inodeHeldLock.exclusive { - testChildLock.err = fmt.Errorf("inodeHeldLock.exclusive should have been false") + if inodeHeldLock.exclusive != testChildLock.exclusive { + testChildLock.err = fmt.Errorf("inodeHeldLock.exclusive should have been %v", testChildLock.exclusive) runtime.Goexit() } @@ -283,80 +283,80 @@ func TestLocks(t *testing.T) { childLock[1].finish() childLock[1].delayAndCheckForError(t) - // if childLock[0].lockHeld || - // childLock[1].lockHeld || - // !childLock[2].lockHeld || - // childLock[3].lockHeld || - // childLock[4].lockHeld || - // childLock[5].lockHeld || - // childLock[6].lockHeld { - // t.Fatalf("After 1 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 2") - // } + if childLock[0].lockHeld || + childLock[1].lockHeld || + !childLock[2].lockHeld || + childLock[3].lockHeld || + childLock[4].lockHeld || + childLock[5].lockHeld || + childLock[6].lockHeld { + t.Fatalf("After 1 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 2") + } childLock[2].finish() childLock[2].delayAndCheckForError(t) - // if childLock[0].lockHeld || - // childLock[1].lockHeld || - // childLock[2].lockHeld || - // !childLock[3].lockHeld || - // !childLock[4].lockHeld || - // childLock[5].lockHeld || - // childLock[6].lockHeld { - // t.Fatalf("After 2 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 3 & 4") - // } + if childLock[0].lockHeld || + childLock[1].lockHeld || + childLock[2].lockHeld || + !childLock[3].lockHeld || + !childLock[4].lockHeld || + childLock[5].lockHeld || + childLock[6].lockHeld { + t.Fatalf("After 2 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 3 & 4") + } childLock[3].finish() childLock[3].delayAndCheckForError(t) - // if childLock[0].lockHeld || - // childLock[1].lockHeld || - // childLock[2].lockHeld || - // childLock[3].lockHeld || - // !childLock[4].lockHeld || - // childLock[5].lockHeld || - // childLock[6].lockHeld { - // t.Fatalf("After 3 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 4") - // } + if childLock[0].lockHeld || + childLock[1].lockHeld || + childLock[2].lockHeld || + childLock[3].lockHeld || + !childLock[4].lockHeld || + childLock[5].lockHeld || + childLock[6].lockHeld { + t.Fatalf("After 3 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 4") + } childLock[4].finish() childLock[4].delayAndCheckForError(t) - // if childLock[0].lockHeld || - // childLock[1].lockHeld || - // childLock[2].lockHeld || - // childLock[3].lockHeld || - // childLock[4].lockHeld || - // !childLock[5].lockHeld || - // childLock[6].lockHeld { - // t.Fatalf("After 4 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 5") - // } + if childLock[0].lockHeld || + childLock[1].lockHeld || + childLock[2].lockHeld || + childLock[3].lockHeld || + childLock[4].lockHeld || + !childLock[5].lockHeld || + childLock[6].lockHeld { + t.Fatalf("After 4 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 5") + } childLock[5].finish() childLock[5].delayAndCheckForError(t) - // if childLock[0].lockHeld || - // childLock[1].lockHeld || - // childLock[2].lockHeld || - // childLock[3].lockHeld || - // childLock[4].lockHeld || - // childLock[5].lockHeld || - // !childLock[6].lockHeld { - // t.Fatalf("After 5 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 6") - // } + if childLock[0].lockHeld || + childLock[1].lockHeld || + childLock[2].lockHeld || + childLock[3].lockHeld || + childLock[4].lockHeld || + childLock[5].lockHeld || + !childLock[6].lockHeld { + t.Fatalf("After 5 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by 6") + } childLock[6].finish() childLock[6].delayAndCheckForError(t) - // if childLock[0].lockHeld || - // childLock[1].lockHeld || - // childLock[2].lockHeld || - // childLock[3].lockHeld || - // childLock[4].lockHeld || - // childLock[5].lockHeld || - // childLock[6].lockHeld { - // t.Fatalf("After 6 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by none") - // } + if childLock[0].lockHeld || + childLock[1].lockHeld || + childLock[2].lockHeld || + childLock[3].lockHeld || + childLock[4].lockHeld || + childLock[5].lockHeld || + childLock[6].lockHeld { + t.Fatalf("After 6 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by none") + } err = stopRPCHandler() if nil != err { From 6b15696c72f6bccf9268c33e8083b3fb11079a8c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 7 Dec 2021 10:13:38 -0800 Subject: [PATCH 190/258] Finished iclient lock functionality Note that lease RPC failures (+ demotions + revocations) not yet handled. Also fixed bad OpenHandle bug... If two FHs were created for a since Inode, a subsequent Lookup-by-InodeNumber would only find the 2nd. The fix is simply to only Lookup-by-FissionFH... and not deal with the unnecessary complexity of tracking all FHs for a given InodeNumber. Instead, we now simply keep a count of # of FissionFH's for each Inode. --- iclient/iclientpkg/fission.go | 17 +- iclient/iclientpkg/globals.go | 12 +- iclient/iclientpkg/inode.go | 42 +-- iclient/iclientpkg/lease.go | 426 +++++++++++++++---------------- iclient/iclientpkg/lease_test.go | 42 +-- 5 files changed, 248 insertions(+), 291 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 809a6b0a..63fd542f 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -571,6 +571,7 @@ Retry: symLinkInode = &inodeStruct{ inodeNumber: symLinkInodeNumber, dirty: true, + openCount: 0, markedForDelete: false, leaseState: inodeLeaseStateNone, listElement: nil, @@ -776,6 +777,7 @@ Retry: childDirInode = &inodeStruct{ inodeNumber: childDirInodeNumber, dirty: true, + openCount: 0, markedForDelete: false, leaseState: inodeLeaseStateNone, listElement: nil, @@ -1584,7 +1586,7 @@ Retry: goto Retry } - openHandle = lookupOpenHandleByFissionFH(readIn.FH) + openHandle = lookupOpenHandle(readIn.FH) if nil == openHandle { inodeLockRequest.unlockAll() readOut = nil @@ -1955,7 +1957,7 @@ Retry: goto Retry } - openHandle = lookupOpenHandleByFissionFH(writeIn.FH) + openHandle = lookupOpenHandle(writeIn.FH) if nil == openHandle { inodeLockRequest.unlockAll() writeOut = nil @@ -2137,7 +2139,7 @@ func (dummy *globalsStruct) DoRelease(inHeader *fission.InHeader, releaseIn *fis globals.stats.DoReleaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - openHandle = lookupOpenHandleByFissionFH(releaseIn.FH) + openHandle = lookupOpenHandle(releaseIn.FH) if nil == openHandle { errno = syscall.EBADF return @@ -2596,7 +2598,7 @@ Retry: goto Retry } - openHandle = lookupOpenHandleByFissionFH(flushIn.FH) + openHandle = lookupOpenHandle(flushIn.FH) if nil == openHandle { inodeLockRequest.unlockAll() errno = syscall.EBADF @@ -2803,7 +2805,7 @@ func (dummy *globalsStruct) DoReadDir(inHeader *fission.InHeader, readDirIn *fis globals.stats.DoReadDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - openHandle = lookupOpenHandleByFissionFH(readDirIn.FH) + openHandle = lookupOpenHandle(readDirIn.FH) if nil == openHandle { readDirOut = nil errno = syscall.EBADF @@ -2969,7 +2971,7 @@ func (dummy *globalsStruct) DoReleaseDir(inHeader *fission.InHeader, releaseDirI globals.stats.DoReleaseDirUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - openHandle = lookupOpenHandleByFissionFH(releaseDirIn.FH) + openHandle = lookupOpenHandle(releaseDirIn.FH) if nil == openHandle { errno = syscall.EBADF return @@ -3262,6 +3264,7 @@ Retry: fileInode = &inodeStruct{ inodeNumber: fileInodeNumber, dirty: true, + openCount: 0, markedForDelete: false, leaseState: inodeLeaseStateNone, listElement: nil, @@ -3517,7 +3520,7 @@ func (dummy *globalsStruct) DoReadDirPlus(inHeader *fission.InHeader, readDirPlu globals.stats.DoReadDirPlusUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - openHandle = lookupOpenHandleByFissionFH(readDirPlusIn.FH) + openHandle = lookupOpenHandle(readDirPlusIn.FH) if nil == openHandle { readDirPlusOut = nil errno = syscall.EBADF diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index be56f42a..b986ed0c 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -113,7 +113,8 @@ type fileInodeFlusherStruct struct { type inodeStruct struct { inodeNumber uint64 // dirty bool // - markedForDelete bool // If true, remove from globalsStruct.inodeTable upon last dereference + openCount uint64 // + markedForDelete bool // If true, remove from globalsStruct.inodeTable once openCount == 0 leaseState inodeLeaseStateType // listElement *list.Element // Maintains position in globalsStruct.{shared|exclusive}LeaseLRU heldList *list.List // List of granted inodeHeldLockStruct's @@ -259,8 +260,7 @@ type globalsStruct struct { readCacheLRU *list.List // LRU-ordered list of readCacheLineStruct.listElement's inodeTable map[uint64]*inodeStruct // inodePayloadCache sortedmap.BPlusTreeCache // - openHandleMapByInodeNumber map[uint64]*openHandleStruct // Key == openHandleStruct.inodeNumber - openHandleMapByFissionFH map[uint64]*openHandleStruct // Key == openHandleStruct.fissionFH + openHandleMap map[uint64]*openHandleStruct // Key == openHandleStruct.fissionFH sharedLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateSharedGranted exclusiveLeaseLRU *list.List // LRU-ordered list of inodeStruct.listElement's in or transitioning to inodeLeaseStateExclusiveGranted httpServer *http.Server // @@ -524,8 +524,7 @@ func initializeGlobals(confMap conf.ConfMap, fissionErrChan chan error) (err err globals.inodeTable = make(map[uint64]*inodeStruct) globals.inodePayloadCache = sortedmap.NewBPlusTreeCache(globals.config.InodePayloadEvictLowLimit, globals.config.InodePayloadEvictHighLimit) - globals.openHandleMapByInodeNumber = make(map[uint64]*openHandleStruct) - globals.openHandleMapByFissionFH = make(map[uint64]*openHandleStruct) + globals.openHandleMap = make(map[uint64]*openHandleStruct) globals.sharedLeaseLRU = list.New() globals.exclusiveLeaseLRU = list.New() @@ -593,8 +592,7 @@ func uninitializeGlobals() (err error) { globals.inodeTable = nil globals.inodePayloadCache = nil - globals.openHandleMapByInodeNumber = nil - globals.openHandleMapByFissionFH = nil + globals.openHandleMap = nil globals.sharedLeaseLRU = nil globals.exclusiveLeaseLRU = nil diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 0ef9ba98..e47bf8d2 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -339,9 +339,9 @@ func lookupInode(inodeNumber uint64) (inode *inodeStruct) { return } -// createOpenHandle allocates an openHandleStruct and inserts it into the globals openHandle -// maps by both inodeNumber and fissionFH. Note that the fissionFlags* fields all default to -// false. Callers are expected to modify as necessary. +// createOpenHandle allocates an openHandleStruct and inserts it into the globals.openHandleMap. +// +// Note that the fissionFlags* fields all default to false. Callers are expected to modify as necessary. // func createOpenHandle(inodeNumber uint64) (openHandle *openHandleStruct) { openHandle = &openHandleStruct{ @@ -354,8 +354,7 @@ func createOpenHandle(inodeNumber uint64) (openHandle *openHandleStruct) { globals.Lock() - globals.openHandleMapByInodeNumber[openHandle.inodeNumber] = openHandle - globals.openHandleMapByFissionFH[openHandle.fissionFH] = openHandle + globals.openHandleMap[openHandle.fissionFH] = openHandle globals.Unlock() @@ -369,47 +368,24 @@ func (openHandle *openHandleStruct) destroy() { globals.Lock() - _, ok = globals.openHandleMapByInodeNumber[openHandle.inodeNumber] + _, ok = globals.openHandleMap[openHandle.fissionFH] if !ok { - logFatalf("globals.openHandleMapByInodeNumber[openHandle.inodeNumber] returned !ok") + logFatalf("globals.openHandleMap[openHandle.fissionFH] returned !ok") } - _, ok = globals.openHandleMapByFissionFH[openHandle.fissionFH] - if !ok { - logFatalf("globals.openHandleMapByFissionFH[openHandle.fissionFH] returned !ok") - } - - delete(globals.openHandleMapByInodeNumber, openHandle.inodeNumber) - delete(globals.openHandleMapByFissionFH, openHandle.fissionFH) + delete(globals.openHandleMap, openHandle.fissionFH) globals.Unlock() } -func lookupOpenHandleByInodeNumber(inodeNumber uint64) (openHandle *openHandleStruct) { - var ( - ok bool - ) - - globals.Lock() - - openHandle, ok = globals.openHandleMapByInodeNumber[inodeNumber] - if !ok { - openHandle = nil - } - - globals.Unlock() - - return -} - -func lookupOpenHandleByFissionFH(fissionFH uint64) (openHandle *openHandleStruct) { +func lookupOpenHandle(fissionFH uint64) (openHandle *openHandleStruct) { var ( ok bool ) globals.Lock() - openHandle, ok = globals.openHandleMapByFissionFH[fissionFH] + openHandle, ok = globals.openHandleMap[fissionFH] if !ok { openHandle = nil } diff --git a/iclient/iclientpkg/lease.go b/iclient/iclientpkg/lease.go index be261cd1..5ccac3e2 100644 --- a/iclient/iclientpkg/lease.go +++ b/iclient/iclientpkg/lease.go @@ -62,7 +62,7 @@ func performInodeLockRetryDelay() { variance int64 ) - variance = rand.Int63n(int64(globals.config.InodeLockRetryDelay) & int64(globals.config.InodeLockRetryDelayVariance) / int64(100)) + variance = rand.Int63n(int64(globals.config.InodeLockRetryDelay) * int64(globals.config.InodeLockRetryDelayVariance) / int64(100)) if (variance % 2) == 0 { delay = time.Duration(int64(globals.config.InodeLockRetryDelay) + variance) @@ -111,6 +111,8 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { logFatalf("*inodeLockRequestStruct)addThisLock() called with .inodeNumber already present in .locksHeld map") } + // First get inode in the proper state required for this inodeLockRequest + inode, ok = globals.inodeTable[inodeLockRequest.inodeNumber] if ok { switch inode.leaseState { @@ -138,6 +140,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inode = &inodeStruct{ inodeNumber: inodeLockRequest.inodeNumber, dirty: false, + openCount: 0, markedForDelete: false, leaseState: inodeLeaseStateNone, listElement: nil, @@ -160,70 +163,159 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { globals.inodeTable[inodeLockRequest.inodeNumber] = inode } - if inode.requestList.Len() != 0 { - // At lease one other inodeLockRequestStruct is blocked, so this one must either block or, to avoid deadlock, release other held locks and retry + if inodeLockRequest.exclusive { + if inode.leaseState == inodeLeaseStateExclusiveGranted { + // Attempt to grant exclusive inodeLockRequest - if len(inodeLockRequest.locksHeld) != 0 { - // We must avoid deadlock by releasing other held locks and return + if inode.heldList.Len() == 0 { + // We can now grant exclusive inodeLockRequest [Case 1] - globals.Unlock() + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: inodeLockRequest, + exclusive: true, + } - inodeLockRequest.unlockAll() + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock - return - } + globals.Unlock() - inodeLockRequest.Add(1) + return + } - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) + // Block exlusive inodeLockRequest unless .locksHeld is non-empty - globals.Unlock() + if len(inodeLockRequest.locksHeld) == 0 { + // We must block exclusive inodeLockRequest [Case 1] - inodeLockRequest.Wait() + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - return - } + inodeLockRequest.Add(1) - if inode.heldList.Len() != 0 { - if inodeLockRequest.exclusive { - // Lock is held, so this exclusive inodeLockRequestStruct must block + globals.Unlock() - inodeLockRequest.Add(1) + inodeLockRequest.Wait() - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) + return + } else { + // We must avoid deadlock for this exclusive inodeLockRequest with non-empty .locksHeld [Case 1] - globals.Unlock() + globals.Unlock() - inodeLockRequest.Wait() + inodeLockRequest.unlockAll() - return + return + } } + } else { // !inodeLockRequest.exclusive + if (inode.leaseState == inodeLeaseStateSharedGranted) || (inode.leaseState == inodeLeaseStateExclusiveGranted) { + // Attempt to grant shared inodeLockRequest + + if inode.heldList.Len() == 0 { + // We can now grant shared inodeLockRequest [Case 1] + + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: inodeLockRequest, + exclusive: false, + } + + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + + globals.Unlock() - inodeHeldLock, ok = inode.heldList.Front().Value.(*inodeHeldLockStruct) - if !ok { - logFatalf("inode.heldList.Front().Value.(*inodeHeldLockStruct) returned !ok") + return + } + + if inode.requestList.Len() == 0 { + inodeHeldLock = inode.heldList.Front().Value.(*inodeHeldLockStruct) + + if !inodeHeldLock.exclusive { + // We can now grant shared inodeLockRequest [Case 2] + + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: inodeLockRequest, + exclusive: false, + } + + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + + globals.Unlock() + + return + } + } + + // Block shared inodeLockRequest unless .locksHeld is non-empty + + if len(inodeLockRequest.locksHeld) == 0 { + // We must block shared inodeLockRequest + + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) + + inodeLockRequest.Add(1) + + globals.Unlock() + + inodeLockRequest.Wait() + + return + } else { + // We must avoid deadlock for this shared inodeLockRequest with non-empty .locksHeld + + globals.Unlock() + + inodeLockRequest.unlockAll() + + return + } } + } - if inodeHeldLock.exclusive { - // Lock is held exclusively, so this inodeLockRequestStruct must block + // If we reach here, inode was not in the proper state required for this inodeLockRequest - inodeLockRequest.Add(1) + if (inode.leaseState != inodeLeaseStateNone) && (inode.leaseState != inodeLeaseStateSharedGranted) { + if len(inodeLockRequest.locksHeld) == 0 { + // We must block inodeLockRequest + // + // Ultimately, .leaseState will transition triggering the servicing of .requestList inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) + inodeLockRequest.Add(1) + globals.Unlock() inodeLockRequest.Wait() + return + } else { + // We must avoid deadlock for this inodeLockRequest with non-empty .locksHeld + + globals.Unlock() + + inodeLockRequest.unlockAll() + return } } + // If we reach here, we are in one of three states: + // + // inodeLockRequest.exclusive == true && inode.leaseState == inodeLeaseStateNone + // inodeLockRequest.exclusive == true && inode.leaseState == inodeLeaseStateSharedGranted + // inodeLockRequest.exclusive == false && inode.leaseState == inodeLeaseStateNone + if inodeLockRequest.exclusive { - switch inode.leaseState { - case inodeLeaseStateNone: - inodeLockRequest.listElement = inode.requestList.PushFront(inodeLockRequest) + if inode.leaseState == inodeLeaseStateNone { + // We need to issue a imgrpkg.LeaseRequestTypeExclusive + inode.leaseState = inodeLeaseStateExclusiveRequested + inode.listElement = globals.exclusiveLeaseLRU.PushBack(inode) globals.Unlock() @@ -248,12 +340,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inode.leaseState = inodeLeaseStateExclusiveGranted - if inode.requestList.Front() != inodeLockRequest.listElement { - logFatalf("inode.requestList.Front() != inodeLockRequest.listElement") - } - - _ = inode.requestList.Remove(inodeLockRequest.listElement) - inodeLockRequest.listElement = nil + // We can now grant exclusive inodeLockRequest [Case 2] inodeHeldLock = &inodeHeldLockStruct{ inode: inode, @@ -263,17 +350,35 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + } else { // inode.leaseState == inodeLeaseStateSharedGranted + if inode.heldList.Len() != 0 { + if len(inodeLockRequest.locksHeld) == 0 { + // We must block exclusive inodeLockRequest [Case 2] + + inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) + + inodeLockRequest.Add(1) + + globals.Unlock() + + inodeLockRequest.Wait() + + return + } else { + // We must avoid deadlock for this exclusive inodeLockRequest with non-empty .locksHeld [Case 2] + + globals.Unlock() + + inodeLockRequest.unlockAll() + + return + } + } + + // We need to issue a imgrpkg.LeaseRequestTypePromote - globals.Unlock() - case inodeLeaseStateSharedRequested: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateSharedGranted: - inodeLockRequest.listElement = inode.requestList.PushFront(inodeLockRequest) inode.leaseState = inodeLeaseStateSharedPromoting + _ = globals.sharedLeaseLRU.Remove(inode.listElement) inode.listElement = globals.exclusiveLeaseLRU.PushBack(inode) @@ -299,12 +404,7 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inode.leaseState = inodeLeaseStateExclusiveGranted - if inode.requestList.Front() != inodeLockRequest.listElement { - logFatalf("inode.requestList.Front() != inodeLockRequest.listElement") - } - - _ = inode.requestList.Remove(inodeLockRequest.listElement) - inodeLockRequest.listElement = nil + // We can now grant exclusive inodeLockRequest [Case 3] inodeHeldLock = &inodeHeldLockStruct{ inode: inode, @@ -314,201 +414,79 @@ func (inodeLockRequest *inodeLockRequestStruct) addThisLock() { inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock - - globals.Unlock() - case inodeLeaseStateSharedPromoting: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateSharedReleasing: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateSharedExpired: - // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up - globals.Unlock() - inodeLockRequest.unlockAll() - case inodeLeaseStateExclusiveRequested: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateExclusiveGranted: - // We can immediately grant the exclusive inodeLockRequestStruct - inodeHeldLock = &inodeHeldLockStruct{ - inode: inode, - inodeLockRequest: inodeLockRequest, - exclusive: true, - } - inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) - inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock - globals.Unlock() - case inodeLeaseStateExclusiveDemoting: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateExclusiveReleasing: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateExclusiveExpired: - // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up - globals.Unlock() - inodeLockRequest.unlockAll() - default: - logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } - } else { // !inodeLockRequest.exclusive - switch inode.leaseState { - case inodeLeaseStateNone: - inodeLockRequest.listElement = inode.requestList.PushFront(inodeLockRequest) - inode.leaseState = inodeLeaseStateSharedRequested - inode.listElement = globals.sharedLeaseLRU.PushBack(inode) + } else { // !inodeLockRequest.exclusive && inode.leaseState == inodeLeaseStateNone + // We need to issue a imgrpkg.LeaseRequestTypeShared - globals.Unlock() + inode.leaseState = inodeLeaseStateSharedRequested - leaseRequest = &imgrpkg.LeaseRequestStruct{ - MountID: globals.mountID, - InodeNumber: inodeLockRequest.inodeNumber, - LeaseRequestType: imgrpkg.LeaseRequestTypeShared, - } - leaseResponse = &imgrpkg.LeaseResponseStruct{} + inode.listElement = globals.sharedLeaseLRU.PushBack(inode) - err = rpcLease(leaseRequest, leaseResponse) - if nil != err { - logFatal(err) - } + globals.Unlock() - if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeShared { - logFatalf("TODO: for now, we don't handle a Lease Request actually failing") - } + leaseRequest = &imgrpkg.LeaseRequestStruct{ + MountID: globals.mountID, + InodeNumber: inodeLockRequest.inodeNumber, + LeaseRequestType: imgrpkg.LeaseRequestTypeShared, + } + leaseResponse = &imgrpkg.LeaseResponseStruct{} - globals.Lock() + err = rpcLease(leaseRequest, leaseResponse) + if nil != err { + logFatal(err) + } - inode.leaseState = inodeLeaseStateSharedGranted + if leaseResponse.LeaseResponseType != imgrpkg.LeaseResponseTypeShared { + logFatalf("TODO: for now, we don't handle a Lease Request actually failing") + } - if inode.requestList.Front() != inodeLockRequest.listElement { - logFatalf("inode.requestList.Front() != inodeLockRequest.listElement") - } + globals.Lock() - _ = inode.requestList.Remove(inodeLockRequest.listElement) - inodeLockRequest.listElement = nil + inode.leaseState = inodeLeaseStateSharedGranted - inodeHeldLock = &inodeHeldLockStruct{ - inode: inode, - inodeLockRequest: inodeLockRequest, - exclusive: false, - } + // We can now grant shared inodeLockRequest [Case 3] - inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) - inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: inodeLockRequest, + exclusive: false, + } - CheckRequestList: + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock - if inode.requestList.Front() != nil { - blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) + // Check to see if there are any grantable blocked shared inodeLockRequest's - if !blockedInodeLockRequest.exclusive { - // We can also unblock blockedInodeLockRequest + CheckRequestList: - _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) - blockedInodeLockRequest.listElement = nil + if inode.requestList.Front() != nil { + blockedInodeLockRequest = inode.requestList.Front().Value.(*inodeLockRequestStruct) - inodeHeldLock = &inodeHeldLockStruct{ - inode: inode, - inodeLockRequest: blockedInodeLockRequest, - exclusive: false, - } + if !blockedInodeLockRequest.exclusive { + // We can now also grant blockedInodeLockRequest - inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) - blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock + _ = inode.requestList.Remove(blockedInodeLockRequest.listElement) + blockedInodeLockRequest.listElement = nil - blockedInodeLockRequest.Done() + inodeHeldLock = &inodeHeldLockStruct{ + inode: inode, + inodeLockRequest: blockedInodeLockRequest, + exclusive: false, + } - // Now go back and check for more + inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) + blockedInodeLockRequest.locksHeld[blockedInodeLockRequest.inodeNumber] = inodeHeldLock - goto CheckRequestList - } - } + blockedInodeLockRequest.Done() - globals.Unlock() - case inodeLeaseStateSharedRequested: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateSharedGranted: - // We can immediately grant the shared inodeLockRequestStruct - inodeHeldLock = &inodeHeldLockStruct{ - inode: inode, - inodeLockRequest: inodeLockRequest, - exclusive: false, - } - inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) - inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock - globals.Unlock() - case inodeLeaseStateSharedPromoting: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateSharedReleasing: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateSharedExpired: - // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up - globals.Unlock() - inodeLockRequest.unlockAll() - case inodeLeaseStateExclusiveRequested: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateExclusiveGranted: - // We can immediately grant the shared inodeLockRequestStruct - inodeHeldLock = &inodeHeldLockStruct{ - inode: inode, - inodeLockRequest: inodeLockRequest, - exclusive: false, + // Now go back and check for more + + goto CheckRequestList } - inodeHeldLock.listElement = inode.heldList.PushBack(inodeHeldLock) - inodeLockRequest.locksHeld[inodeLockRequest.inodeNumber] = inodeHeldLock - globals.Unlock() - case inodeLeaseStateExclusiveDemoting: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateExclusiveReleasing: - // Let whatever entity receives imgrpkg.LeaseResponseType* complete this inodeLockRequestStruct - inodeLockRequest.Add(1) - inodeLockRequest.listElement = inode.requestList.PushBack(inodeLockRequest) - globals.Unlock() - inodeLockRequest.Wait() - case inodeLeaseStateExclusiveExpired: - // Our mount point has expired... so let owner of inodeLockRequestStruct clean-up - globals.Unlock() - inodeLockRequest.unlockAll() - default: - logFatalf("switch inode.leaseState unexpected: %v", inode.leaseState) } } + + globals.Unlock() } // unlockAll is called to explicitly release all locks listed in the locksHeld map. diff --git a/iclient/iclientpkg/lease_test.go b/iclient/iclientpkg/lease_test.go index 40840a90..26c19187 100644 --- a/iclient/iclientpkg/lease_test.go +++ b/iclient/iclientpkg/lease_test.go @@ -189,6 +189,8 @@ func TestLocks(t *testing.T) { t.Fatalf("inodeHeldLock.exclusive should have been false") } + // Verify we can release the shared lock on testInodeLockInodeNumberA + inodeLockRequestB.unlockAll() if len(inodeLockRequestB.locksHeld) != 0 { t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) @@ -196,29 +198,29 @@ func TestLocks(t *testing.T) { // Verify attempting an exclusive lock on a currently shared locked testInodeLockInodeNumberA while holding a shared lock on testInodeLockInodeNumberC fails releasing the shared lock on testInodeLockInodeNumberC - // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberC - // inodeLockRequestB.exclusive = false - // inodeLockRequestB.addThisLock() - // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberA - // inodeLockRequestB.exclusive = true - // inodeLockRequestB.addThisLock() - // if len(inodeLockRequestB.locksHeld) != 0 { - // t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) - // } + inodeLockRequestB.inodeNumber = testInodeLockInodeNumberC + inodeLockRequestB.exclusive = false + inodeLockRequestB.addThisLock() + inodeLockRequestB.inodeNumber = testInodeLockInodeNumberA + inodeLockRequestB.exclusive = true + inodeLockRequestB.addThisLock() + if len(inodeLockRequestB.locksHeld) != 0 { + t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) + } // Verify attempting a shared lock on a currently exclusively locked testInodeLockInodeNumberB while holding a shared lock on testInodeLockInodeNumberC fails releasing the shared lock on testInodeLockInodeNumberC - // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberC - // inodeLockRequestB.exclusive = false - // inodeLockRequestB.addThisLock() - // inodeLockRequestB.inodeNumber = testInodeLockInodeNumberB - // inodeLockRequestB.exclusive = false - // inodeLockRequestB.addThisLock() - // if len(inodeLockRequestB.locksHeld) != 0 { - // t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) - // } + inodeLockRequestB.inodeNumber = testInodeLockInodeNumberC + inodeLockRequestB.exclusive = false + inodeLockRequestB.addThisLock() + inodeLockRequestB.inodeNumber = testInodeLockInodeNumberB + inodeLockRequestB.exclusive = false + inodeLockRequestB.addThisLock() + if len(inodeLockRequestB.locksHeld) != 0 { + t.Fatalf("len(inodeLockRequestB.locksHeld) (%v) should have been == 0", len(inodeLockRequestB.locksHeld)) + } - // Verify we can release the shared lock on testInodeLockInodeNumberB and the exclusive lock on testInodeLockInodeNumberB + // Verify we can release the shared lock on testInodeLockInodeNumberA and the exclusive lock on testInodeLockInodeNumberB inodeLockRequestA.unlockAll() if len(inodeLockRequestA.locksHeld) != 0 { @@ -355,7 +357,7 @@ func TestLocks(t *testing.T) { childLock[4].lockHeld || childLock[5].lockHeld || childLock[6].lockHeld { - t.Fatalf("After 6 releases their lock, the sequence should have had testInodeLockInodeNumberA only held by none") + t.Fatalf("After 6 releases their lock, the sequence should have had testInodeLockInodeNumberA held by none") } err = stopRPCHandler() From 51c558affce432d6e412a4ee442af8ae5576ec5b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 8 Dec 2021 10:08:34 -0800 Subject: [PATCH 191/258] Hard-coded Read/Write flags to TRUE in iclientpkg Linux VFS and/or fuse.ko appears to mask them to FALSE... and enforce access before upcall'ing iclientpkg --- iclient/iclientpkg/fission.go | 22 +++++++--------------- iclient/iclientpkg/inode.go | 11 ++++++----- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 63fd542f..75862191 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1510,11 +1510,11 @@ Retry: logFatal(err) } - openHandle = createOpenHandle(inode.inodeNumber) - - openHandle.fissionFlagsAppend = (openIn.Flags & syscall.O_APPEND) == syscall.O_APPEND - openHandle.fissionFlagsRead = ((openIn.Flags & syscall.O_ACCMODE) == syscall.O_RDONLY) || ((openIn.Flags & syscall.O_ACCMODE) == syscall.O_RDWR) - openHandle.fissionFlagsWrite = ((openIn.Flags & syscall.O_ACCMODE) == syscall.O_RDWR) || ((openIn.Flags & syscall.O_ACCMODE) == syscall.O_WRONLY) + openHandle = createOpenHandle( + inode.inodeNumber, + (openIn.Flags&syscall.O_APPEND) == syscall.O_APPEND, + ((openIn.Flags&syscall.O_ACCMODE) == syscall.O_RDONLY) || ((openIn.Flags&syscall.O_ACCMODE) == syscall.O_RDWR), + ((openIn.Flags&syscall.O_ACCMODE) == syscall.O_RDWR) || ((openIn.Flags&syscall.O_ACCMODE) == syscall.O_WRONLY)) openOut = &fission.OpenOut{ FH: openHandle.fissionFH, @@ -2757,11 +2757,7 @@ Retry: logFatal(err) } - openHandle = createOpenHandle(inode.inodeNumber) - - openHandle.fissionFlagsAppend = false - openHandle.fissionFlagsRead = true - openHandle.fissionFlagsWrite = false + openHandle = createOpenHandle(inode.inodeNumber, false, true, false) openDirOut = &fission.OpenDirOut{ FH: openHandle.fissionFH, @@ -3351,11 +3347,7 @@ Retry: logFatal(err) } - openHandle = createOpenHandle(fileInode.inodeNumber) - - openHandle.fissionFlagsAppend = false - openHandle.fissionFlagsRead = false - openHandle.fissionFlagsWrite = true + openHandle = createOpenHandle(fileInode.inodeNumber, false, false, true) createOut = &fission.CreateOut{ EntryOut: fission.EntryOut{ diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index e47bf8d2..6515eaa4 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -341,15 +341,16 @@ func lookupInode(inodeNumber uint64) (inode *inodeStruct) { // createOpenHandle allocates an openHandleStruct and inserts it into the globals.openHandleMap. // -// Note that the fissionFlags* fields all default to false. Callers are expected to modify as necessary. +// Note that fissionFlags{Read|Write} are forced to be TRUE per the behavior of Linux VFS +// and/or fuse.ko choosing to mask these during Do{Create|Open|OpenDir}() upcalls. // -func createOpenHandle(inodeNumber uint64) (openHandle *openHandleStruct) { +func createOpenHandle(inodeNumber uint64, fissionFlagsAppend bool, fissionFlagsRead bool, fissionFlagsWrite bool) (openHandle *openHandleStruct) { openHandle = &openHandleStruct{ inodeNumber: inodeNumber, fissionFH: fetchNonce(), - fissionFlagsAppend: false, // To be filled in by caller - fissionFlagsRead: false, // To be filled in by caller - fissionFlagsWrite: false, // To be filled in by caller + fissionFlagsAppend: fissionFlagsAppend, + fissionFlagsRead: true, // Should be fissionFlagsRead but Linux VFS and/or fuse.ko masks these + fissionFlagsWrite: true, // Should be fissionFlagsWrite but Linux VFS and/or fuse.ko masks these } globals.Lock() From ceaadc9cbe98f7acb7cd5ff270aa33ae8e3d2cfc Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 8 Dec 2021 16:39:05 -0800 Subject: [PATCH 192/258] Fixed ReadPlan construction bug in iclient when curOffset > extent's key (starting offset) --- iclient/iclientpkg/fission.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 75862191..ed0ac965 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1537,6 +1537,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R extentMapEntryIndexV1Min int // First entry at or just before where readIn.Offset may reside extentMapEntryKeyV1 uint64 extentMapEntryKeyV1AsKey sortedmap.Key + extentMapEntrySkipSize uint64 extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct extentMapEntryValueV1AsValue sortedmap.Value inode *inodeStruct @@ -1724,16 +1725,18 @@ Retry: } } + extentMapEntrySkipSize = curOffset - extentMapEntryKeyV1 + extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) if !ok { logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1) returned !ok") } - if remainingSize <= extentMapEntryValueV1.Length { + if remainingSize <= (extentMapEntryValueV1.Length - extentMapEntrySkipSize) { readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ Length: remainingSize, ObjectNumber: extentMapEntryValueV1.ObjectNumber, - ObjectOffset: extentMapEntryValueV1.ObjectOffset, + ObjectOffset: extentMapEntryValueV1.ObjectOffset + extentMapEntrySkipSize, }) curOffset += remainingSize @@ -1743,13 +1746,13 @@ Retry: } readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ - Length: extentMapEntryValueV1.Length, + Length: extentMapEntryValueV1.Length - extentMapEntrySkipSize, ObjectNumber: extentMapEntryValueV1.ObjectNumber, - ObjectOffset: extentMapEntryValueV1.ObjectOffset, + ObjectOffset: extentMapEntryValueV1.ObjectOffset + extentMapEntrySkipSize, }) - curOffset += extentMapEntryValueV1.Length - remainingSize -= extentMapEntryValueV1.Length + curOffset += extentMapEntryValueV1.Length - extentMapEntrySkipSize + remainingSize -= extentMapEntryValueV1.Length - extentMapEntrySkipSize } if remainingSize > 0 { From 763af71d6d76936cbe9eacf30e3b546cc368c3e3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 9 Dec 2021 17:53:40 -0800 Subject: [PATCH 193/258] Fixed unmapExtent()... but git clone of proxyfs.git still fails --- iclient/iclientpkg/fission.go | 19 +--- iclient/iclientpkg/inode.go | 187 ++++++++++++++++++++-------------- 2 files changed, 114 insertions(+), 92 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index ed0ac965..1d3890e5 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -2078,10 +2078,7 @@ Retry: func (dummy *globalsStruct) DoStatFS(inHeader *fission.InHeader) (statFSOut *fission.StatFSOut, errno syscall.Errno) { var ( - err error - startTime time.Time = time.Now() - volumeStatusRequest *imgrpkg.VolumeStatusRequestStruct - volumeStatusResponse *imgrpkg.VolumeStatusResponseStruct + startTime time.Time = time.Now() ) logTracef("==> DoStatFS(inHeader: %+v)", inHeader) @@ -2093,22 +2090,12 @@ func (dummy *globalsStruct) DoStatFS(inHeader *fission.InHeader) (statFSOut *fis globals.stats.DoStatFSUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - volumeStatusRequest = &imgrpkg.VolumeStatusRequestStruct{ - MountID: globals.mountID, - } - volumeStatusResponse = &imgrpkg.VolumeStatusResponseStruct{} - - err = rpcVolumeStatus(volumeStatusRequest, volumeStatusResponse) - if nil != err { - logFatal(err) - } - statFSOut = &fission.StatFSOut{ KStatFS: fission.KStatFS{ - Blocks: (volumeStatusResponse.ObjectSize + uint64(globals.config.FUSEBlockSize) - 1) / uint64(globals.config.FUSEBlockSize), + Blocks: math.MaxUint64, BFree: math.MaxUint64, BAvail: math.MaxUint64, - Files: volumeStatusResponse.NumInodes, + Files: math.MaxUint64, FFree: math.MaxUint64, BSize: globals.config.FUSEBlockSize, FRSize: globals.config.FUSEBlockSize, diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 6515eaa4..b9a3e93e 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -982,16 +982,18 @@ func (fileInode *inodeStruct) recordExtent(startingFileOffset uint64, length uin func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint64) { var ( err error + extentLengthOfTail uint64 extentLengthToTrim uint64 extentMapEntryKeyV1 uint64 extentMapEntryKeyV1AsKey sortedmap.Key extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct extentMapEntryValueV1AsValue sortedmap.Value + extentObjectNumberOfTail uint64 + extentObjectOffsetOfTail uint64 found bool index int layoutMapEntry layoutMapEntryStruct ok bool - subsequentFileOffset uint64 ) if length == 0 { @@ -1004,7 +1006,7 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint } if !found { - // See if there is an extent just to the left of the extent to unmap + // See if there is an extent starting just to the left of the unmap range extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = fileInode.payload.GetByIndex(index) if nil != err { @@ -1012,7 +1014,7 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint } if ok { - // Potentially trim the extent just to the left of the extent to unmap + // There is an extent starting just to the left of the unmap range extentMapEntryKeyV1, ok = extentMapEntryKeyV1AsKey.(uint64) if !ok { @@ -1023,50 +1025,92 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") } - if extentMapEntryKeyV1+extentMapEntryValueV1.Length > startingFileOffset { - extentLengthToTrim = (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) - startingFileOffset + layoutMapEntry, ok = fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] + if !ok { + logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") + } - layoutMapEntry, ok = fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] - if !ok { - logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") - } + if (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) > startingFileOffset { + // The extent starting just to the left of the unmap range either needs + // to have a hole punched in it or it needs to be trimmed on the right + + if (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) > (startingFileOffset + length) { + // The extent starting just to the left of the unmap range needs to have a hole punched in it + + extentLengthOfTail = (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) - (startingFileOffset + length) + extentObjectNumberOfTail = extentMapEntryValueV1.ObjectNumber + extentObjectOffsetOfTail = extentMapEntryValueV1.ObjectOffset + (extentMapEntryValueV1.Length - extentLengthOfTail) + + extentMapEntryValueV1.Length = startingFileOffset - extentMapEntryKeyV1 - if extentLengthToTrim > layoutMapEntry.bytesReferenced { - logFatalf("extentLengthToTrim > layoutMapEntry.bytesReferenced") - } else if (extentLengthToTrim == layoutMapEntry.bytesReferenced) && (extentMapEntryValueV1.ObjectNumber != fileInode.putObjectNumber) { - delete(fileInode.layoutMap, extentMapEntryValueV1.ObjectNumber) + ok, err = fileInode.payload.PatchByIndex(index, extentMapEntryValueV1) + if nil != err { + logFatalf("fileInode.payload.PatchByIndex() failed: %v", err) + } + if !ok { + logFatalf("fileInode.payload.PatchByIndex() returned !ok") + } - fileInode.superBlockInodeObjectCountAdjustment-- - fileInode.superBlockInodeObjectSizeAdjustment -= int64(layoutMapEntry.objectSize) - fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) + extentMapEntryKeyV1 = startingFileOffset + length + + extentMapEntryValueV1 = &ilayout.ExtentMapEntryValueV1Struct{ + Length: extentLengthOfTail, + ObjectNumber: extentObjectNumberOfTail, + ObjectOffset: extentObjectOffsetOfTail, + } + + ok, err = fileInode.payload.Put(extentMapEntryKeyV1, extentMapEntryValueV1) + if nil != err { + logFatalf("fileInode.payload.Put() failed: %v", err) + } + if !ok { + logFatalf("fileInode.payload.Put() returned !ok") + } - fileInode.dereferencedObjectNumberArray = append(fileInode.dereferencedObjectNumberArray, extentMapEntryValueV1.ObjectNumber) - } else { - layoutMapEntry.bytesReferenced -= extentLengthToTrim + layoutMapEntry.bytesReferenced -= length fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry - fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(length) + + // And since we've now fully accounted for the unmap range, we are done + + return } + // If we reach here, the extent starting just to the left of the unmap range needs to be trimmed on the right + + extentLengthToTrim = (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) - startingFileOffset + + extentMapEntryValueV1.Length -= extentLengthToTrim + ok, err = fileInode.payload.PatchByIndex(index, extentMapEntryValueV1) if nil != err { - logFatalf("fileInode.payload.GetByIndex(index) failed: %v", err) + logFatalf("fileInode.payload.PatchByIndex() failed: %v", err) } if !ok { - logFatalf("fileInode.payload.GetByIndex(index) returned !ok") + logFatalf("fileInode.payload.PatchByIndex() returned !ok") } - } - // Adjust index to start at the next extent that might overlap with the extent to unmap + layoutMapEntry.bytesReferenced -= extentLengthToTrim + + fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry + + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) - index++ + // Adjust the unmap range to cover only what remains to be unmapped following the above + + startingFileOffset += extentLengthToTrim + length -= extentLengthToTrim + } } } - // Now delete or trim existing extents that overlap with the extent to unmap + // Now advance index to refer to the next (or first) extent known to start at or to the right of the start of the unmap range (if any) + + index++ - subsequentFileOffset = startingFileOffset + length + // Finally proceed deleting or trimming on the left the next extent (if any) until the unmap range is removed for { extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = fileInode.payload.GetByIndex(index) @@ -1084,79 +1128,68 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint if !ok { logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") } - - if subsequentFileOffset <= extentMapEntryKeyV1 { - // We reached an extent that starts after the extent to unmap so we are done - - return - } - extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) if !ok { logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") } - if (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) > subsequentFileOffset { - // Trim this extent on the left - - extentLengthToTrim = subsequentFileOffset - extentMapEntryKeyV1 - - layoutMapEntry, ok = fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] - if !ok { - logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") - } - - if extentLengthToTrim > layoutMapEntry.bytesReferenced { - logFatalf("extentLengthToTrim > layoutMapEntry.bytesReferenced") - } else if (extentLengthToTrim == layoutMapEntry.bytesReferenced) && (extentMapEntryValueV1.ObjectNumber != fileInode.putObjectNumber) { - delete(fileInode.layoutMap, extentMapEntryValueV1.ObjectNumber) - - fileInode.superBlockInodeObjectCountAdjustment-- - fileInode.superBlockInodeObjectSizeAdjustment -= int64(layoutMapEntry.objectSize) - fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) + layoutMapEntry, ok = fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] + if !ok { + logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") + } - fileInode.dereferencedObjectNumberArray = append(fileInode.dereferencedObjectNumberArray, extentMapEntryValueV1.ObjectNumber) - } else { - layoutMapEntry.bytesReferenced -= extentLengthToTrim + // This extent either is completely consumed by the unmap range or needs to be trimmed on the left - fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry + if (startingFileOffset + length) < (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) { + // This extent needs to be trimmed on the left - fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) - } + extentLengthToTrim = (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) - (startingFileOffset + length) ok, err = fileInode.payload.DeleteByIndex(index) if nil != err { - logFatalf("fileInode.payload.DeleteByIndex(index) failed: %v", err) + logFatalf("fileInode.payload.DeleteByIndex() failed: %v", err) } if !ok { - logFatalf("fileInode.payload.DeleteByIndex(index) returned !ok") + logFatalf("fileInode.payload.DeleteByIndex() returned !ok") } + extentMapEntryKeyV1 += extentLengthToTrim + extentMapEntryValueV1.Length -= extentLengthToTrim + extentMapEntryValueV1.ObjectOffset += extentLengthToTrim - ok, err = fileInode.payload.Put(subsequentFileOffset, extentMapEntryValueV1) + ok, err = fileInode.payload.Put(extentMapEntryKeyV1, extentMapEntryValueV1) if nil != err { - logFatalf("fileInode.payload.Put(subsequentFileOffset, extentMapEntryValueV1) failed: %v", err) + logFatalf("fileInode.payload.Put() failed: %v", err) } if !ok { - logFatalf("fileInode.payload.Put(subsequentFileOffset, extentMapEntryValueV1) returned !ok") + logFatalf("fileInode.payload.Put() returned !ok") } - // We know the next loop would find the trimmed extent that starts after the extnet to unmap so we are done + layoutMapEntry.bytesReferenced -= extentLengthToTrim + + fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry + + fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentLengthToTrim) + + // And since we've now fully accounted for the unmap range, we are done return } - // This extent to be totally unmapped + // If we reach here, the extent is completely consumed by the unmap range - layoutMapEntry, ok = fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] + ok, err = fileInode.payload.DeleteByIndex(index) + if nil != err { + logFatalf("fileInode.payload.DeleteByIndex() failed: %v", err) + } if !ok { - logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") + logFatalf("fileInode.payload.DeleteByIndex() returned !ok") } - if extentMapEntryValueV1.Length > layoutMapEntry.bytesReferenced { - logFatalf("extentMapEntryValueV1.Length > layoutMapEntry.bytesReferenced") - } else if (extentMapEntryValueV1.Length == layoutMapEntry.bytesReferenced) && (extentMapEntryValueV1.ObjectNumber != fileInode.putObjectNumber) { + layoutMapEntry.bytesReferenced -= extentMapEntryValueV1.Length + + if layoutMapEntry.bytesReferenced == 0 { delete(fileInode.layoutMap, extentMapEntryValueV1.ObjectNumber) fileInode.superBlockInodeObjectCountAdjustment-- @@ -1172,14 +1205,16 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentMapEntryValueV1.Length) } - ok, err = fileInode.payload.DeleteByIndex(index) - if nil != err { - logFatalf("fileInode.payload.DeleteByIndex(index) failed: %v", err) - } - if !ok { - logFatalf("fileInode.payload.DeleteByIndex(index) returned !ok") + length = (startingFileOffset + length) - (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) + + if length == 0 { + // Since we've now fully accounted for the unmap range, we are done + + return } + startingFileOffset = extentMapEntryKeyV1 + extentMapEntryValueV1.Length + // Now loop back to fetch the next existing extent } } From 5bd238763c8506953f663011320b10c7b71a9a14 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 10 Dec 2021 10:00:00 -0800 Subject: [PATCH 194/258] Fixed unmapExtent() case where the initial BisectLeft() returns true --- iclient/iclientpkg/inode.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index b9a3e93e..bee0068b 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -1104,11 +1104,11 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint length -= extentLengthToTrim } } - } - // Now advance index to refer to the next (or first) extent known to start at or to the right of the start of the unmap range (if any) + // Now advance index to refer to the next (or first) extent known to start at or to the right of the start of the unmap range (if any) - index++ + index++ + } // Finally proceed deleting or trimming on the left the next extent (if any) until the unmap range is removed From d66361db991d59735600c112f43b542287beaa5f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 10 Dec 2021 17:54:46 -0800 Subject: [PATCH 195/258] Ok... so {unmap|record}Extent() totally reworked... but still fails ...dd/cp/diff test proves a sequentially written file is still stored correctly... but diff fails !!! --- iclient/iclientpkg/fission.go | 183 +++++++++++++++++++++------------- iclient/iclientpkg/inode.go | 38 +++---- 2 files changed, 130 insertions(+), 91 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 1d3890e5..a167a74e 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1533,11 +1533,8 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R curOffset uint64 err error extentMapEntryIndexV1 int - extentMapEntryIndexV1Max int // Entry entry at or just after where readIn.Offset+readIn.Size may reside - extentMapEntryIndexV1Min int // First entry at or just before where readIn.Offset may reside extentMapEntryKeyV1 uint64 extentMapEntryKeyV1AsKey sortedmap.Key - extentMapEntrySkipSize uint64 extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct extentMapEntryValueV1AsValue sortedmap.Value inode *inodeStruct @@ -1648,18 +1645,28 @@ Retry: logFatalf("inode.inodeHeadV1.InodeType(%v) unexpected - must be either ilayout.InodeTypeDir(%v) or ilayout.InodeTypeFile(%v)", inode.inodeHeadV1.InodeType, ilayout.InodeTypeDir, ilayout.InodeTypeFile) } + if readIn.Size == 0 { + inodeLockRequest.unlockAll() + readOut = &fission.ReadOut{ + Data: make([]byte, 0), + } + errno = 0 + return + } + curOffset = readIn.Offset if curOffset >= inode.inodeHeadV1.Size { inodeLockRequest.unlockAll() readOut = &fission.ReadOut{ - Data: make([]byte, 0, 0), + Data: make([]byte, 0), } errno = 0 return } - remainingSize = uint64(readIn.Size) - if (curOffset + remainingSize) > inode.inodeHeadV1.Size { + if (curOffset + uint64(readIn.Size)) <= inode.inodeHeadV1.Size { + remainingSize = uint64(readIn.Size) + } else { remainingSize = inode.inodeHeadV1.Size - curOffset } @@ -1674,35 +1681,113 @@ Retry: } } - extentMapEntryIndexV1Min, _, err = inode.payload.BisectLeft(curOffset) + extentMapEntryIndexV1, _, err = inode.payload.BisectLeft(curOffset) if nil != err { logFatal(err) } - - if extentMapEntryIndexV1Min < 0 { - extentMapEntryIndexV1Min = 0 + if extentMapEntryIndexV1 < 0 { + // Correct for case where curOffset is to the left of the first extent + extentMapEntryIndexV1 = 0 } - extentMapEntryIndexV1Max, _, err = inode.payload.BisectLeft(readIn.Offset + remainingSize) - if nil != err { - logFatal(err) - } - - readPlan = make([]*ilayout.ExtentMapEntryValueV1Struct, 0, 2*(extentMapEntryIndexV1Max-extentMapEntryIndexV1Min)) + readPlan = make([]*ilayout.ExtentMapEntryValueV1Struct, 0) - for extentMapEntryIndexV1 = extentMapEntryIndexV1Min; extentMapEntryIndexV1 <= extentMapEntryIndexV1Max; extentMapEntryIndexV1++ { - extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, _, err = inode.payload.GetByIndex(extentMapEntryIndexV1) + for remainingSize > 0 { + extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = inode.payload.GetByIndex(extentMapEntryIndexV1) if nil != err { logFatal(err) } - extentMapEntryKeyV1, ok = extentMapEntryKeyV1AsKey.(uint64) - if !ok { - logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") - } + if ok { + extentMapEntryKeyV1, ok = extentMapEntryKeyV1AsKey.(uint64) + if !ok { + logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") + } + extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) + if !ok { + logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1) returned !ok") + } + + // At this point, curOffset could be <, ==, or > extentMapEntryKeyV1: + // + // curOffset < extentMapEntryKeyV1 might happen on the first loop iteration if the above BisectLeft() + // call found no extent at or to the left of curOffset in which case + // we then adjusted extentMapEntryKeyV1 from -1 to zero + // + // it will also happen on subsequent loop iterations where there is + // a gap between the last extent and this new one since we would only + // have advanced curOffset to the offset just after that last extent + // + // curOffset == extentMapEntryKeyV1 might happen on the first loop iteration if the above BisectLeft() + // found an extent starting at exactly curOffset (found == true) + // + // it will also happen on subsequent loop iterations where there is + // no gap between the last extent and this new one since we would + // have advanced curOffset to the offset just after that last extent + // + // curOffset > extentMapEntryKeyV1 might happen on the first loop iteration if the above BisectLeft() + // did not find an extent starting at exactly curOffset (fount == false) + // but returned the nearest extent starting to the left of curOffset + + if curOffset < extentMapEntryKeyV1 { + if (curOffset + remainingSize) <= extentMapEntryKeyV1 { + // Zero-fill the remainder of the readPlan + + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: remainingSize, + ObjectNumber: 0, + ObjectOffset: 0, + }) + + curOffset += remainingSize + remainingSize = 0 + } else { // (curOffset+remainingSize) > extentMapEntryKeyV1 + // Zero-fill to advance to the start of this extent + + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: extentMapEntryKeyV1 - curOffset, + ObjectNumber: 0, + ObjectOffset: 0, + }) + + remainingSize -= extentMapEntryKeyV1 - curOffset + curOffset = extentMapEntryKeyV1 + } + } + + if remainingSize > 0 { + // In the case where we didn't already zero-fill the remainder of the readPlan above... + + if remainingSize <= extentMapEntryValueV1.Length { + // Populate the remainder of the readPlan with the first remainingSize bytes of this extent + + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: remainingSize, + ObjectNumber: extentMapEntryValueV1.ObjectNumber, + ObjectOffset: extentMapEntryValueV1.ObjectOffset, + }) + + curOffset += remainingSize + remainingSize = 0 + } else { // remainingSize > extentMapEntryValueV1.Length + // Populate the readPlan with this extent - if curOffset < extentMapEntryKeyV1 { - if remainingSize < (extentMapEntryKeyV1 - curOffset) { + readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ + Length: extentMapEntryValueV1.Length, + ObjectNumber: extentMapEntryValueV1.ObjectNumber, + ObjectOffset: extentMapEntryValueV1.ObjectOffset, + }) + + curOffset += extentMapEntryValueV1.Length + remainingSize -= extentMapEntryValueV1.Length + + extentMapEntryIndexV1++ + } + } + } else { // inode.payload.GetByIndex(extentMapEntryIndexV1) returned !ok + // Zero-fill the remainder of the readPlan + + if (curOffset + remainingSize) < extentMapEntryKeyV1 { readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ Length: remainingSize, ObjectNumber: 0, @@ -1711,56 +1796,8 @@ Retry: curOffset += remainingSize remainingSize = 0 - - break - } else { - readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ - Length: extentMapEntryKeyV1 - curOffset, - ObjectNumber: 0, - ObjectOffset: 0, - }) - - remainingSize -= extentMapEntryKeyV1 - curOffset - curOffset = extentMapEntryKeyV1 } } - - extentMapEntrySkipSize = curOffset - extentMapEntryKeyV1 - - extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) - if !ok { - logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1) returned !ok") - } - - if remainingSize <= (extentMapEntryValueV1.Length - extentMapEntrySkipSize) { - readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ - Length: remainingSize, - ObjectNumber: extentMapEntryValueV1.ObjectNumber, - ObjectOffset: extentMapEntryValueV1.ObjectOffset + extentMapEntrySkipSize, - }) - - curOffset += remainingSize - remainingSize = 0 - - break - } - - readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ - Length: extentMapEntryValueV1.Length - extentMapEntrySkipSize, - ObjectNumber: extentMapEntryValueV1.ObjectNumber, - ObjectOffset: extentMapEntryValueV1.ObjectOffset + extentMapEntrySkipSize, - }) - - curOffset += extentMapEntryValueV1.Length - extentMapEntrySkipSize - remainingSize -= extentMapEntryValueV1.Length - extentMapEntrySkipSize - } - - if remainingSize > 0 { - readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ - Length: remainingSize, - ObjectNumber: 0, - ObjectOffset: 0, - }) } // Now process readPlan @@ -2203,6 +2240,7 @@ func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission var ( startTime time.Time = time.Now() ) + fmt.Printf("UNDO: got a DoFSync(inHeader: %+v, fSyncIn: %+v)\n", inHeader, fSyncIn) logTracef("==> DoFSync(inHeader: %+v, fSyncIn: %+v)", inHeader, fSyncIn) defer func() { @@ -2568,6 +2606,7 @@ func (dummy *globalsStruct) DoFlush(inHeader *fission.InHeader, flushIn *fission openHandle *openHandleStruct startTime time.Time = time.Now() ) + fmt.Printf("UNDO: got a DoFlush(inHeader: %+v, flushIn: %+v)\n", inHeader, flushIn) logTracef("==> DoFlush(inHeader: %+v, flushIn: %+v)", inHeader, flushIn) defer func() { diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index bee0068b..0d7d2935 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -881,6 +881,7 @@ func (fileInode *inodeStruct) recordExtent(startingFileOffset uint64, length uin extentMapEntryKeyV1AsKey sortedmap.Key extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct extentMapEntryValueV1AsValue sortedmap.Value + found bool index int layoutMapEntry layoutMapEntryStruct ok bool @@ -903,10 +904,13 @@ func (fileInode *inodeStruct) recordExtent(startingFileOffset uint64, length uin fileInode.superBlockInodeObjectSizeAdjustment += int64(length) fileInode.superBlockInodeBytesReferencedAdjustment += int64(length) - index, _, err = fileInode.payload.BisectLeft(startingFileOffset) + index, found, err = fileInode.payload.BisectLeft(startingFileOffset) if nil != err { logFatalf("fileInode.payload.BisectLeft(startingFileOffset) failed: %v", err) } + if found { + logFatalf("fileInode.payload.BisectLeft(startingFileOffset) should have returned !found") + } extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = fileInode.payload.GetByIndex(index) if nil != err { @@ -1110,9 +1114,9 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint index++ } - // Finally proceed deleting or trimming on the left the next extent (if any) until the unmap range is removed + // If we reach here, proceed deleting or trimming on the left the next extent (if any) until the unmap range is removed - for { + for length > 0 { extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = fileInode.payload.GetByIndex(index) if nil != err { logFatalf("fileInode.payload.GetByIndex(index) failed: %v", err) @@ -1138,12 +1142,17 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] returned !ok") } + // Might as well simplify the logic below by trimming the unmap range on the left up to extentMapEntryKeyV1 + + length -= extentMapEntryKeyV1 - startingFileOffset + startingFileOffset = extentMapEntryKeyV1 + // This extent either is completely consumed by the unmap range or needs to be trimmed on the left - if (startingFileOffset + length) < (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) { + if length < extentMapEntryValueV1.Length { // This extent needs to be trimmed on the left - extentLengthToTrim = (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) - (startingFileOffset + length) + extentLengthToTrim = length - (extentMapEntryKeyV1 - startingFileOffset) ok, err = fileInode.payload.DeleteByIndex(index) if nil != err { @@ -1153,10 +1162,10 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("fileInode.payload.DeleteByIndex() returned !ok") } - extentMapEntryKeyV1 += extentLengthToTrim + extentMapEntryKeyV1 += length - extentMapEntryValueV1.Length -= extentLengthToTrim - extentMapEntryValueV1.ObjectOffset += extentLengthToTrim + extentMapEntryValueV1.Length -= length + extentMapEntryValueV1.ObjectOffset += length ok, err = fileInode.payload.Put(extentMapEntryKeyV1, extentMapEntryValueV1) if nil != err { @@ -1205,16 +1214,7 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentMapEntryValueV1.Length) } - length = (startingFileOffset + length) - (extentMapEntryKeyV1 + extentMapEntryValueV1.Length) - - if length == 0 { - // Since we've now fully accounted for the unmap range, we are done - - return - } - - startingFileOffset = extentMapEntryKeyV1 + extentMapEntryValueV1.Length - - // Now loop back to fetch the next existing extent + startingFileOffset += extentMapEntryValueV1.Length + length -= extentMapEntryValueV1.Length } } From adab756cc7fd0ea977bb61f72c9da4a8eb51effb Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 10 Dec 2021 18:43:51 -0800 Subject: [PATCH 196/258] Fixed bad readPlan construction in DoRead() --- iclient/iclientpkg/fission.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index a167a74e..456244e3 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1537,6 +1537,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R extentMapEntryKeyV1AsKey sortedmap.Key extentMapEntryValueV1 *ilayout.ExtentMapEntryValueV1Struct extentMapEntryValueV1AsValue sortedmap.Value + extentMapEntrySkipLength uint64 inode *inodeStruct inodeLockRequest *inodeLockRequestStruct obtainExclusiveLock bool @@ -1757,29 +1758,33 @@ Retry: if remainingSize > 0 { // In the case where we didn't already zero-fill the remainder of the readPlan above... + // + // Account for the portion of the extent to be skipped over - if remainingSize <= extentMapEntryValueV1.Length { + extentMapEntrySkipLength = curOffset - extentMapEntryKeyV1 + + if (extentMapEntrySkipLength + remainingSize) <= extentMapEntryValueV1.Length { // Populate the remainder of the readPlan with the first remainingSize bytes of this extent readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ Length: remainingSize, ObjectNumber: extentMapEntryValueV1.ObjectNumber, - ObjectOffset: extentMapEntryValueV1.ObjectOffset, + ObjectOffset: extentMapEntryValueV1.ObjectOffset + extentMapEntrySkipLength, }) curOffset += remainingSize remainingSize = 0 - } else { // remainingSize > extentMapEntryValueV1.Length + } else { // (extentMapEntrySkipLength + remainingSize) > extentMapEntryValueV1.Length // Populate the readPlan with this extent readPlan = append(readPlan, &ilayout.ExtentMapEntryValueV1Struct{ - Length: extentMapEntryValueV1.Length, + Length: extentMapEntryValueV1.Length - extentMapEntrySkipLength, ObjectNumber: extentMapEntryValueV1.ObjectNumber, - ObjectOffset: extentMapEntryValueV1.ObjectOffset, + ObjectOffset: extentMapEntryValueV1.ObjectOffset + extentMapEntrySkipLength, }) - curOffset += extentMapEntryValueV1.Length - remainingSize -= extentMapEntryValueV1.Length + curOffset += extentMapEntryValueV1.Length - extentMapEntrySkipLength + remainingSize -= extentMapEntryValueV1.Length - extentMapEntrySkipLength extentMapEntryIndexV1++ } From 003cf20e0af01b53ecf3d6b51fef92b80a8fd450 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 12 Dec 2021 16:51:14 -0800 Subject: [PATCH 197/258] Ported the old pfs-stress program over to iclient/iclientpkg/inode_test.go Note: With testInodeStressFileSize set to the commented out value, the test will very likely fail as of now --- iclient/iclientpkg/fission.go | 29 +- iclient/iclientpkg/inode_test.go | 489 +++++++++++++++++++++++++++++++ 2 files changed, 514 insertions(+), 4 deletions(-) create mode 100644 iclient/iclientpkg/inode_test.go diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 456244e3..9faed475 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1553,6 +1553,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R readCacheLineWG *sync.WaitGroup readPlan []*ilayout.ExtentMapEntryValueV1Struct readPlanEntry *ilayout.ExtentMapEntryValueV1Struct // If .ObjectNumber == 0, .ObjectOffset is ignored... .Length is the number of zero fill bytes + readPlanEntryIndex int // UNDO remainingSize uint64 startTime time.Time = time.Now() ) @@ -1573,7 +1574,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R } }() - obtainExclusiveLock = false + obtainExclusiveLock = true // UNDO: change it back to false Retry: inodeLockRequest = newLockRequest() @@ -1807,7 +1808,10 @@ Retry: // Now process readPlan - for _, readPlanEntry = range readPlan { + fmt.Printf("UNDO: DoRead(inHeader: %+v, readIn: %+v) with len(readPlan): %v\n", inHeader, readIn, len(readPlan)) + // for _, readPlanEntry = range readPlan { + for readPlanEntryIndex, readPlanEntry = range readPlan { // UNDO (swap with above) + fmt.Printf("UNDO: ... readPlanEntry[%v]: %+v\n", readPlanEntryIndex, readPlanEntry) switch readPlanEntry.ObjectNumber { case 0: readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length)...) @@ -1815,12 +1819,14 @@ Retry: // Note that if putObject is inactive, case 0: will have already matched readPlanEntry.ObjectNumber readOut.Data = append(readOut.Data, inode.putObjectBuffer[readPlanEntry.ObjectOffset:(readPlanEntry.ObjectOffset+readPlanEntry.Length)]...) default: + fmt.Printf("UNDO: - hit the \"not in inode.putObjectBuffer\" case\n") for readPlanEntry.Length > 0 { readCacheKey = readCacheKeyStruct{ objectNumber: readPlanEntry.ObjectNumber, lineNumber: readPlanEntry.ObjectOffset / globals.config.ReadCacheLineSize, } readCacheLineOffset = readPlanEntry.ObjectOffset - (readCacheKey.lineNumber * globals.config.ReadCacheLineSize) + fmt.Printf("UNDO: while readPlanEntry.Length (%v) > 0, readCacheKey: %+v readCacheLineOffset: %v\n", readPlanEntry.Length, readCacheKey, readCacheLineOffset) globals.Lock() @@ -1829,22 +1835,26 @@ Retry: if ok { // readCacheLine is in globals.readCacheMap but may be being filled + fmt.Printf("UNDO: readCache \"hit\"... but may be filling\n") globals.readCacheLRU.MoveToBack(readCacheLine.listElement) if readCacheLine.wg == nil { // readCacheLine is already filled... + fmt.Printf("UNDO: readCache \"hit\"... line already filled\n") readCacheLineBuf = readCacheLine.buf globals.Unlock() } else { // readCacheLine is being filled... so just wait for it + fmt.Printf("UNDO: readCache \"hit\"... line being filled\n") readCacheLineWG = readCacheLine.wg globals.Unlock() readCacheLineWG.Wait() + fmt.Printf("UNDO: readCache \"hit\"... line fill completed\n") // If readCacheLine fill failed... we must exit @@ -1853,15 +1863,18 @@ Retry: globals.Unlock() if nil == readCacheLineBuf { + fmt.Printf("UNDO: readCache \"hit\"... line fill failed\n") inodeLockRequest.unlockAll() readOut = nil errno = syscall.EIO return } + fmt.Printf("UNDO: readCache \"hit\"... line fill succeeded\n") } } else { // readCacheLine is absent from globals.readCacheMap... so put it there and fill it + fmt.Printf("UNDO: readCache \"miss\"...\n") readCacheLineWG = &sync.WaitGroup{} readCacheLine = &readCacheLineStruct{ @@ -1874,6 +1887,8 @@ Retry: readCacheLine.listElement = globals.readCacheLRU.PushBack(readCacheLine) + globals.readCacheMap[readCacheKey] = readCacheLine + // Need to evict LRU'd readCacheLine if globals.config.ReadCacheLineCountMax is exceeded for globals.config.ReadCacheLineCountMax < uint64(globals.readCacheLRU.Len()) { @@ -1897,6 +1912,7 @@ Retry: if nil != err { // readCacheLine fill failed... so tell others and exit + fmt.Printf("UNDO: readCache \"miss\"...fill failed\n") globals.Lock() delete(globals.readCacheMap, readCacheKey) @@ -1912,6 +1928,7 @@ Retry: errno = syscall.EIO return } + fmt.Printf("UNDO: readCache \"miss\"...fill succeeded\n") // readCacheLine fill succeeded... so tell others and continue @@ -1930,6 +1947,7 @@ Retry: if readCacheLineOffset >= uint64(len(readCacheLineBuf)) { // readCacheLineBuf unexpectedly too short... we must exit + fmt.Printf("UNDO: readCache line unexpectedly too short\n") inodeLockRequest.unlockAll() readOut = nil errno = syscall.EIO @@ -1941,6 +1959,7 @@ Retry: if readPlanEntry.Length > readCacheLineBufLengthAvailableToConsume { // Consume tail of readCacheLineBuf starting at readCacheLineOffset and continue looping + fmt.Printf("UNDO: readCache line tail being consumed\n") readOut.Data = append(readOut.Data, readCacheLineBuf[readCacheLineOffset:]...) readPlanEntry.Length -= readCacheLineBufLengthAvailableToConsume @@ -1948,6 +1967,7 @@ Retry: } else { // Consume only the portion of readCacheLineBuf needed and trigger loop exit + fmt.Printf("UNDO: readCache line chunk being consumed\n") readOut.Data = append(readOut.Data, readCacheLineBuf[readCacheLineOffset:(readCacheLineOffset+readPlanEntry.Length)]...) readPlanEntry.Length = 0 @@ -2070,6 +2090,7 @@ Retry: // Pre-flush if this write would cause inode.putObjectBuffer to exceed globals.config.FileFlushTriggerSize if (uint64(len(inode.putObjectBuffer)) + uint64(len(writeIn.Data))) > globals.config.FileFlushTriggerSize { + fmt.Printf("UNDO: We are triggering a size-based flush...\n") flushInodesInSlice([]*inodeStruct{inode}) inode.dirty = true @@ -2245,7 +2266,7 @@ func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission var ( startTime time.Time = time.Now() ) - fmt.Printf("UNDO: got a DoFSync(inHeader: %+v, fSyncIn: %+v)\n", inHeader, fSyncIn) + // fmt.Printf("UNDO: got a DoFSync(inHeader: %+v, fSyncIn: %+v)\n", inHeader, fSyncIn) logTracef("==> DoFSync(inHeader: %+v, fSyncIn: %+v)", inHeader, fSyncIn) defer func() { @@ -2611,7 +2632,7 @@ func (dummy *globalsStruct) DoFlush(inHeader *fission.InHeader, flushIn *fission openHandle *openHandleStruct startTime time.Time = time.Now() ) - fmt.Printf("UNDO: got a DoFlush(inHeader: %+v, flushIn: %+v)\n", inHeader, flushIn) + // fmt.Printf("UNDO: got a DoFlush(inHeader: %+v, flushIn: %+v)\n", inHeader, flushIn) logTracef("==> DoFlush(inHeader: %+v, flushIn: %+v)", inHeader, flushIn) defer func() { diff --git a/iclient/iclientpkg/inode_test.go b/iclient/iclientpkg/inode_test.go new file mode 100644 index 00000000..460f3310 --- /dev/null +++ b/iclient/iclientpkg/inode_test.go @@ -0,0 +1,489 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +import ( + "bytes" + "crypto/rand" + "fmt" + "math/big" + "runtime" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/NVIDIA/fission" +) + +const ( + testInodeStressDisplayUpdateInterval = time.Duration(100 * time.Millisecond) + testInodeStressFileNamePrefix = "_inode_stress_" + testInodeStressFileSize uint64 = 100000 // UNDO 1000000 + testInodeStressMaxExtentSize uint64 = 10000 + testInodeStressMinExtentSize uint64 = 1 + testInodeStressNumExtentWritesPerFile uint64 = 1000 + testInodeStressNumExtentWritesPerFlush uint64 = 50 // 0 means only perform Flush function at the end + testInodeStressNumExtentWritesPerValidate uint64 = 50 // 0 means only perform Validate function at the end + testInodeStressNumFiles uint64 = 1 +) + +type testInodeStressGlobalsStruct struct { + sync.Mutex // Protects err + sync.WaitGroup // Signaled by each retiring testInodeStresser + numExtentWritesPerFlush uint64 + numExtentWritesPerValidate uint64 + numExtentsToWriteInTotal uint64 + numExtentsWrittenInTotal uint64 + err []error +} + +var testInodeStressGlobals *testInodeStressGlobalsStruct + +func TestInodeStress(t *testing.T) { + var ( + err error + fissionErrChan chan error + progressPercentage uint64 + stresserIndex uint64 + ) + + // Perform baseline setup + + testSetup(t) + + fissionErrChan = make(chan error, 1) + + err = initializeGlobals(testGlobals.confMap, fissionErrChan) + if nil != err { + t.Fatalf("initializeGlobals(testGlobals.confMap, fissionErrChan) failed: %v", err) + } + + err = startRPCHandler() + if nil != err { + t.Fatalf("startRPCHandler() failed: %v", err) + } + + // Setup testInodeStressGlobals + + testInodeStressGlobals = &testInodeStressGlobalsStruct{ + err: make([]error, 0, testInodeStressNumFiles), + } + + if testInodeStressNumExtentWritesPerFlush == 0 { + testInodeStressGlobals.numExtentWritesPerFlush = testInodeStressNumExtentWritesPerFile + } else { + testInodeStressGlobals.numExtentWritesPerFlush = testInodeStressNumExtentWritesPerFlush + } + + if testInodeStressNumExtentWritesPerFlush == 0 { + testInodeStressGlobals.numExtentWritesPerValidate = testInodeStressNumExtentWritesPerFile + } else { + testInodeStressGlobals.numExtentWritesPerValidate = testInodeStressNumExtentWritesPerFlush + } + + testInodeStressGlobals.numExtentsToWriteInTotal = testInodeStressNumFiles * testInodeStressNumExtentWritesPerFile + testInodeStressGlobals.numExtentsWrittenInTotal = 0 + + // Launch InodeStresser goroutines + + testInodeStressGlobals.Add(int(testInodeStressNumFiles)) + + for stresserIndex = 0; stresserIndex < testInodeStressNumFiles; stresserIndex++ { + go testInodeStresser(stresserIndex) + } + + // Monitor InodeStresser goroutines + + for { + time.Sleep(testInodeStressDisplayUpdateInterval) + progressPercentage = 100 * atomic.LoadUint64(&testInodeStressGlobals.numExtentsWrittenInTotal) / testInodeStressGlobals.numExtentsToWriteInTotal + testInodeStressGlobals.Lock() + if len(testInodeStressGlobals.err) > 0 { + for _, err = range testInodeStressGlobals.err { + t.Error(err) + } + t.Fatalf("...exiting...") + } + testInodeStressGlobals.Unlock() + t.Logf("Progress: %3d%%", progressPercentage) + if progressPercentage == 100 { + break + } + } + + // Wait for goroutines to cleanly exit + + testInodeStressGlobals.Wait() + + // Cleanly shutdown + + err = stopRPCHandler() + if nil != err { + t.Fatalf("stopRPCHandler() failed: %v", err) + } + + err = uninitializeGlobals() + if nil != err { + t.Fatalf("uninitializeGlobals() failed: %v", err) + } + + testTeardown(t) +} + +type testInodeStresserContext struct { + fileName string + NodeID uint64 + FH uint64 + written []byte +} + +func testInodeStresser(stresserIndex uint64) { + var ( + b byte + createIn *fission.CreateIn + createOut *fission.CreateOut + err error + errno syscall.Errno + extentIndex uint64 + inHeader *fission.InHeader + mustBeLessThanBigIntPtr *big.Int + numExtentWritesSinceLastFlush uint64 + numExtentWritesSinceLastValidate uint64 + offset uint64 + size uint32 + releaseIn *fission.ReleaseIn + tISC *testInodeStresserContext + u64BigIntPtr *big.Int + unlinkIn *fission.UnlinkIn + ) + + // Construct this instance's testInodeStresserContext + + tISC = &testInodeStresserContext{ + fileName: fmt.Sprintf("%s%04X", testInodeStressFileNamePrefix, stresserIndex), + written: make([]byte, testInodeStressFileSize), + } + + inHeader = &fission.InHeader{ + NodeID: 1, + UID: 0, + GID: 0, + } + createIn = &fission.CreateIn{ + Mode: 0600, + UMask: 0666, + Name: []byte(tISC.fileName), + } + + createOut, errno = globals.DoCreate(inHeader, createIn) + if errno != 0 { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, fmt.Errorf("globals.DoCreate(inHeader, createIn) returned errno: %v", errno)) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + tISC.NodeID = createOut.EntryOut.NodeID + tISC.FH = createOut.FH + + err = tISC.writeAt(uint64(0), uint32(testInodeStressFileSize), 0x00) + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + err = tISC.validate() + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + // Perform extent writes + + b = 0x00 + numExtentWritesSinceLastFlush = 0 + numExtentWritesSinceLastValidate = 0 + + for extentIndex = 0; extentIndex < testInodeStressNumExtentWritesPerFile; extentIndex++ { + // Pick a size value such that testInodeStressMinExtentSize <= size <= testInodeStressMaxExtentSize + + mustBeLessThanBigIntPtr = big.NewInt(int64(testInodeStressMaxExtentSize - testInodeStressMinExtentSize + 1)) + u64BigIntPtr, err = rand.Int(rand.Reader, mustBeLessThanBigIntPtr) + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + size = uint32(testInodeStressMinExtentSize + u64BigIntPtr.Uint64()) + + // Pick an offset value such that 0 <= offset <= (testInodeStressFileSize - size) + + mustBeLessThanBigIntPtr = big.NewInt(int64(testInodeStressFileSize) - int64(size)) + u64BigIntPtr, err = rand.Int(rand.Reader, mustBeLessThanBigIntPtr) + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + offset = u64BigIntPtr.Uint64() + + // Pick next b value (skipping 0x00 for as-yet-un-over-written bytes) + + b++ + if b == 0x00 { + b = 0x01 + } + + // Now perform the selected overwrite + + err = tISC.writeAt(offset, size, b) + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + // Perform a flush if required + + numExtentWritesSinceLastFlush++ + + if testInodeStressNumExtentWritesPerFlush == numExtentWritesSinceLastFlush { + err = tISC.flush() + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + numExtentWritesSinceLastFlush = 0 + } + + // Perform a validate if required + + numExtentWritesSinceLastValidate++ + + if testInodeStressNumExtentWritesPerValidate == numExtentWritesSinceLastValidate { + err = tISC.validate() + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + numExtentWritesSinceLastValidate = 0 + } + + // Finally, update testInodeStressGlobals.numExtentsWrittenInTotal + + testInodeStressGlobals.Lock() + testInodeStressGlobals.numExtentsWrittenInTotal++ + testInodeStressGlobals.Unlock() + } + + // Do one final tISC.flush call if necessary to flush final writes + + if 0 < numExtentWritesSinceLastFlush { + err = tISC.flush() + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + } + + // Do one final tISC.validate call if necessary to validate final writes + + if 0 < numExtentWritesSinceLastValidate { + err = tISC.validate() + if err != nil { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + } + + // Clean up and exit + + inHeader = &fission.InHeader{ + NodeID: tISC.NodeID, + UID: 0, + GID: 0, + } + releaseIn = &fission.ReleaseIn{ + FH: tISC.FH, + } + + errno = globals.DoRelease(inHeader, releaseIn) + if errno != 0 { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, fmt.Errorf("globals.DoRelease(inHeader, releaseIn) returned errno: %v", errno)) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + inHeader = &fission.InHeader{ + NodeID: 1, + UID: 0, + GID: 0, + } + unlinkIn = &fission.UnlinkIn{ + Name: []byte(tISC.fileName), + } + + errno = globals.DoUnlink(inHeader, unlinkIn) + if errno != 0 { + testInodeStressGlobals.Lock() + testInodeStressGlobals.err = append(testInodeStressGlobals.err, fmt.Errorf("globals.DoUnlink(inHeader, unlinkIn) returned errno: %v", errno)) + testInodeStressGlobals.Unlock() + + testInodeStressGlobals.Done() + + runtime.Goexit() + } + + testInodeStressGlobals.Done() +} + +func (tISC *testInodeStresserContext) writeAt(offset uint64, size uint32, b byte) (err error) { + var ( + errno syscall.Errno + i uint32 + inHeader *fission.InHeader + writeIn *fission.WriteIn + ) + + inHeader = &fission.InHeader{ + NodeID: tISC.NodeID, + } + writeIn = &fission.WriteIn{ + FH: tISC.FH, + Offset: offset, + Size: size, + Data: make([]byte, size), + } + + for i = 0; i < size; i++ { + writeIn.Data[i] = b + tISC.written[offset+uint64(i)] = b + } + + _, errno = globals.DoWrite(inHeader, writeIn) + if errno == 0 { + err = nil + } else { + err = fmt.Errorf("globals.DoWrite(inHeader, writeIn) returned errno: %v", errno) + } + + return +} + +func (tISC *testInodeStresserContext) flush() (err error) { + var ( + errno syscall.Errno + inHeader *fission.InHeader + flushIn *fission.FlushIn + ) + + inHeader = &fission.InHeader{ + NodeID: tISC.NodeID, + } + flushIn = &fission.FlushIn{ + FH: tISC.FH, + } + + errno = globals.DoFlush(inHeader, flushIn) + if errno == 0 { + err = nil + } else { + err = fmt.Errorf("globals.DoFlush(inHeader, fSyncIn) returned errno: %v", errno) + } + + return +} + +func (tISC *testInodeStresserContext) validate() (err error) { + var ( + errno syscall.Errno + i int + inHeader *fission.InHeader + readIn *fission.ReadIn + readOut *fission.ReadOut + ) + + inHeader = &fission.InHeader{ + NodeID: tISC.NodeID, + } + readIn = &fission.ReadIn{ + FH: tISC.FH, + Offset: 0, + Size: uint32(testInodeStressFileSize), + } + + readOut, errno = globals.DoRead(inHeader, readIn) + if errno == 0 { + if bytes.Equal(tISC.written, readOut.Data) { + err = nil + } else { + err = fmt.Errorf("Miscompare in fileName %s\n", tISC.fileName) + for i = range tISC.written { + if tISC.written[i] != readOut.Data[i] { + err = fmt.Errorf("First miscompare in %s at position %v", tISC.fileName, i) + return + } + } + } + } else { + err = fmt.Errorf("globals.DoRead(inHeader, readIn) returned errno: %v", errno) + } + + return +} From 85919843e5f9f2dec84d945ecc3a5009c2af8d5a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 13 Dec 2021 10:25:45 -0800 Subject: [PATCH 198/258] Enabled DoFSync() to do what DoFlush() does ...and cleaned up the printf debugging lines in fission.go --- iclient/iclientpkg/fission.go | 89 +++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 9faed475..0cd088fc 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1553,7 +1553,6 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R readCacheLineWG *sync.WaitGroup readPlan []*ilayout.ExtentMapEntryValueV1Struct readPlanEntry *ilayout.ExtentMapEntryValueV1Struct // If .ObjectNumber == 0, .ObjectOffset is ignored... .Length is the number of zero fill bytes - readPlanEntryIndex int // UNDO remainingSize uint64 startTime time.Time = time.Now() ) @@ -1574,7 +1573,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R } }() - obtainExclusiveLock = true // UNDO: change it back to false + obtainExclusiveLock = false Retry: inodeLockRequest = newLockRequest() @@ -1808,10 +1807,7 @@ Retry: // Now process readPlan - fmt.Printf("UNDO: DoRead(inHeader: %+v, readIn: %+v) with len(readPlan): %v\n", inHeader, readIn, len(readPlan)) - // for _, readPlanEntry = range readPlan { - for readPlanEntryIndex, readPlanEntry = range readPlan { // UNDO (swap with above) - fmt.Printf("UNDO: ... readPlanEntry[%v]: %+v\n", readPlanEntryIndex, readPlanEntry) + for _, readPlanEntry = range readPlan { switch readPlanEntry.ObjectNumber { case 0: readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length)...) @@ -1819,14 +1815,12 @@ Retry: // Note that if putObject is inactive, case 0: will have already matched readPlanEntry.ObjectNumber readOut.Data = append(readOut.Data, inode.putObjectBuffer[readPlanEntry.ObjectOffset:(readPlanEntry.ObjectOffset+readPlanEntry.Length)]...) default: - fmt.Printf("UNDO: - hit the \"not in inode.putObjectBuffer\" case\n") for readPlanEntry.Length > 0 { readCacheKey = readCacheKeyStruct{ objectNumber: readPlanEntry.ObjectNumber, lineNumber: readPlanEntry.ObjectOffset / globals.config.ReadCacheLineSize, } readCacheLineOffset = readPlanEntry.ObjectOffset - (readCacheKey.lineNumber * globals.config.ReadCacheLineSize) - fmt.Printf("UNDO: while readPlanEntry.Length (%v) > 0, readCacheKey: %+v readCacheLineOffset: %v\n", readPlanEntry.Length, readCacheKey, readCacheLineOffset) globals.Lock() @@ -1835,26 +1829,22 @@ Retry: if ok { // readCacheLine is in globals.readCacheMap but may be being filled - fmt.Printf("UNDO: readCache \"hit\"... but may be filling\n") globals.readCacheLRU.MoveToBack(readCacheLine.listElement) if readCacheLine.wg == nil { // readCacheLine is already filled... - fmt.Printf("UNDO: readCache \"hit\"... line already filled\n") readCacheLineBuf = readCacheLine.buf globals.Unlock() } else { // readCacheLine is being filled... so just wait for it - fmt.Printf("UNDO: readCache \"hit\"... line being filled\n") readCacheLineWG = readCacheLine.wg globals.Unlock() readCacheLineWG.Wait() - fmt.Printf("UNDO: readCache \"hit\"... line fill completed\n") // If readCacheLine fill failed... we must exit @@ -1863,18 +1853,15 @@ Retry: globals.Unlock() if nil == readCacheLineBuf { - fmt.Printf("UNDO: readCache \"hit\"... line fill failed\n") inodeLockRequest.unlockAll() readOut = nil errno = syscall.EIO return } - fmt.Printf("UNDO: readCache \"hit\"... line fill succeeded\n") } } else { // readCacheLine is absent from globals.readCacheMap... so put it there and fill it - fmt.Printf("UNDO: readCache \"miss\"...\n") readCacheLineWG = &sync.WaitGroup{} readCacheLine = &readCacheLineStruct{ @@ -1912,7 +1899,6 @@ Retry: if nil != err { // readCacheLine fill failed... so tell others and exit - fmt.Printf("UNDO: readCache \"miss\"...fill failed\n") globals.Lock() delete(globals.readCacheMap, readCacheKey) @@ -1928,7 +1914,6 @@ Retry: errno = syscall.EIO return } - fmt.Printf("UNDO: readCache \"miss\"...fill succeeded\n") // readCacheLine fill succeeded... so tell others and continue @@ -1947,7 +1932,6 @@ Retry: if readCacheLineOffset >= uint64(len(readCacheLineBuf)) { // readCacheLineBuf unexpectedly too short... we must exit - fmt.Printf("UNDO: readCache line unexpectedly too short\n") inodeLockRequest.unlockAll() readOut = nil errno = syscall.EIO @@ -1959,7 +1943,6 @@ Retry: if readPlanEntry.Length > readCacheLineBufLengthAvailableToConsume { // Consume tail of readCacheLineBuf starting at readCacheLineOffset and continue looping - fmt.Printf("UNDO: readCache line tail being consumed\n") readOut.Data = append(readOut.Data, readCacheLineBuf[readCacheLineOffset:]...) readPlanEntry.Length -= readCacheLineBufLengthAvailableToConsume @@ -1967,7 +1950,6 @@ Retry: } else { // Consume only the portion of readCacheLineBuf needed and trigger loop exit - fmt.Printf("UNDO: readCache line chunk being consumed\n") readOut.Data = append(readOut.Data, readCacheLineBuf[readCacheLineOffset:(readCacheLineOffset+readPlanEntry.Length)]...) readPlanEntry.Length = 0 @@ -2090,7 +2072,6 @@ Retry: // Pre-flush if this write would cause inode.putObjectBuffer to exceed globals.config.FileFlushTriggerSize if (uint64(len(inode.putObjectBuffer)) + uint64(len(writeIn.Data))) > globals.config.FileFlushTriggerSize { - fmt.Printf("UNDO: We are triggering a size-based flush...\n") flushInodesInSlice([]*inodeStruct{inode}) inode.dirty = true @@ -2264,9 +2245,12 @@ Retry: func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission.FSyncIn) (errno syscall.Errno) { var ( - startTime time.Time = time.Now() + err error + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + openHandle *openHandleStruct + startTime time.Time = time.Now() ) - // fmt.Printf("UNDO: got a DoFSync(inHeader: %+v, fSyncIn: %+v)\n", inHeader, fSyncIn) logTracef("==> DoFSync(inHeader: %+v, fSyncIn: %+v)", inHeader, fSyncIn) defer func() { @@ -2277,8 +2261,62 @@ func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission globals.stats.DoFSyncUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - // TODO - errno = syscall.ENOSYS +Retry: + inodeLockRequest = newLockRequest() + inodeLockRequest.inodeNumber = inHeader.NodeID + inodeLockRequest.exclusive = true + inodeLockRequest.addThisLock() + if len(inodeLockRequest.locksHeld) == 0 { + performInodeLockRetryDelay() + goto Retry + } + + openHandle = lookupOpenHandle(fSyncIn.FH) + if nil == openHandle { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + if openHandle.inodeNumber != inHeader.NodeID { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + if !openHandle.fissionFlagsWrite { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + + inode = lookupInode(openHandle.inodeNumber) + if nil == inode { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + + if nil == inode.inodeHeadV1 { + err = inode.populateInodeHeadV1() + if nil != err { + inodeLockRequest.unlockAll() + errno = syscall.ENOENT + return + } + } + + if inode.inodeHeadV1.InodeType != ilayout.InodeTypeFile { + inodeLockRequest.unlockAll() + errno = syscall.EBADF + return + } + + if inode.dirty { + flushInodesInSlice([]*inodeStruct{inode}) + } + + inodeLockRequest.unlockAll() + + errno = 0 return } @@ -2632,7 +2670,6 @@ func (dummy *globalsStruct) DoFlush(inHeader *fission.InHeader, flushIn *fission openHandle *openHandleStruct startTime time.Time = time.Now() ) - // fmt.Printf("UNDO: got a DoFlush(inHeader: %+v, flushIn: %+v)\n", inHeader, flushIn) logTracef("==> DoFlush(inHeader: %+v, flushIn: %+v)", inHeader, flushIn) defer func() { From 795d2d79fc13fd1dc51f0d15185be957f3d25f00 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 13 Dec 2021 10:59:07 -0800 Subject: [PATCH 199/258] Fixed double-subtraction of layoutMapEntry.bytesReferenced --- iclient/iclientpkg/inode.go | 12 +++++++++--- iclient/iclientpkg/inode_test.go | 14 +++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index 0d7d2935..edc9de34 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -1071,6 +1071,8 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("fileInode.payload.Put() returned !ok") } + // Update the layoutMapEntry... note that, since we only punched a hole in the extent, we know .bytesReferenced will remain > 0 + layoutMapEntry.bytesReferenced -= length fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry @@ -1096,6 +1098,8 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("fileInode.payload.PatchByIndex() returned !ok") } + // Update the layoutMapEntry... note that, since we only trimmed the extent, we know .bytesReferenced will remain > 0 + layoutMapEntry.bytesReferenced -= extentLengthToTrim fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry @@ -1114,7 +1118,7 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint index++ } - // If we reach here, proceed deleting or trimming on the left the next extent (if any) until the unmap range is removed + // If we reach here, proceed deleting or trimming on the left the next extent(s) (if any) until the unmap range is removed for length > 0 { extentMapEntryKeyV1AsKey, extentMapEntryValueV1AsValue, ok, err = fileInode.payload.GetByIndex(index) @@ -1175,6 +1179,8 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("fileInode.payload.Put() returned !ok") } + // Update the layoutMapEntry... note that, since we only trimmed the extent, we know .bytesReferenced will remain > 0 + layoutMapEntry.bytesReferenced -= extentLengthToTrim fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry @@ -1196,6 +1202,8 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("fileInode.payload.DeleteByIndex() returned !ok") } + // Update the layoutMapEntry... possibly deleting it if .bytesReferences reaches 0 + layoutMapEntry.bytesReferenced -= extentMapEntryValueV1.Length if layoutMapEntry.bytesReferenced == 0 { @@ -1207,8 +1215,6 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint fileInode.dereferencedObjectNumberArray = append(fileInode.dereferencedObjectNumberArray, extentMapEntryValueV1.ObjectNumber) } else { - layoutMapEntry.bytesReferenced -= extentMapEntryValueV1.Length - fileInode.layoutMap[extentMapEntryValueV1.ObjectNumber] = layoutMapEntry fileInode.superBlockInodeBytesReferencedAdjustment -= int64(extentMapEntryValueV1.Length) diff --git a/iclient/iclientpkg/inode_test.go b/iclient/iclientpkg/inode_test.go index 460f3310..c04af920 100644 --- a/iclient/iclientpkg/inode_test.go +++ b/iclient/iclientpkg/inode_test.go @@ -19,15 +19,15 @@ import ( ) const ( - testInodeStressDisplayUpdateInterval = time.Duration(100 * time.Millisecond) + testInodeStressDisplayUpdateInterval = time.Duration(time.Second) testInodeStressFileNamePrefix = "_inode_stress_" - testInodeStressFileSize uint64 = 100000 // UNDO 1000000 - testInodeStressMaxExtentSize uint64 = 10000 + testInodeStressFileSize uint64 = 100 // UNDO 1000000 + testInodeStressMaxExtentSize uint64 = 4 // UNDO 10000 testInodeStressMinExtentSize uint64 = 1 - testInodeStressNumExtentWritesPerFile uint64 = 1000 - testInodeStressNumExtentWritesPerFlush uint64 = 50 // 0 means only perform Flush function at the end - testInodeStressNumExtentWritesPerValidate uint64 = 50 // 0 means only perform Validate function at the end - testInodeStressNumFiles uint64 = 1 + testInodeStressNumExtentWritesPerFile uint64 = 100 // UNDO 1000 + testInodeStressNumExtentWritesPerFlush uint64 = 1 // UNDO 50 // 0 means only perform Flush function at the end + testInodeStressNumExtentWritesPerValidate uint64 = 1 // UNDO 100 // 0 means only perform Validate function at the end + testInodeStressNumFiles uint64 = 1 // UNDO 10 ) type testInodeStressGlobalsStruct struct { From 9bd23013e77889bf383fed1135efe81a79f5b06d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 13 Dec 2021 11:03:54 -0800 Subject: [PATCH 200/258] Now that .bytesReferenced is fixed... we are back to hitting the mis-compoare :-( --- iclient/iclientpkg/inode_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/iclient/iclientpkg/inode_test.go b/iclient/iclientpkg/inode_test.go index c04af920..d8ba4ec6 100644 --- a/iclient/iclientpkg/inode_test.go +++ b/iclient/iclientpkg/inode_test.go @@ -21,13 +21,13 @@ import ( const ( testInodeStressDisplayUpdateInterval = time.Duration(time.Second) testInodeStressFileNamePrefix = "_inode_stress_" - testInodeStressFileSize uint64 = 100 // UNDO 1000000 - testInodeStressMaxExtentSize uint64 = 4 // UNDO 10000 - testInodeStressMinExtentSize uint64 = 1 - testInodeStressNumExtentWritesPerFile uint64 = 100 // UNDO 1000 - testInodeStressNumExtentWritesPerFlush uint64 = 1 // UNDO 50 // 0 means only perform Flush function at the end - testInodeStressNumExtentWritesPerValidate uint64 = 1 // UNDO 100 // 0 means only perform Validate function at the end - testInodeStressNumFiles uint64 = 1 // UNDO 10 + testInodeStressFileSize uint64 = 1000000 // UNDO 1000000 + testInodeStressMaxExtentSize uint64 = 10000 // UNDO 10000 + testInodeStressMinExtentSize uint64 = 1 // UNDO + testInodeStressNumExtentWritesPerFile uint64 = 1000 // UNDO 1000 + testInodeStressNumExtentWritesPerFlush uint64 = 50 // UNDO 50 // 0 means only perform Flush function at the end + testInodeStressNumExtentWritesPerValidate uint64 = 100 // UNDO 100 // 0 means only perform Validate function at the end + testInodeStressNumFiles uint64 = 1 // UNDO 10 ) type testInodeStressGlobalsStruct struct { From 979aa76653226efa25638e26ccab207f85fea3e7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 13 Dec 2021 13:56:08 -0800 Subject: [PATCH 201/258] Adjust TestInodeStress() to support repeatable pseudo-random sequence --- iclient/iclientpkg/inode_test.go | 63 ++++++++++++++++---------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/iclient/iclientpkg/inode_test.go b/iclient/iclientpkg/inode_test.go index d8ba4ec6..6562e2af 100644 --- a/iclient/iclientpkg/inode_test.go +++ b/iclient/iclientpkg/inode_test.go @@ -5,9 +5,11 @@ package iclientpkg import ( "bytes" - "crypto/rand" + crand "crypto/rand" "fmt" + "math" "math/big" + mrand "math/rand" "runtime" "sync" "sync/atomic" @@ -21,13 +23,14 @@ import ( const ( testInodeStressDisplayUpdateInterval = time.Duration(time.Second) testInodeStressFileNamePrefix = "_inode_stress_" - testInodeStressFileSize uint64 = 1000000 // UNDO 1000000 - testInodeStressMaxExtentSize uint64 = 10000 // UNDO 10000 - testInodeStressMinExtentSize uint64 = 1 // UNDO - testInodeStressNumExtentWritesPerFile uint64 = 1000 // UNDO 1000 - testInodeStressNumExtentWritesPerFlush uint64 = 50 // UNDO 50 // 0 means only perform Flush function at the end - testInodeStressNumExtentWritesPerValidate uint64 = 100 // UNDO 100 // 0 means only perform Validate function at the end - testInodeStressNumFiles uint64 = 1 // UNDO 10 + testInodeStressFileSize uint64 = 100000 + testInodeStressMaxExtentSize uint64 = 10000 + testInodeStressMinExtentSize uint64 = 1 + testInodeStressNumExtentWritesPerFile uint64 = 1000 + testInodeStressNumExtentWritesPerFlush uint64 = 0 // 0 means only perform Flush function at the end + testInodeStressNumExtentWritesPerValidate uint64 = 0 // 0 means only perform Validate function at the end + testInodeStressNumFiles uint64 = 1 + testInodeStressSeed int64 = 1 // if 0, use crypto/rand.Reader; else, use this seed + stresserIndex ) type testInodeStressGlobalsStruct struct { @@ -149,7 +152,8 @@ func testInodeStresser(stresserIndex uint64) { errno syscall.Errno extentIndex uint64 inHeader *fission.InHeader - mustBeLessThanBigIntPtr *big.Int + mrandRand *mrand.Rand + mrandSource mrand.Source numExtentWritesSinceLastFlush uint64 numExtentWritesSinceLastValidate uint64 offset uint64 @@ -214,17 +218,10 @@ func testInodeStresser(stresserIndex uint64) { runtime.Goexit() } - // Perform extent writes - - b = 0x00 - numExtentWritesSinceLastFlush = 0 - numExtentWritesSinceLastValidate = 0 - - for extentIndex = 0; extentIndex < testInodeStressNumExtentWritesPerFile; extentIndex++ { - // Pick a size value such that testInodeStressMinExtentSize <= size <= testInodeStressMaxExtentSize + // Construct this instance's pseudo-random sequence from appropriate seed - mustBeLessThanBigIntPtr = big.NewInt(int64(testInodeStressMaxExtentSize - testInodeStressMinExtentSize + 1)) - u64BigIntPtr, err = rand.Int(rand.Reader, mustBeLessThanBigIntPtr) + if testInodeStressSeed == 0 { + u64BigIntPtr, err = crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) if err != nil { testInodeStressGlobals.Lock() testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) @@ -235,23 +232,27 @@ func testInodeStresser(stresserIndex uint64) { runtime.Goexit() } - size = uint32(testInodeStressMinExtentSize + u64BigIntPtr.Uint64()) + mrandSource = mrand.NewSource(u64BigIntPtr.Int64()) + } else { + mrandSource = mrand.NewSource(testInodeStressSeed + int64(stresserIndex)) + } - // Pick an offset value such that 0 <= offset <= (testInodeStressFileSize - size) + mrandRand = mrand.New(mrandSource) - mustBeLessThanBigIntPtr = big.NewInt(int64(testInodeStressFileSize) - int64(size)) - u64BigIntPtr, err = rand.Int(rand.Reader, mustBeLessThanBigIntPtr) - if err != nil { - testInodeStressGlobals.Lock() - testInodeStressGlobals.err = append(testInodeStressGlobals.err, err) - testInodeStressGlobals.Unlock() + // Perform extent writes - testInodeStressGlobals.Done() + b = 0x00 + numExtentWritesSinceLastFlush = 0 + numExtentWritesSinceLastValidate = 0 - runtime.Goexit() - } + for extentIndex = 0; extentIndex < testInodeStressNumExtentWritesPerFile; extentIndex++ { + // Pick a size value such that testInodeStressMinExtentSize <= size <= testInodeStressMaxExtentSize + + size = uint32(mrandRand.Int63n(int64(testInodeStressMaxExtentSize - testInodeStressMinExtentSize + 1))) + + // Pick an offset value such that 0 <= offset <= (testInodeStressFileSize - size) - offset = u64BigIntPtr.Uint64() + offset = uint64(mrandRand.Int63n(int64(testInodeStressFileSize) - int64(size))) // Pick next b value (skipping 0x00 for as-yet-un-over-written bytes) From 906dca1288335238fe394f6e8f84024628d24f4c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 13 Dec 2021 15:16:08 -0800 Subject: [PATCH 202/258] Fixed bad ReadCacheLine ObjectOffset calculation --- iclient/iclientpkg/fission.go | 2 +- iclient/iclientpkg/inode_test.go | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 0cd088fc..14f01500 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1893,7 +1893,7 @@ Retry: readCacheLineBuf, err = objectGETRange( readCacheLine.key.objectNumber, - readCacheLine.key.lineNumber&globals.config.ReadCacheLineSize, + readCacheLine.key.lineNumber*globals.config.ReadCacheLineSize, globals.config.ReadCacheLineSize) if nil != err { diff --git a/iclient/iclientpkg/inode_test.go b/iclient/iclientpkg/inode_test.go index 6562e2af..33e58ff8 100644 --- a/iclient/iclientpkg/inode_test.go +++ b/iclient/iclientpkg/inode_test.go @@ -23,13 +23,13 @@ import ( const ( testInodeStressDisplayUpdateInterval = time.Duration(time.Second) testInodeStressFileNamePrefix = "_inode_stress_" - testInodeStressFileSize uint64 = 100000 + testInodeStressFileSize uint64 = 10000000 testInodeStressMaxExtentSize uint64 = 10000 testInodeStressMinExtentSize uint64 = 1 testInodeStressNumExtentWritesPerFile uint64 = 1000 - testInodeStressNumExtentWritesPerFlush uint64 = 0 // 0 means only perform Flush function at the end - testInodeStressNumExtentWritesPerValidate uint64 = 0 // 0 means only perform Validate function at the end - testInodeStressNumFiles uint64 = 1 + testInodeStressNumExtentWritesPerFlush uint64 = 50 // 0 means only perform Flush function at the end + testInodeStressNumExtentWritesPerValidate uint64 = 100 // 0 means only perform Validate function at the end + testInodeStressNumFiles uint64 = 10 testInodeStressSeed int64 = 1 // if 0, use crypto/rand.Reader; else, use this seed + stresserIndex ) @@ -454,7 +454,6 @@ func (tISC *testInodeStresserContext) flush() (err error) { func (tISC *testInodeStresserContext) validate() (err error) { var ( errno syscall.Errno - i int inHeader *fission.InHeader readIn *fission.ReadIn readOut *fission.ReadOut @@ -475,12 +474,6 @@ func (tISC *testInodeStresserContext) validate() (err error) { err = nil } else { err = fmt.Errorf("Miscompare in fileName %s\n", tISC.fileName) - for i = range tISC.written { - if tISC.written[i] != readOut.Data[i] { - err = fmt.Errorf("First miscompare in %s at position %v", tISC.fileName, i) - return - } - } } } else { err = fmt.Errorf("globals.DoRead(inHeader, readIn) returned errno: %v", errno) From 0eb9a11fabb548f73bf0d013db0c7a7602ea631a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 13 Dec 2021 16:34:00 -0800 Subject: [PATCH 203/258] Fix gap handling and avoid deleting active PUT Object in unmapExtent() --- iclient/iclientpkg/inode.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index edc9de34..c8ea719f 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -1136,6 +1136,13 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint if !ok { logFatalf("extentMapEntryKeyV1AsKey.(uint64) returned !ok") } + + if extentMapEntryKeyV1 >= (startingFileOffset + length) { + // The remaining unmap range fits in the gap before the next extent begins so we are done + + return + } + extentMapEntryValueV1, ok = extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) if !ok { logFatalf("extentMapEntryValueV1AsValue.(*ilayout.ExtentMapEntryValueV1Struct) returned !ok") @@ -1202,11 +1209,11 @@ func (fileInode *inodeStruct) unmapExtent(startingFileOffset uint64, length uint logFatalf("fileInode.payload.DeleteByIndex() returned !ok") } - // Update the layoutMapEntry... possibly deleting it if .bytesReferences reaches 0 + // Update the layoutMapEntry... possibly deleting it if .bytesReferences reaches 0 (and it's not the current putObject) layoutMapEntry.bytesReferenced -= extentMapEntryValueV1.Length - if layoutMapEntry.bytesReferenced == 0 { + if (layoutMapEntry.bytesReferenced == 0) && (extentMapEntryValueV1.ObjectNumber != fileInode.putObjectNumber) { delete(fileInode.layoutMap, extentMapEntryValueV1.ObjectNumber) fileInode.superBlockInodeObjectCountAdjustment-- From 45f5a72f092343975dd1a4a7d89e4f68304b38a0 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 14 Dec 2021 14:39:20 -0800 Subject: [PATCH 204/258] Added Read/Write Cache bucketstats --- iclient/iclientpkg/fission.go | 29 +++++++++++++++++++++-------- iclient/iclientpkg/globals.go | 7 +++++++ iclient/iclientpkg/inode.go | 7 ++++++- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/iclient/iclientpkg/fission.go b/iclient/iclientpkg/fission.go index 14f01500..5f41db04 100644 --- a/iclient/iclientpkg/fission.go +++ b/iclient/iclientpkg/fission.go @@ -1547,6 +1547,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R readCacheLine *readCacheLineStruct readCacheLineBuf []byte readCacheLineBufLengthAvailableToConsume uint64 + readCacheLineFillStartTime time.Time readCacheLineOffset uint64 readCacheLineToEvict *readCacheLineStruct readCacheLineToEvictListElement *list.Element @@ -1813,6 +1814,7 @@ Retry: readOut.Data = append(readOut.Data, make([]byte, readPlanEntry.Length)...) case inode.putObjectNumber: // Note that if putObject is inactive, case 0: will have already matched readPlanEntry.ObjectNumber + globals.stats.WriteBackCacheHits.Increment() readOut.Data = append(readOut.Data, inode.putObjectBuffer[readPlanEntry.ObjectOffset:(readPlanEntry.ObjectOffset+readPlanEntry.Length)]...) default: for readPlanEntry.Length > 0 { @@ -1829,6 +1831,8 @@ Retry: if ok { // readCacheLine is in globals.readCacheMap but may be being filled + globals.stats.ReadCacheHits.Increment() + globals.readCacheLRU.MoveToBack(readCacheLine.listElement) if readCacheLine.wg == nil { @@ -1891,6 +1895,8 @@ Retry: globals.Unlock() + readCacheLineFillStartTime = time.Now() + readCacheLineBuf, err = objectGETRange( readCacheLine.key.objectNumber, readCacheLine.key.lineNumber*globals.config.ReadCacheLineSize, @@ -1915,6 +1921,8 @@ Retry: return } + globals.stats.ReadCacheMissUsecs.Add(uint64(time.Since(readCacheLineFillStartTime) / time.Microsecond)) + // readCacheLine fill succeeded... so tell others and continue globals.Lock() @@ -1966,14 +1974,15 @@ Retry: func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission.WriteIn) (writeOut *fission.WriteOut, errno syscall.Errno) { var ( - err error - inode *inodeStruct - inodeLockRequest *inodeLockRequestStruct - newSize uint64 - offset uint64 - oldSize uint64 - openHandle *openHandleStruct - startTime time.Time = time.Now() + err error + fileFlushSizeTriggerStartTime time.Time + inode *inodeStruct + inodeLockRequest *inodeLockRequestStruct + newSize uint64 + offset uint64 + oldSize uint64 + openHandle *openHandleStruct + startTime time.Time = time.Now() ) logTracef("==> DoWrite(inHeader: %+v, writeIn: &{FH:%v Offset:%v Size:%v: WriteFlags:%v LockOwner:%v Flags:%v Padding:%v len(Data):%v})", inHeader, writeIn.FH, writeIn.Offset, writeIn.Size, writeIn.WriteFlags, writeIn.LockOwner, writeIn.Flags, writeIn.Padding, len(writeIn.Data)) @@ -2072,8 +2081,12 @@ Retry: // Pre-flush if this write would cause inode.putObjectBuffer to exceed globals.config.FileFlushTriggerSize if (uint64(len(inode.putObjectBuffer)) + uint64(len(writeIn.Data))) > globals.config.FileFlushTriggerSize { + fileFlushSizeTriggerStartTime = time.Now() + flushInodesInSlice([]*inodeStruct{inode}) + globals.stats.FileFlushSizeTriggerUsecs.Add(uint64(time.Since(fileFlushSizeTriggerStartTime) / time.Microsecond)) + inode.dirty = true inode.ensurePutObjectIsActive() diff --git a/iclient/iclientpkg/globals.go b/iclient/iclientpkg/globals.go index b986ed0c..d7d14932 100644 --- a/iclient/iclientpkg/globals.go +++ b/iclient/iclientpkg/globals.go @@ -232,6 +232,13 @@ type statsStruct struct { DoReadBytes bucketstats.BucketLog2Round // (*globalsStruct)DoRead() DoWriteBytes bucketstats.BucketLog2Round // (*globalsStruct)DoWrite() + + WriteBackCacheHits bucketstats.Total // DoRead() readPlanEntry hit unflushed fileInode.putObjectBuffer + ReadCacheHits bucketstats.Total // DoRead() readPlanEntry hit readCacheLine already in memory + ReadCacheMissUsecs bucketstats.BucketLog2Round // DoRead() readPlanEntry hit readCacheLine needing to be fetched + + FileFlushSizeTriggerUsecs bucketstats.BucketLog2Round // FileFlushTriggerSize reached + FileFlushDurationTriggerUsecs bucketstats.BucketLog2Round // FileFlushTriggerDuration reached } type globalsStruct struct { diff --git a/iclient/iclientpkg/inode.go b/iclient/iclientpkg/inode.go index c8ea719f..09ec59c9 100644 --- a/iclient/iclientpkg/inode.go +++ b/iclient/iclientpkg/inode.go @@ -670,7 +670,8 @@ func (fileInode *inodeStruct) launchFlusher() { func (fileFlusher *fileInodeFlusherStruct) gor() { var ( - inodeLockRequest *inodeLockRequestStruct + fileFlushDurationTriggerStartTime time.Time + inodeLockRequest *inodeLockRequestStruct ) select { @@ -699,7 +700,11 @@ func (fileFlusher *fileInodeFlusherStruct) gor() { // We now know that fileFlusher.inode.dirty == true and // that we need to flush "our" fileFlusher.putObjectNumber + fileFlushDurationTriggerStartTime = time.Now() + flushInodesInSlice([]*inodeStruct{fileFlusher.inode}) + + globals.stats.FileFlushDurationTriggerUsecs.Add(uint64(time.Since(fileFlushDurationTriggerStartTime) / time.Microsecond)) } inodeLockRequest.unlockAll() From 0c38bdac1386441710e0a47de887b699b11810eb Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 14 Dec 2021 15:28:16 -0800 Subject: [PATCH 205/258] Fixed bad calculation for extent size to write ...the unintentional truncate of the test file was hit when selecting an extent size of 0 (and bypassing the check for that) --- iclient/iclientpkg/inode_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/iclient/iclientpkg/inode_test.go b/iclient/iclientpkg/inode_test.go index 33e58ff8..639ec3fd 100644 --- a/iclient/iclientpkg/inode_test.go +++ b/iclient/iclientpkg/inode_test.go @@ -26,11 +26,11 @@ const ( testInodeStressFileSize uint64 = 10000000 testInodeStressMaxExtentSize uint64 = 10000 testInodeStressMinExtentSize uint64 = 1 - testInodeStressNumExtentWritesPerFile uint64 = 1000 + testInodeStressNumExtentWritesPerFile uint64 = 10000 testInodeStressNumExtentWritesPerFlush uint64 = 50 // 0 means only perform Flush function at the end testInodeStressNumExtentWritesPerValidate uint64 = 100 // 0 means only perform Validate function at the end testInodeStressNumFiles uint64 = 10 - testInodeStressSeed int64 = 1 // if 0, use crypto/rand.Reader; else, use this seed + stresserIndex + testInodeStressSeed int64 = 0 // if 0, use crypto/rand.Reader; else, use this seed + stresserIndex ) type testInodeStressGlobalsStruct struct { @@ -248,7 +248,7 @@ func testInodeStresser(stresserIndex uint64) { for extentIndex = 0; extentIndex < testInodeStressNumExtentWritesPerFile; extentIndex++ { // Pick a size value such that testInodeStressMinExtentSize <= size <= testInodeStressMaxExtentSize - size = uint32(mrandRand.Int63n(int64(testInodeStressMaxExtentSize - testInodeStressMinExtentSize + 1))) + size = uint32(mrandRand.Int63n(int64(testInodeStressMaxExtentSize-testInodeStressMinExtentSize+1))) + uint32(testInodeStressMinExtentSize) // Pick an offset value such that 0 <= offset <= (testInodeStressFileSize - size) From e91c7ff37284a76a58488a9bf4af6aa9eb2c3d0c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 11:09:06 -0800 Subject: [PATCH 206/258] Reworked the Open Count logic a bit...to trigger GC on deleted inodes --- imgr/imgrpkg/api.go | 5 +- imgr/imgrpkg/globals.go | 7 +- imgr/imgrpkg/retry-rpc.go | 137 +++++++++++++++++++------------------- imgr/imgrpkg/volume.go | 24 +++++-- 4 files changed, 94 insertions(+), 79 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 0918ec9c..73370e4d 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -402,10 +402,7 @@ type AdjustInodeTableEntryOpenCountRequestStruct struct { // AdjustInodeTableEntryOpenCountResponseStruct is the response object for AdjustInodeTableEntryOpenCount. // -type AdjustInodeTableEntryOpenCountResponseStruct struct { - CurrentOpenCountThisMount uint64 - CurrentOpenCountAllMounts uint64 -} +type AdjustInodeTableEntryOpenCountResponseStruct struct{} // AdjustInodeTableEntryOpenCount requests the specified Inode's OpenCount be // adjusted. If the referenced InodeNumber is non-zero, a (Shared or Exclusive) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 57ee1435..fe99737a 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -226,6 +226,11 @@ type inodeTableLayoutElementStruct struct { bytesReferenced uint64 // matches ilayout.InodeTableLayoutEntryV1Struct.BytesReferenced } +type inodeOpenMapElementStruct struct { + numMounts uint64 // number of mountStruct's with a non-zero open count for this inode + markedForDeletion bool // if true, when numMounts falls to zero, the inode will be removed from the inodeTable +} + type volumeStruct struct { name string // storageURL string // @@ -247,7 +252,7 @@ type volumeStruct struct { checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG checkPointPutObjectNumber uint64 // checkPointPutObjectBuffer *bytes.Buffer // if nil, no CheckPoint data to PUT has yet accumulated - inodeOpenMap map[uint64]uint64 // key == inodeNumber; value == open count for all mountStruct's for this inodeNumber + inodeOpenMap map[uint64]*inodeOpenMapElementStruct // key == inodeNumber inodeLeaseMap map[uint64]*inodeLeaseStruct // key == inodeLeaseStruct.inodeNumber leaseHandlerWG sync.WaitGroup // .Add(1) each inodeLease insertion into inodeLeaseMap // .Done() each inodeLease after it is removed from inodeLeaseMap diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 57e2bbea..9ab81d6c 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -479,10 +479,11 @@ func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesReque func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { var ( - leaseRequest *leaseRequestStruct - mount *mountStruct - ok bool - startTime time.Time = time.Now() + inodeOpenMapElement *inodeOpenMapElementStruct + leaseRequest *leaseRequestStruct + mount *mountStruct + ok bool + startTime time.Time = time.Now() ) defer func() { @@ -511,14 +512,11 @@ func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRe return } - // TODO: Need to actually clean up the Inode... but for now, just remove it - - ok, err = mount.volume.inodeTable.DeleteByKey(deleteInodeTableEntryRequest.InodeNumber) - if nil != err { - logFatalf("volume.inodeTable.DeleteByKey(n.InodeNumber) failed: %v", err) - } - if !ok { - logFatalf("volume.inodeTable.DeleteByKey(n.InodeNumber) returned !ok") + inodeOpenMapElement, ok = mount.volume.inodeOpenMap[deleteInodeTableEntryRequest.InodeNumber] + if ok { + inodeOpenMapElement.markedForDeletion = true + } else { + mount.volume.removeInodeWhileLocked(deleteInodeTableEntryRequest.InodeNumber) } globals.Unlock() @@ -527,25 +525,25 @@ func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRe return } -// TODO - The thinking is that some "to-be-deleted-upon-last-close" logic will substitute -// for the InodeTable entry... and that some background garbage collector will do the work. - func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { var ( - leaseRequest *leaseRequestStruct - mount *mountStruct - ok bool - newMountOpenCount uint64 - newVolumeOpenCount uint64 - oldMountOpenCount uint64 - oldVolumeOpenCount uint64 - startTime time.Time = time.Now() + inodeOpenCount uint64 + inodeOpenMapElement *inodeOpenMapElementStruct + leaseRequest *leaseRequestStruct + mount *mountStruct + ok bool + startTime time.Time = time.Now() ) defer func() { globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() + if adjustInodeTableEntryOpenCountRequest.Adjustment == 0 { + err = fmt.Errorf("%s %016X %v", EBadOpenCountAdjustment, adjustInodeTableEntryOpenCountRequest.InodeNumber, adjustInodeTableEntryOpenCountRequest.Adjustment) + return + } + globals.Lock() mount, ok = globals.mountMap[adjustInodeTableEntryOpenCountRequest.MountID] @@ -568,66 +566,67 @@ func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *Adjus return } - if adjustInodeTableEntryOpenCountRequest.Adjustment >= 0 { - oldMountOpenCount, ok = mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] - if ok { - oldVolumeOpenCount, ok = mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] - if !ok { - logFatalf("mount.volumeinodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] returned !ok after mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] returned ok") - } - } else { - oldMountOpenCount = 0 - oldVolumeOpenCount, ok = mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] - if !ok { - oldVolumeOpenCount = 0 - } + inodeOpenCount, ok = mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] + if ok { + if inodeOpenCount == 0 { + logFatalf("mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] returned 0") } - - if adjustInodeTableEntryOpenCountRequest.Adjustment == 0 { - newMountOpenCount = oldMountOpenCount - newVolumeOpenCount = oldVolumeOpenCount - } else { - newMountOpenCount = oldMountOpenCount + uint64(adjustInodeTableEntryOpenCountRequest.Adjustment) - newVolumeOpenCount = oldVolumeOpenCount + uint64(adjustInodeTableEntryOpenCountRequest.Adjustment) - - mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = newMountOpenCount - mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = newVolumeOpenCount - } - } else { // adjustInodeTableEntryOpenCountRequest.Adjustment < 0 - oldMountOpenCount, ok = mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] - if !ok || (oldMountOpenCount < uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment)) { + if (adjustInodeTableEntryOpenCountRequest.Adjustment < 0) && (uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment) > inodeOpenCount) { globals.Unlock() err = fmt.Errorf("%s %016X %v", EBadOpenCountAdjustment, adjustInodeTableEntryOpenCountRequest.InodeNumber, adjustInodeTableEntryOpenCountRequest.Adjustment) return } - oldVolumeOpenCount, ok = mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] - if !ok || (oldVolumeOpenCount < uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment)) { - logFatalf("mount.volumeinodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] returned !ok || oldVolumeOpenCount < uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment)") + } else { + if adjustInodeTableEntryOpenCountRequest.Adjustment < 0 { + globals.Unlock() + err = fmt.Errorf("%s %016X %v", EBadOpenCountAdjustment, adjustInodeTableEntryOpenCountRequest.InodeNumber, adjustInodeTableEntryOpenCountRequest.Adjustment) + return } + inodeOpenCount = 0 + } - newMountOpenCount = oldMountOpenCount - uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment) - newVolumeOpenCount = oldVolumeOpenCount - uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment) - - if newMountOpenCount == 0 { - delete(mount.inodeOpenMap, adjustInodeTableEntryOpenCountRequest.InodeNumber) - } else { - mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = newMountOpenCount + inodeOpenMapElement, ok = mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] + if ok { + if inodeOpenMapElement.numMounts == 0 { + logFatalf("mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] returned inodeOpenMapElement.numMounts == 0") } + } else { + if inodeOpenCount != 0 { + logFatalf("inodeOpenCount can't be != 0 if inodeOpenMapElement is missing") + } + if adjustInodeTableEntryOpenCountRequest.Adjustment < 0 { + logFatalf("adjustInodeTableEntryOpenCountRequest.Adjustment can't be < 0 if inodeOpenMapElement is missing") + } + inodeOpenMapElement = &inodeOpenMapElementStruct{ + numMounts: 0, + markedForDeletion: false, + } + mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = inodeOpenMapElement + } - if newVolumeOpenCount == 0 { - delete(mount.volume.inodeOpenMap, adjustInodeTableEntryOpenCountRequest.InodeNumber) - - // TODO: Check for pending delete... and, if so, actuially delete the inode - } else { - mount.volume.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = newVolumeOpenCount + if adjustInodeTableEntryOpenCountRequest.Adjustment > 0 { + if inodeOpenCount == 0 { + mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = uint64(adjustInodeTableEntryOpenCountRequest.Adjustment) + inodeOpenMapElement.numMounts++ + } + } else { // adjustInodeTableEntryOpenCountRequest.Adjustment < 0 [we already know it is != 0] + inodeOpenCount -= uint64(-adjustInodeTableEntryOpenCountRequest.Adjustment) + if inodeOpenCount == 0 { + delete(mount.inodeOpenMap, adjustInodeTableEntryOpenCountRequest.InodeNumber) + inodeOpenMapElement.numMounts-- + if inodeOpenMapElement.numMounts == 0 { + delete(mount.volume.inodeOpenMap, adjustInodeTableEntryOpenCountRequest.InodeNumber) + if inodeOpenMapElement.markedForDeletion { + mount.volume.removeInodeWhileLocked(adjustInodeTableEntryOpenCountRequest.InodeNumber) + } + } + } else { // [adjusted] inodeOpenCount > 0 + mount.inodeOpenMap[adjustInodeTableEntryOpenCountRequest.InodeNumber] = inodeOpenCount } } globals.Unlock() - adjustInodeTableEntryOpenCountResponse.CurrentOpenCountThisMount = newMountOpenCount - adjustInodeTableEntryOpenCountResponse.CurrentOpenCountAllMounts = newVolumeOpenCount - err = nil return } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 392ba218..14b3c742 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -630,7 +630,7 @@ func putVolume(name string, storageURL string, authToken string) (err error) { checkPointControlChan: nil, checkPointPutObjectNumber: 0, checkPointPutObjectBuffer: nil, - inodeOpenMap: make(map[uint64]uint64), + inodeOpenMap: make(map[uint64]*inodeOpenMapElementStruct), inodeLeaseMap: make(map[uint64]*inodeLeaseStruct), } @@ -761,10 +761,6 @@ func (volume *volumeStruct) doCheckPoint() (err error) { }) } - // volume.superBlock.InodeObjectCount is already set - // volume.superBlock.InodeObjectSize is already set - // volume.superBlock.InodeBytesReferenced is already set - volume.superBlock.PendingDeleteObjectNumberArray = make([]uint64, 0, volume.activeDeleteObjectNumberList.Len()+volume.pendingDeleteObjectNumberList.Len()) deleteObjectNumberListElement = volume.activeDeleteObjectNumberList.Front() @@ -832,6 +828,7 @@ func (volume *volumeStruct) doCheckPoint() (err error) { err = fmt.Errorf("volume.swiftObjectPut(ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) returned !authOK") return } + // TODO: Somewhere near here I need to kick off the Object Delete logic... globals.Unlock() @@ -1156,3 +1153,20 @@ func (volume *volumeStruct) fetchNonceRangeWhileLocked() (nextNonce uint64, numN return } + +func (volume *volumeStruct) removeInodeWhileLocked(inodeNumber uint64) { + var ( + err error + ok bool + ) + + // TODO: Need to do the garbage collection/cleanup for the inode here... + + ok, err = volume.inodeTable.DeleteByKey(inodeNumber) + if nil != err { + logFatalf("volume.inodeTable.DeleteByKey(inodeNumber: %016X) failed: %v", inodeNumber, err) + } + if !ok { + logFatalf("volume.inodeTable.DeleteByKey(inodeNumber: %016X) returned !ok", inodeNumber) + } +} From bf69f38d1786ef5edccfe04a89114ce9bb0967c1 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 14:09:33 -0800 Subject: [PATCH 207/258] Fixed scheduling of pending object deletes --- ilayout/api.go | 6 ++--- imgr/imgrpkg/volume.go | 55 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index f9668ba0..506c0a90 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -215,9 +215,9 @@ type SuperBlockV1Struct struct { InodeTableRootObjectOffset uint64 // Starting offset in the Object of the root of the InodeTable InodeTableRootObjectLength uint64 // Number of bytes in the Object of the root of the InodeTable InodeTableLayout []InodeTableLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable - InodeObjectCount uint64 // Number of Objects holding Inodes - InodeObjectSize uint64 // Sum of sizes of all Objects holding Inodes - InodeBytesReferenced uint64 // Sum of bytes referenced in all Objects holding Inodes + InodeObjectCount uint64 // Number of Objects holding {Dir|File}Inode Payload-described B+Tree as well as the FileInode's contents + InodeObjectSize uint64 // Sum of sizes of all Objects holding {Dir|File}Inode Payload-described B+Tree as well as the FileInode's contents + InodeBytesReferenced uint64 // Sum of bytes referenced in all Objects holding {Dir|File}Inode Payload-described B+Tree as well as the FileInode's contents PendingDeleteObjectNumberArray []uint64 // List of Objects to be deleted after the this CheckPoint } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 14b3c742..14bd9107 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -1156,11 +1156,60 @@ func (volume *volumeStruct) fetchNonceRangeWhileLocked() (nextNonce uint64, numN func (volume *volumeStruct) removeInodeWhileLocked(inodeNumber uint64) { var ( - err error - ok bool + authOK bool + err error + inodeHeadLayoutEntryV1 ilayout.InodeHeadLayoutEntryV1Struct + inodeHeadObjectNumberInLayout bool + inodeHeadV1 *ilayout.InodeHeadV1Struct + inodeHeadV1Buf []byte + inodeTableEntryValue ilayout.InodeTableEntryValueV1Struct + inodeTableEntryValueRaw sortedmap.Value + ok bool ) - // TODO: Need to do the garbage collection/cleanup for the inode here... + inodeTableEntryValueRaw, ok, err = volume.inodeTable.GetByKey(inodeNumber) + if nil != err { + logFatalf("volume.inodeTable.GetByKey(inodeNumber) failed: %v", err) + } + if !ok { + logFatalf("volume.inodeTable.GetByKey(inodeNumber) returned !ok") + } + + inodeTableEntryValue, ok = inodeTableEntryValueRaw.(ilayout.InodeTableEntryValueV1Struct) + if !ok { + logFatalf("inodeTableEntryValueRaw.(ilayout.InodeTableEntryValueV1Struct) returned !ok") + } + + inodeHeadV1Buf, authOK, err = volume.swiftObjectGetTail(inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadLength) + if nil != err { + logFatalf("volume.swiftObjectGetTail(inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadObjectNumber) failed: %v", err) + } + if !authOK { + logFatalf("volume.swiftObjectGetTail(inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadObjectNumber) returned !authOK") + } + + inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) + if nil != err { + logFatalf("ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) failed: %v", err) + } + + inodeHeadObjectNumberInLayout = false + + for _, inodeHeadLayoutEntryV1 = range inodeHeadV1.Layout { + _ = volume.pendingDeleteObjectNumberList.PushBack(inodeHeadLayoutEntryV1.ObjectNumber) + + volume.superBlock.InodeObjectCount-- + volume.superBlock.InodeObjectSize -= inodeHeadLayoutEntryV1.ObjectSize + volume.superBlock.InodeBytesReferenced -= inodeHeadLayoutEntryV1.BytesReferenced + + if inodeHeadLayoutEntryV1.ObjectNumber == inodeTableEntryValue.InodeHeadObjectNumber { + inodeHeadObjectNumberInLayout = true + } + } + + if !inodeHeadObjectNumberInLayout { + _ = volume.pendingDeleteObjectNumberList.PushBack(inodeTableEntryValue.InodeHeadObjectNumber) + } ok, err = volume.inodeTable.DeleteByKey(inodeNumber) if nil != err { From b29953c32512d16d99d56bb52ef7cd324df4cc47 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 15:04:26 -0800 Subject: [PATCH 208/258] Inode ObjectDelete's largely working... but not CheckPoint ObjectDelete's ...also, CheckPoint's aren't cleaning up PendingDeleteObjectNumberArray --- iclient/iclientpkg/utils_test.go | 2 +- imgr/dev.conf | 2 +- imgr/imgr.conf | 2 +- imgr/imgrpkg/api.go | 2 +- imgr/imgrpkg/globals.go | 23 +++------------- imgr/imgrpkg/utils_test.go | 2 +- imgr/imgrpkg/volume.go | 47 +++++++++++++++++++++++++++++++- 7 files changed, 55 insertions(+), 25 deletions(-) diff --git a/iclient/iclientpkg/utils_test.go b/iclient/iclientpkg/utils_test.go index bda85152..5d7c40a7 100644 --- a/iclient/iclientpkg/utils_test.go +++ b/iclient/iclientpkg/utils_test.go @@ -205,7 +205,7 @@ func testSetup(t *testing.T) { "IMGR.SwiftTimeout=10m", "IMGR.SwiftConnectionPoolSize=128", - "IMGR.ParallelObjectDeleteMax=10", + "IMGR.ParallelObjectDeletePerVolumeLimit=10", "IMGR.InodeTableCacheEvictLowLimit=10000", "IMGR.InodeTableCacheEvictHighLimit=10010", diff --git a/imgr/dev.conf b/imgr/dev.conf index f4eff423..d4714834 100644 --- a/imgr/dev.conf +++ b/imgr/dev.conf @@ -34,7 +34,7 @@ SwiftRetryLimit: 4 SwiftTimeout: 10m SwiftConnectionPoolSize: 128 -ParallelObjectDeleteMax: 10 +ParallelObjectDeletePerVolumeLimit: 10 InodeTableCacheEvictLowLimit: 10000 InodeTableCacheEvictHighLimit: 10010 diff --git a/imgr/imgr.conf b/imgr/imgr.conf index fb98ef88..baac4a7f 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -34,7 +34,7 @@ SwiftRetryLimit: 4 SwiftTimeout: 10m SwiftConnectionPoolSize: 128 -ParallelObjectDeleteMax: 10 +ParallelObjectDeletePerVolumeLimit: 10 InodeTableCacheEvictLowLimit: 10000 InodeTableCacheEvictHighLimit: 10010 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 73370e4d..2cc5e086 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -47,7 +47,7 @@ // SwiftTimeout: 10m // SwiftConnectionPoolSize: 128 // -// ParallelObjectDeleteMax: 10 +// ParallelObjectDeletePerVolumeLimit: 10 // // InodeTableCacheEvictLowLimit: 10000 // InodeTableCacheEvictHighLimit: 10010 diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index fe99737a..209dda5e 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -54,7 +54,7 @@ type configStruct struct { SwiftTimeout time.Duration SwiftConnectionPoolSize uint32 - ParallelObjectDeleteMax uint32 + ParallelObjectDeletePerVolumeLimit uint32 InodeTableCacheEvictLowLimit uint64 InodeTableCacheEvictHighLimit uint64 @@ -67,22 +67,6 @@ type configStruct struct { TraceEnabled bool } -type chunkedPutContextStruct struct { - sync.WaitGroup // Used to await completion of performChunkedPut goroutine - containerName string // - objectName string // - buf []byte // - chunkedPutListElement *list.Element // FIFO Element of fileInodeStruct.chunkedPutList - state uint8 // One of chunkedPutContextState{Open|Closing|Closed} - pos int // ObjectOffset just after last sent chunk - sendChan chan struct{} // Single element buffered chan to wake up *chunkedPutContextStruct.sendDaemon() - // will be closed to indicate a flush is requested - wakeChan chan struct{} // Single element buffered chan to wake up *chunkedPutContextStruct.Read() - // will be closed to indicate a flush is requested - inRead bool // Set when in Read() as a hint to Close() to help Read() cleanly exit - flushRequested bool // Set to remember that a flush has been requested of *chunkedPutContextStruct.Read() -} - type statsStruct struct { DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ GetConfigUsecs bucketstats.BucketLog2Round // GET /config @@ -246,6 +230,7 @@ type volumeStruct struct { inodeTableLayout map[uint64]*inodeTableLayoutElementStruct // == nil if not currently mounted and/or checkpointing; key == objectNumber (matching ilayout.InodeTableLayoutEntryV1Struct.ObjectNumber) nextNonce uint64 // next nonce in that checkpoint reserve numNoncesReserved uint64 // number of nonce's reserved for checkpointing + activeDeleteObjectWG sync.WaitGroup // doObjectDelete() indicates it is done by calling .Done() on this WG activeDeleteObjectNumberList *list.List // list of objectNumber's to be deleted since last CheckPoint pendingDeleteObjectNumberList *list.List // list of objectNumber's pending deletion after next CheckPoint checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() @@ -430,7 +415,7 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { logFatal(err) } - globals.config.ParallelObjectDeleteMax, err = confMap.FetchOptionValueUint32("IMGR", "ParallelObjectDeleteMax") + globals.config.ParallelObjectDeletePerVolumeLimit, err = confMap.FetchOptionValueUint32("IMGR", "ParallelObjectDeletePerVolumeLimit") if nil != err { logFatal(err) } @@ -516,7 +501,7 @@ func uninitializeGlobals() (err error) { globals.config.SwiftTimeout = time.Duration(0) globals.config.SwiftConnectionPoolSize = 0 - globals.config.ParallelObjectDeleteMax = 0 + globals.config.ParallelObjectDeletePerVolumeLimit = 0 globals.config.InodeTableCacheEvictLowLimit = 0 globals.config.InodeTableCacheEvictHighLimit = 0 diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index b40c6373..0e82c516 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -151,7 +151,7 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { "IMGR.SwiftTimeout=10m", "IMGR.SwiftConnectionPoolSize=128", - "IMGR.ParallelObjectDeleteMax=10", + "IMGR.ParallelObjectDeletePerVolumeLimit=10", "IMGR.InodeTableCacheEvictLowLimit=10000", "IMGR.InodeTableCacheEvictHighLimit=10010", diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 14bd9107..948fe387 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -99,6 +99,8 @@ func deleteVolume(volumeName string) (err error) { logFatalf("No support for deleting actively mounted volume \"%s\"", volumeName) } + volumeAsStruct.activeDeleteObjectWG.Wait() + ok, err = globals.volumeMap.DeleteByKey(volumeAsStruct.name) if nil != err { logFatal(err) @@ -828,7 +830,19 @@ func (volume *volumeStruct) doCheckPoint() (err error) { err = fmt.Errorf("volume.swiftObjectPut(ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) returned !authOK") return } - // TODO: Somewhere near here I need to kick off the Object Delete logic... + + for (uint32(volume.activeDeleteObjectNumberList.Len()) < globals.config.ParallelObjectDeletePerVolumeLimit) && (volume.pendingDeleteObjectNumberList.Len() > 0) { + deleteObjectNumberListElement = volume.pendingDeleteObjectNumberList.Front() + _ = volume.pendingDeleteObjectNumberList.Remove(deleteObjectNumberListElement) + objectNumber, ok = deleteObjectNumberListElement.Value.(uint64) + if !ok { + err = fmt.Errorf("deleteObjectNumberListElement.Value.(uint64) returned !ok") + logFatal(err) + } + deleteObjectNumberListElement = volume.activeDeleteObjectNumberList.PushBack(objectNumber) + volume.activeDeleteObjectWG.Add(1) + go volume.doObjectDelete(deleteObjectNumberListElement) + } globals.Unlock() @@ -836,6 +850,37 @@ func (volume *volumeStruct) doCheckPoint() (err error) { return } +func (volume *volumeStruct) doObjectDelete(activeDeleteObjectNumberListElement *list.Element) { + var ( + authOK bool + err error + objectNumber uint64 + ok bool + ) + + objectNumber, ok = activeDeleteObjectNumberListElement.Value.(uint64) + if !ok { + logFatalf("activeDeleteObjectNumberListElement.Value.(uint64) returned !ok") + } + + authOK, err = volume.swiftObjectDelete(objectNumber) + if nil != err { + logFatalf("volume.swiftObjectDelete(objectNumber: %016X) failed: %v", objectNumber, err) + } + + globals.Lock() + + volume.activeDeleteObjectNumberList.Remove(activeDeleteObjectNumberListElement) + + if !authOK { + volume.pendingDeleteObjectNumberList.PushBack(objectNumber) + } + + globals.Unlock() + + volume.activeDeleteObjectWG.Done() +} + func (volume *volumeStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { var ( keyAsInodeNumber uint64 From a185ecbb8d0901b21703f23366e58219d93f7f68 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 16:26:12 -0800 Subject: [PATCH 209/258] Completed the rest of the object delete garbage collection paths --- imgr/imgrpkg/globals.go | 3 +- imgr/imgrpkg/retry-rpc.go | 4 ++ imgr/imgrpkg/volume.go | 118 +++++++++++++++++++++++--------------- 3 files changed, 79 insertions(+), 46 deletions(-) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 209dda5e..28269215 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -217,6 +217,7 @@ type inodeOpenMapElementStruct struct { type volumeStruct struct { name string // + dirty bool // storageURL string // authToken string // if != "" & healthyMountList is empty, this AuthToken will be used; cleared on auth failure mountMap map[string]*mountStruct // key == mountStruct.mountID @@ -235,7 +236,7 @@ type volumeStruct struct { pendingDeleteObjectNumberList *list.List // list of objectNumber's pending deletion after next CheckPoint checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG - checkPointPutObjectNumber uint64 // + checkPointObjectNumber uint64 // if non-zero, contains ObjectNumber of the current or last CheckPoint checkPointPutObjectBuffer *bytes.Buffer // if nil, no CheckPoint data to PUT has yet accumulated inodeOpenMap map[uint64]*inodeOpenMapElementStruct // key == inodeNumber inodeLeaseMap map[uint64]*inodeLeaseStruct // key == inodeLeaseStruct.inodeNumber diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 9ab81d6c..2418e782 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -194,6 +194,8 @@ retryGenerateMountID: volume.checkPointControlChan = make(chan chan error) + volume.checkPointObjectNumber = lastCheckPoint.SuperBlockObjectNumber + volume.checkPointControlWG.Add(1) go volume.checkPointDaemon(volume.checkPointControlChan) @@ -471,6 +473,8 @@ func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesReque _ = volume.pendingDeleteObjectNumberList.PushBack(dereferencedObjectNumber) } + volume.dirty = true + globals.Unlock() err = nil diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 948fe387..b298be52 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -614,6 +614,7 @@ func putVolume(name string, storageURL string, authToken string) (err error) { volume = &volumeStruct{ name: name, + dirty: false, storageURL: storageURL, authToken: authToken, mountMap: make(map[string]*mountStruct), @@ -630,7 +631,7 @@ func putVolume(name string, storageURL string, authToken string) (err error) { activeDeleteObjectNumberList: list.New(), pendingDeleteObjectNumberList: list.New(), checkPointControlChan: nil, - checkPointPutObjectNumber: 0, + checkPointObjectNumber: 0, checkPointPutObjectBuffer: nil, inodeOpenMap: make(map[uint64]*inodeOpenMapElementStruct), inodeLeaseMap: make(map[uint64]*inodeLeaseStruct), @@ -692,17 +693,19 @@ func (volume *volumeStruct) checkPointDaemon(checkPointControlChan chan chan err func (volume *volumeStruct) doCheckPoint() (err error) { var ( - authOK bool - checkPointV1String string - deleteObjectNumberListElement *list.Element - inodeTableLayoutElement *inodeTableLayoutElementStruct - inodeTableRootObjectLength uint64 - inodeTableRootObjectNumber uint64 - inodeTableRootObjectOffset uint64 - objectNumber uint64 - ok bool - startTime time.Time = time.Now() - superBlockV1Buf []byte + authOK bool + checkPointV1String string + deleteObjectNumberListElement *list.Element + inodeTableLayoutElement *inodeTableLayoutElementStruct + inodeTableRootObjectLength uint64 + inodeTableRootObjectNumber uint64 + inodeTableRootObjectOffset uint64 + lastCheckPointObjectNumber uint64 + lastCheckPointObjectNumberSeen bool + objectNumber uint64 + ok bool + startTime time.Time = time.Now() + superBlockV1Buf []byte ) defer func() { @@ -711,6 +714,12 @@ func (volume *volumeStruct) doCheckPoint() (err error) { globals.Lock() + if !volume.dirty { + globals.Unlock() + err = nil + return + } + for volume.numNoncesReserved == 0 { volume.nextNonce, volume.numNoncesReserved, err = volume.fetchNonceRangeWhileLocked() if nil != err { @@ -720,7 +729,9 @@ func (volume *volumeStruct) doCheckPoint() (err error) { } } - volume.checkPointPutObjectNumber = volume.nextNonce + lastCheckPointObjectNumber = volume.checkPointObjectNumber + volume.checkPointObjectNumber = volume.nextNonce + volume.checkPointPutObjectBuffer = &bytes.Buffer{} volume.nextNonce++ volume.numNoncesReserved-- @@ -732,16 +743,6 @@ func (volume *volumeStruct) doCheckPoint() (err error) { return } - if nil == volume.checkPointPutObjectBuffer { - // No need to create a new CheckPoint since nothing changed since the last one - volume.checkPointPutObjectNumber = 0 - volume.nextNonce-- - volume.numNoncesReserved++ - globals.Unlock() - err = nil - return - } - err = volume.inodeTable.Prune() if nil != err { globals.Unlock() @@ -765,6 +766,8 @@ func (volume *volumeStruct) doCheckPoint() (err error) { volume.superBlock.PendingDeleteObjectNumberArray = make([]uint64, 0, volume.activeDeleteObjectNumberList.Len()+volume.pendingDeleteObjectNumberList.Len()) + lastCheckPointObjectNumberSeen = false + deleteObjectNumberListElement = volume.activeDeleteObjectNumberList.Front() for nil != deleteObjectNumberListElement { @@ -774,6 +777,9 @@ func (volume *volumeStruct) doCheckPoint() (err error) { logFatal(err) } volume.superBlock.PendingDeleteObjectNumberArray = append(volume.superBlock.PendingDeleteObjectNumberArray, objectNumber) + if objectNumber == lastCheckPointObjectNumber { + lastCheckPointObjectNumberSeen = true + } deleteObjectNumberListElement = deleteObjectNumberListElement.Next() } @@ -786,9 +792,22 @@ func (volume *volumeStruct) doCheckPoint() (err error) { logFatal(err) } volume.superBlock.PendingDeleteObjectNumberArray = append(volume.superBlock.PendingDeleteObjectNumberArray, objectNumber) + if objectNumber == lastCheckPointObjectNumber { + lastCheckPointObjectNumberSeen = true + } deleteObjectNumberListElement = deleteObjectNumberListElement.Next() } + if !lastCheckPointObjectNumberSeen { + _, ok = volume.inodeTableLayout[lastCheckPointObjectNumber] + if !ok { + // After the current CheckPoint is posted, we can finally delete the last one's now comp[letely unreferenced Object + + _ = volume.pendingDeleteObjectNumberList.PushBack(lastCheckPointObjectNumber) + volume.superBlock.PendingDeleteObjectNumberArray = append(volume.superBlock.PendingDeleteObjectNumberArray, objectNumber) + } + } + superBlockV1Buf, err = volume.superBlock.MarshalSuperBlockV1() if nil != err { err = fmt.Errorf("deleteObjectNumberListElement.Value.(uint64) failed: %v", err) @@ -797,21 +816,20 @@ func (volume *volumeStruct) doCheckPoint() (err error) { _, _ = volume.checkPointPutObjectBuffer.Write(superBlockV1Buf) - volume.checkPoint.SuperBlockObjectNumber = volume.checkPointPutObjectNumber + volume.checkPoint.SuperBlockObjectNumber = volume.checkPointObjectNumber volume.checkPoint.SuperBlockLength = uint64(len(superBlockV1Buf)) - authOK, err = volume.swiftObjectPut(volume.checkPointPutObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) + authOK, err = volume.swiftObjectPut(volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) if nil != err { - err = fmt.Errorf("volume.swiftObjectPut(volume.checkPointPutObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) failed: %v", err) + err = fmt.Errorf("volume.swiftObjectPut(volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) failed: %v", err) logFatal(err) } if !authOK { globals.Unlock() - err = fmt.Errorf("volume.swiftObjectPut(volume.checkPointPutObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) returned !authOK") + err = fmt.Errorf("volume.swiftObjectPut(volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) returned !authOK") return } - volume.checkPointPutObjectNumber = 0 volume.checkPointPutObjectBuffer = nil checkPointV1String, err = volume.checkPoint.MarshalCheckPointV1() @@ -844,6 +862,8 @@ func (volume *volumeStruct) doCheckPoint() (err error) { go volume.doObjectDelete(deleteObjectNumberListElement) } + volume.dirty = false + globals.Unlock() err = nil @@ -858,11 +878,15 @@ func (volume *volumeStruct) doObjectDelete(activeDeleteObjectNumberListElement * ok bool ) + globals.Lock() + objectNumber, ok = activeDeleteObjectNumberListElement.Value.(uint64) if !ok { logFatalf("activeDeleteObjectNumberListElement.Value.(uint64) returned !ok") } + globals.Unlock() + authOK, err = volume.swiftObjectDelete(objectNumber) if nil != err { logFatalf("volume.swiftObjectDelete(objectNumber: %016X) failed: %v", objectNumber, err) @@ -876,6 +900,8 @@ func (volume *volumeStruct) doObjectDelete(activeDeleteObjectNumberListElement * volume.pendingDeleteObjectNumberList.PushBack(objectNumber) } + volume.dirty = true + globals.Unlock() volume.activeDeleteObjectWG.Done() @@ -940,32 +966,29 @@ func (volume *volumeStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, ok bool ) - if volume.checkPointPutObjectNumber == 0 { - err = fmt.Errorf("(*volumeStruct)PutNode() called with volume.checkPointPutObjectNumber == 0") + if volume.checkPointObjectNumber == 0 { + err = fmt.Errorf("(*volumeStruct)PutNode() called with volume.checkPointObjectNumber == 0") + logFatal(err) + } + if volume.checkPointPutObjectBuffer == nil { + err = fmt.Errorf("(*volumeStruct)PutNode() called with volume.checkPointPutObjectBuffer == nil") logFatal(err) } - if nil == volume.checkPointPutObjectBuffer { - volume.checkPointPutObjectBuffer = &bytes.Buffer{} - + inodeTableLayoutElement, ok = volume.inodeTableLayout[volume.checkPointObjectNumber] + if ok { + inodeTableLayoutElement.objectSize += uint64(len(nodeByteSlice)) + inodeTableLayoutElement.bytesReferenced += uint64(len(nodeByteSlice)) + } else { inodeTableLayoutElement = &inodeTableLayoutElementStruct{ objectSize: uint64(len(nodeByteSlice)), bytesReferenced: uint64(len(nodeByteSlice)), } - volume.inodeTableLayout[volume.checkPointPutObjectNumber] = inodeTableLayoutElement - } else { - inodeTableLayoutElement, ok = volume.inodeTableLayout[volume.checkPointPutObjectNumber] - if !ok { - err = fmt.Errorf("volume.inodeTableLayout[volume.checkPointPutObjectNumber] returned !ok") - return - } - - inodeTableLayoutElement.objectSize += uint64(uint64(len(nodeByteSlice))) - inodeTableLayoutElement.bytesReferenced += uint64(uint64(len(nodeByteSlice))) + volume.inodeTableLayout[volume.checkPointObjectNumber] = inodeTableLayoutElement } - objectNumber = volume.checkPointPutObjectNumber + objectNumber = volume.checkPointObjectNumber objectOffset = uint64(volume.checkPointPutObjectBuffer.Len()) _, _ = volume.checkPointPutObjectBuffer.Write(nodeByteSlice) @@ -994,7 +1017,12 @@ func (volume *volumeStruct) DiscardNode(objectNumber uint64, objectOffset uint64 if inodeTableLayoutElement.bytesReferenced == 0 { delete(volume.inodeTableLayout, objectNumber) - volume.superBlock.PendingDeleteObjectNumberArray = append(volume.superBlock.PendingDeleteObjectNumberArray, objectNumber) + + // We avoid scheduling the Object for deletion here if it contains the last CheckPoint + + if objectNumber != volume.checkPointObjectNumber { + _ = volume.pendingDeleteObjectNumberList.PushBack(objectNumber) + } } err = nil From d7deaf8cd0331c0301edf29fc9faeb4287d1075f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 17:35:01 -0800 Subject: [PATCH 210/258] Crafted the very dangerous idestroy tool to clean out a Swift Container --- Makefile | 1 + idestroy/.gitignore | 1 + idestroy/Makefile | 9 ++ idestroy/dummy_test.go | 11 +++ idestroy/main.go | 186 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 208 insertions(+) create mode 100644 idestroy/.gitignore create mode 100644 idestroy/Makefile create mode 100644 idestroy/dummy_test.go create mode 100644 idestroy/main.go diff --git a/Makefile b/Makefile index 9888920d..031baba7 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ goplugindirs = \ gobindirs = \ icert \ iclient \ + idestroy \ imgr \ iswift diff --git a/idestroy/.gitignore b/idestroy/.gitignore new file mode 100644 index 00000000..24cadda3 --- /dev/null +++ b/idestroy/.gitignore @@ -0,0 +1 @@ +idestroy diff --git a/idestroy/Makefile b/idestroy/Makefile new file mode 100644 index 00000000..a034e51f --- /dev/null +++ b/idestroy/Makefile @@ -0,0 +1,9 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/idestroy + +generatedfiles := \ + idestroy + +include ../GoMakefile diff --git a/idestroy/dummy_test.go b/idestroy/dummy_test.go new file mode 100644 index 00000000..d4299172 --- /dev/null +++ b/idestroy/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/idestroy/main.go b/idestroy/main.go new file mode 100644 index 00000000..6469498c --- /dev/null +++ b/idestroy/main.go @@ -0,0 +1,186 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +// Program idestroy provides a tool to destroy all contents of a ProxyFS container. +// +// The program requires a single argument that is a path to a package config +// formatted configuration to load. Optionally, overrides the the config may +// be passed as additional arguments in the form .=. +// +// As it is expected to be used (e.g.. during development/testing) to clear out +// a container ultimately used by iclient/imgr, idestroy is designed to simply +// leverage the same iclient .conf file. +// +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/iauth" +) + +func main() { + const ( + HTTPUserAgent = "idestroy" + ) + + var ( + AuthPlugInEnvName string + AuthPlugInEnvValue string + AuthPlugInPath string + confMap conf.ConfMap + err error + httpClient http.Client + httpDELETERequest *http.Request + httpDELETEResponse *http.Response + httpGETRequest *http.Request + httpGETResponse *http.Response + httpGETResponseBodyAsByteSlice []byte + httpGETResponseBodyAsString string + httpGetResponseBodyAsStringSlice []string + objectName string + swiftAuthToken string + swiftStorageURL string + ) + + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "no .conf file specified\n") + os.Exit(1) + } + + confMap, err = conf.MakeConfMapFromFile(os.Args[1]) + if nil != err { + fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err) + os.Exit(1) + } + + err = confMap.UpdateFromStrings(os.Args[2:]) + if nil != err { + fmt.Fprintf(os.Stderr, "failed to apply config overrides: %v\n", err) + os.Exit(1) + } + + AuthPlugInPath, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInPath") + if nil != err { + fmt.Printf("confMap.FetchOptionValueString(\"ICLIENT\", \"AuthPlugInPath\") failed: %v\n", err) + os.Exit(1) + } + err = confMap.VerifyOptionIsMissing("ICLIENT", "AuthPlugInEnvName") + if nil == err { + AuthPlugInEnvName = "" + AuthPlugInEnvValue, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInEnvValue") + if nil != err { + fmt.Printf("confMap.FetchOptionValueString(\"ICLIENT\", \"AuthPlugInEnvValue\") failed: %v\n", err) + os.Exit(1) + } + } else { + err = confMap.VerifyOptionValueIsEmpty("ICLIENT", "AuthPlugInEnvName") + if nil == err { + AuthPlugInEnvName = "" + AuthPlugInEnvValue, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInEnvValue") + if nil != err { + fmt.Printf("confMap.FetchOptionValueString(\"ICLIENT\", \"AuthPlugInEnvValue\") failed: %v\n", err) + os.Exit(1) + } + } else { + AuthPlugInEnvName, err = confMap.FetchOptionValueString("ICLIENT", "AuthPlugInEnvName") + if nil != err { + fmt.Printf("confMap.FetchOptionValueString(\"ICLIENT\", \"AuthPlugInEnvName\") failed: %v\n", err) + os.Exit(1) + } + err = confMap.VerifyOptionIsMissing("ICLIENT", "AuthPlugInEnvValue") + if nil == err { + AuthPlugInEnvValue = "" + } else { + err = confMap.VerifyOptionValueIsEmpty("ICLIENT", "AuthPlugInEnvValue") + if nil == err { + AuthPlugInEnvValue = "" + } else { + fmt.Printf("If [ICLIENT]AuthPlugInEnvName is present and non-empty, [ICLIENT]AuthPlugInEnvValue must be missing or empty\n") + os.Exit(1) + } + } + } + } + + if AuthPlugInEnvName != "" { + AuthPlugInEnvValue = os.Getenv(AuthPlugInEnvName) + } + + swiftAuthToken, swiftStorageURL, err = iauth.PerformAuth(AuthPlugInPath, AuthPlugInEnvValue) + if nil != err { + fmt.Printf("iauth.PerformAuth(AuthPlugInPath, AuthPlugInEnvValue) failed: %v\n", err) + os.Exit(1) + } + + httpClient = http.Client{} + +ReFetchObjectNameList: + + httpGETRequest, err = http.NewRequest("GET", swiftStorageURL, nil) + if nil != err { + fmt.Printf("http.NewRequest(\"GET\", swiftStorageURL, nil) failed: %v\n", err) + os.Exit(1) + } + + httpGETRequest.Header.Add("User-Agent", HTTPUserAgent) + httpGETRequest.Header.Add("X-Auth-Token", swiftAuthToken) + + httpGETResponse, err = httpClient.Do(httpGETRequest) + if nil != err { + fmt.Printf("httpClient.Do(httpGETRequest) failed: %v\n", err) + os.Exit(1) + } + if (httpGETResponse.StatusCode < 200) || (httpGETResponse.StatusCode > 299) { + fmt.Printf("httpGETResponse.Status unexpected: %v\n", httpGETResponse.Status) + os.Exit(1) + } + + httpGETResponseBodyAsByteSlice, err = ioutil.ReadAll(httpGETResponse.Body) + if nil != err { + fmt.Printf("ioutil.ReadAll(httpGETResponse.Body) failed: %v\n", err) + os.Exit(1) + } + httpGETResponseBodyAsString = string(httpGETResponseBodyAsByteSlice[:]) + + httpGetResponseBodyAsStringSlice = strings.Fields(httpGETResponseBodyAsString) + + if len(httpGetResponseBodyAsStringSlice) == 0 { + os.Exit(0) + } + + for _, objectName = range httpGetResponseBodyAsStringSlice { + httpDELETERequest, err = http.NewRequest("DELETE", swiftStorageURL+"/"+objectName, nil) + if nil != err { + fmt.Printf("http.NewRequest(\"DELETE\", swiftStorageURL+\"/\"+objectName, nil) failed: %v\n", err) + os.Exit(1) + } + + httpDELETERequest.Header.Add("User-Agent", HTTPUserAgent) + httpDELETERequest.Header.Add("X-Auth-Token", swiftAuthToken) + + httpDELETEResponse, err = httpClient.Do(httpDELETERequest) + if nil != err { + fmt.Printf("httpClient.Do(httpDELETERequest) failed: %v\n", err) + os.Exit(1) + } + + _, err = ioutil.ReadAll(httpDELETEResponse.Body) + if nil != err { + fmt.Printf("ioutil.ReadAll(httpDELETEResponse.Body) failed: %v\n", err) + os.Exit(1) + } + + if (httpDELETEResponse.StatusCode < 200) || (httpDELETEResponse.StatusCode > 299) { + fmt.Printf("httpDELETEResponse.Status unexpected: %v\n", httpDELETEResponse.Status) + os.Exit(1) + } + } + + goto ReFetchObjectNameList +} From 21b047c9ddf4deb4c5619287720b16774c3888f1 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 18:47:37 -0800 Subject: [PATCH 211/258] Fixed errant insertion in PendingDeleteObjectNumberArray ...also resolved the chicken-and-egg problem of a checkpoint only due to pruning PendingDeleteObjectNumberArray --- imgr/imgrpkg/volume.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index b298be52..2d44e1bb 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -804,7 +804,7 @@ func (volume *volumeStruct) doCheckPoint() (err error) { // After the current CheckPoint is posted, we can finally delete the last one's now comp[letely unreferenced Object _ = volume.pendingDeleteObjectNumberList.PushBack(lastCheckPointObjectNumber) - volume.superBlock.PendingDeleteObjectNumberArray = append(volume.superBlock.PendingDeleteObjectNumberArray, objectNumber) + volume.superBlock.PendingDeleteObjectNumberArray = append(volume.superBlock.PendingDeleteObjectNumberArray, lastCheckPointObjectNumber) } } @@ -897,10 +897,24 @@ func (volume *volumeStruct) doObjectDelete(activeDeleteObjectNumberListElement * volume.activeDeleteObjectNumberList.Remove(activeDeleteObjectNumberListElement) if !authOK { - volume.pendingDeleteObjectNumberList.PushBack(objectNumber) + _ = volume.pendingDeleteObjectNumberList.PushBack(objectNumber) } - volume.dirty = true + // The last doObjectDelete() instance to complete, if ever reached, has the + // responsibility of determining if a new batch should be launched. A curious + // reverse-chicked-and-egg situation can arrise if the only reason for a subsequent + // CheckPoint is to update the at the time now empty PendingDeleteObjectNumberArray. + // Of course this would generate a new entry to delete the just previous CheckPoint. + // As such, we will only mark the volume dirty if the PendingDeleteObjectNumberArray + // has at least 2 entries. + + if volume.activeDeleteObjectNumberList.Len() > 0 { + volume.dirty = true + } else { + if volume.pendingDeleteObjectNumberList.Len() >= 2 { + volume.dirty = true + } + } globals.Unlock() From ee066f5b21f4a131588c2ec9592b568f19946948 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 19:15:49 -0800 Subject: [PATCH 212/258] Up'd ParallelObjectDeletePerVolumeLimit from 10 to 100 --- iclient/iclientpkg/utils_test.go | 2 +- imgr/dev.conf | 2 +- imgr/imgr.conf | 2 +- imgr/imgrpkg/api.go | 2 +- imgr/imgrpkg/utils_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/iclient/iclientpkg/utils_test.go b/iclient/iclientpkg/utils_test.go index 5d7c40a7..f0c6309e 100644 --- a/iclient/iclientpkg/utils_test.go +++ b/iclient/iclientpkg/utils_test.go @@ -205,7 +205,7 @@ func testSetup(t *testing.T) { "IMGR.SwiftTimeout=10m", "IMGR.SwiftConnectionPoolSize=128", - "IMGR.ParallelObjectDeletePerVolumeLimit=10", + "IMGR.ParallelObjectDeletePerVolumeLimit=100", "IMGR.InodeTableCacheEvictLowLimit=10000", "IMGR.InodeTableCacheEvictHighLimit=10010", diff --git a/imgr/dev.conf b/imgr/dev.conf index d4714834..735ae409 100644 --- a/imgr/dev.conf +++ b/imgr/dev.conf @@ -34,7 +34,7 @@ SwiftRetryLimit: 4 SwiftTimeout: 10m SwiftConnectionPoolSize: 128 -ParallelObjectDeletePerVolumeLimit: 10 +ParallelObjectDeletePerVolumeLimit: 100 InodeTableCacheEvictLowLimit: 10000 InodeTableCacheEvictHighLimit: 10010 diff --git a/imgr/imgr.conf b/imgr/imgr.conf index baac4a7f..b6d87883 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -34,7 +34,7 @@ SwiftRetryLimit: 4 SwiftTimeout: 10m SwiftConnectionPoolSize: 128 -ParallelObjectDeletePerVolumeLimit: 10 +ParallelObjectDeletePerVolumeLimit: 100 InodeTableCacheEvictLowLimit: 10000 InodeTableCacheEvictHighLimit: 10010 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 2cc5e086..ff9071cb 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -47,7 +47,7 @@ // SwiftTimeout: 10m // SwiftConnectionPoolSize: 128 // -// ParallelObjectDeletePerVolumeLimit: 10 +// ParallelObjectDeletePerVolumeLimit: 100 // // InodeTableCacheEvictLowLimit: 10000 // InodeTableCacheEvictHighLimit: 10010 diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index 0e82c516..8a9790dd 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -151,7 +151,7 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { "IMGR.SwiftTimeout=10m", "IMGR.SwiftConnectionPoolSize=128", - "IMGR.ParallelObjectDeletePerVolumeLimit=10", + "IMGR.ParallelObjectDeletePerVolumeLimit=100", "IMGR.InodeTableCacheEvictLowLimit=10000", "IMGR.InodeTableCacheEvictHighLimit=10010", From be191de79114dd3e29e1af0382c9b01dddf015e9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 20:21:17 -0800 Subject: [PATCH 213/258] Fixed cut&paste coding mistake in all but (*volumeStruct).swiftObjectPut() ...these receiver versions of swift ops all should just return authOK & err as is when the corresponding ...Once() func returned err == nil --- imgr/imgrpkg/swift-client.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index 7d5bf567..c42100d4 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -221,9 +221,6 @@ func (volume *volumeStruct) swiftObjectDelete(objectNumber uint64) (authOK bool, for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { authOK, err = volume.swiftObjectDeleteOnce(objectURL) if nil == err { - if !authOK { - err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") - } return } @@ -406,9 +403,6 @@ func (volume *volumeStruct) swiftObjectGet(objectNumber uint64) (buf []byte, aut for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { buf, authOK, err = volume.swiftObjectGetOnce(objectURL, "") if nil == err { - if !authOK { - err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") - } return } @@ -489,9 +483,6 @@ func (volume *volumeStruct) swiftObjectGetRange(objectNumber uint64, objectOffse for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { buf, authOK, err = volume.swiftObjectGetOnce(objectURL, rangeHeaderValue) if nil == err { - if !authOK { - err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") - } return } @@ -572,9 +563,6 @@ func (volume *volumeStruct) swiftObjectGetTail(objectNumber uint64, objectLength for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { buf, authOK, err = volume.swiftObjectGetOnce(objectURL, rangeHeaderValue) if nil == err { - if !authOK { - err = fmt.Errorf("httpResponse.Status: http.StatusUnauthorized") - } return } From e896e5278627f4c7b02c80387712dffab36ea93f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 20:29:25 -0800 Subject: [PATCH 214/258] Attempt to get past old/stale/unpinned build dependencies (i.e. go mod tidy) --- go.mod | 36 +------ go.sum | 299 --------------------------------------------------------- 2 files changed, 1 insertion(+), 334 deletions(-) diff --git a/go.mod b/go.mod index a13d762f..75f2010a 100644 --- a/go.mod +++ b/go.mod @@ -25,79 +25,45 @@ require ( require ( github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cilium/ebpf v0.7.0 // indirect github.com/coreos/bbolt v1.3.6 // indirect github.com/coreos/etcd v3.3.25+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect - github.com/cosiner/argv v0.1.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/derekparker/trie v0.0.0-20200317170641-1fdf38b7b0e9 // indirect - github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/go-delve/delve v1.7.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-dap v0.6.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kr/pretty v0.2.1 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/ory/viper v1.7.5 // indirect - github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/peterh/liner v1.2.1 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/common v0.30.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/cobra v1.2.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.2.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.starlark.net v0.0.0-20211013185944-b0039bd2cfe3 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.19.0 // indirect - golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f // indirect google.golang.org/grpc v1.40.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/go.sum b/go.sum index d79bd5c8..34c5e700 100644 --- a/go.sum +++ b/go.sum @@ -14,11 +14,6 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -27,7 +22,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -43,15 +37,10 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= -github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066 h1:K/8Os/A5UFB6mJiQaeGJtXjvb4sOLm6xj7ulQxqUACw= -github.com/NVIDIA/fission v0.0.0-20211019224312-9bcd4fabb066/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= -github.com/NVIDIA/fission v0.0.0-20211105235624-b9d1d59bb6c2 h1:aeeSFyX1qAyT7AgMJcAqj2f3mpoKJC6jvhqkl14qtr4= -github.com/NVIDIA/fission v0.0.0-20211105235624-b9d1d59bb6c2/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b h1:iJ0ZiDsn950iZV10cAAxltZxmJleVa5Cfza3BS4WJ24= github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -65,79 +54,41 @@ github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZt github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg= -github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/derekparker/trie v0.0.0-20200317170641-1fdf38b7b0e9 h1:G765iDCq7bP5opdrPkXk+4V3yfkgV9iGFuheWZ/X/zY= -github.com/derekparker/trie v0.0.0-20200317170641-1fdf38b7b0e9/go.mod h1:D6ICZm05D9VN1n/8iOtBxLpXtoGp6HDFUJ1RNVieOSE= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-delve/delve v1.7.3 h1:5I8KjqwKIz6I7JQ4QA+eY5PRB0+INs9h7wEDRzFnuHQ= -github.com/go-delve/delve v1.7.3/go.mod h1:mBVf2XSFxRX8Y8AuGPhANnsxuZAnTFRM4l8KeaO5pm8= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -150,15 +101,12 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -171,7 +119,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -187,7 +134,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -201,17 +147,12 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-dap v0.6.0 h1:Y1RHGUtv3R8y6sXq2dtGRMYrFB2hSqyFVws7jucrzX4= -github.com/google/go-dap v0.6.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -219,57 +160,22 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -279,57 +185,25 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -338,33 +212,14 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= -github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= -github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= -github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/peterh/liner v1.2.1 h1:O4BlKaq/LWu6VRWmol4ByWfzx6MfXc5Op5HETyIy5yg= -github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= @@ -374,111 +229,56 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.starlark.net v0.0.0-20200821142938-949cc6f4b097/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= -go.starlark.net v0.0.0-20211013185944-b0039bd2cfe3 h1:oBcONsksxvpeodDrLjiMDaKHXKAVVfAydhe/792CE/o= -go.starlark.net v0.0.0-20211013185944-b0039bd2cfe3/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -490,20 +290,13 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -528,7 +321,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -539,22 +331,16 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -575,17 +361,10 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -593,13 +372,6 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -611,10 +383,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -626,7 +395,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -634,7 +402,6 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -645,45 +412,20 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 h1:KzbpndAYEM+4oHRp9JmB2ewj0NHHxO3Z0g7Gus2O1kk= -golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211020174200-9d6173849985 h1:LOlKVhfDyahgmqa97awczplwkjzNaELFg3zRIJ13RYo= -golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211022215931-8e5104632af7 h1:e2q1CMOFXDvurT2sa2yhJAkuA2n8Rd9tMDd7Tcfvs6M= -golang.org/x/sys v0.0.0-20211022215931-8e5104632af7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42 h1:G2DDmludOQZoWbpCr7OKDxnl478ZBGMcOhrv+ooX/Q4= -golang.org/x/sys v0.0.0-20211107104306-e0b2ad06fe42/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e h1:zeJt6jBtVDK23XK9QXcmG0FvO0elikp0dYZQZOeL1y0= -golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -692,7 +434,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= @@ -702,13 +443,11 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -719,7 +458,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -747,15 +485,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736 h1:cw6nUxdoEN5iEIWYD8aAsTZ8iYjLVNiHAb7xz/80WO4= golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= @@ -781,19 +511,11 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -824,18 +546,7 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f h1:enWPderunHptc5pzJkSYGx0olpF8goXzG0rY3kL0eSg= google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -860,15 +571,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= -gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -889,7 +591,6 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= From e9d0bde671d5365a350865ba7db3d301b50fea0d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 21:03:48 -0800 Subject: [PATCH 215/258] Upgraded go Golang 1.17.5 & Alpine:3.15.0 --- Dockerfile | 4 ++-- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ccef303e..4f8c39d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,11 +59,11 @@ # 3) only useful for --target dev # --env DISPLAY: tells Docker to set ENV DISPLAY for X apps (e.g. wireshark) -FROM alpine:3.14.0 as base +FROM alpine:3.15.0 as base RUN apk add --no-cache libc6-compat FROM base as dev -ARG GolangVersion=1.17.2 +ARG GolangVersion=1.17.5 RUN apk add --no-cache bind-tools \ curl \ fuse \ diff --git a/go.mod b/go.mod index 75f2010a..9cab15a1 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 - github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b + github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 github.com/ansel1/merry v1.6.1 github.com/creachadair/cityhash v0.1.0 diff --git a/go.sum b/go.sum index 34c5e700..07302b8c 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1Dnxq github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b h1:iJ0ZiDsn950iZV10cAAxltZxmJleVa5Cfza3BS4WJ24= github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= +github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b h1:Jhps18HvZUrJKJR2E57rF5Kp/7chK6NDo/wZnD0gtF0= +github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b/go.mod h1:9wVslsyxZaBvW/ollg7JLxJOxKb+Ik2KH1WVs1nicMA= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= From cafbc761c3de2857099e6d8a75e1ba100f0a83c3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 21:57:07 -0800 Subject: [PATCH 216/258] Hacks to avoid go module breakage when running make cover ...until this is resolved, make ci will *drop* the make cover step... --- Makefile | 8 +- go.mod | 44 ++++---- go.sum | 335 ++++++------------------------------------------------- 3 files changed, 58 insertions(+), 329 deletions(-) diff --git a/Makefile b/Makefile index 031baba7..d8f746d7 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,13 @@ generatedfiles := \ all: version fmt pre-generate generate build test -ci: version fmt pre-generate generate build test cover +# Due to etcd breaking go modules when running make cover, +# we will, for the time being, disable the make cover step +# in make ci :-( +# +# ci: version fmt pre-generate generate build test cover + +ci: all minimal: version pre-generate generate build diff --git a/go.mod b/go.mod index 9cab15a1..c22d15f1 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,9 @@ replace github.com/coreos/bbolt => go.etcd.io/bbolt v1.3.6 replace go.etcd.io/bbolt v1.3.6 => github.com/coreos/bbolt v1.3.6 -replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 +replace github.com/coreos/etcd => github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible + +replace google.golang.org/grpc => google.golang.org/grpc v1.27.0 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 @@ -14,57 +16,49 @@ require ( github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 github.com/ansel1/merry v1.6.1 - github.com/creachadair/cityhash v0.1.0 + github.com/creachadair/cityhash v0.1.1 github.com/google/btree v1.0.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 - go.etcd.io/etcd v3.3.25+incompatible - golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f - golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 + go.etcd.io/etcd v3.3.27+incompatible + golang.org/x/net v0.0.0-20211216030914-fe4d6282115f + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e ) require ( github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/coreos/bbolt v1.3.6 // indirect - github.com/coreos/etcd v3.3.25+incompatible // indirect + github.com/coreos/bbolt v0.0.0-00010101000000-000000000000 // indirect + github.com/coreos/etcd v0.0.0-00010101000000-000000000000 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect - github.com/dustin/go-humanize v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.4.3 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect - github.com/kr/pretty v0.2.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect - github.com/prometheus/common v0.30.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.7.0 // indirect - go.uber.org/zap v1.19.0 // indirect - golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.1 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f // indirect - google.golang.org/grpc v1.40.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 // indirect + google.golang.org/grpc v1.36.0 // indirect + google.golang.org/protobuf v1.26.0-rc.1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - sigs.k8s.io/yaml v1.2.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 07302b8c..32166f89 100644 --- a/go.sum +++ b/go.sum @@ -1,44 +1,9 @@ bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= -github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b h1:iJ0ZiDsn950iZV10cAAxltZxmJleVa5Cfza3BS4WJ24= -github.com/NVIDIA/fission v0.0.0-20211107202641-3e344bcb438b/go.mod h1:Cf+nuyjoIDp/RE+QDwIDWKpOzl3S81S5N3ajaewMe+U= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b h1:Jhps18HvZUrJKJR2E57rF5Kp/7chK6NDo/wZnD0gtF0= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b/go.mod h1:9wVslsyxZaBvW/ollg7JLxJOxKb+Ik2KH1WVs1nicMA= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= @@ -63,29 +28,21 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY= -github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= -github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= +github.com/creachadair/cityhash v0.1.1 h1:lFsixblDY79QBwzGNQWRvMu4mOBIuV+g3sz/3Vy3pk0= +github.com/creachadair/cityhash v0.1.1/go.mod h1:QhWJSDPs/2AkcOUL8Ogz0ZzA5acyIihEzKBG/9NTUHs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -93,9 +50,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -106,27 +60,15 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -134,39 +76,21 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= @@ -175,9 +99,6 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -185,8 +106,6 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= @@ -196,9 +115,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -215,6 +133,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible h1:CAG0PUvo1fen+ZEfxKJjFIc8GuuN5RuaBuCAuaP2Hno= +github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible/go.mod h1:iIubILNIN6Jq9h8uiSLrN9L1tuj3iSSFwz3R61skm/A= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -233,18 +153,15 @@ github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= -github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -266,157 +183,73 @@ github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDH github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= -go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0= +go.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= -go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -425,134 +258,41 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736 h1:cw6nUxdoEN5iEIWYD8aAsTZ8iYjLVNiHAb7xz/80WO4= -golang.org/x/tools v0.1.8-0.20211028023602-8de2a7fd1736/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 h1:bXUwz2WkXXrXgiLxww3vWmoSHLOGv4ipdPdTvKymcKw= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f h1:enWPderunHptc5pzJkSYGx0olpF8goXzG0rY3kL0eSg= -google.golang.org/genproto v0.0.0-20210818220304-27ea9cc85d9f/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -563,16 +303,13 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -586,14 +323,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 401bd4dbe7b2efe809f5612530653a4659f6c035 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 22 Dec 2021 23:39:31 -0800 Subject: [PATCH 217/258] Added stack dump in logFatal{|f}() calls of both iclientpkg & imgrpkg --- iclient/iclientpkg/log.go | 21 +++++++++++++++++++++ imgr/imgrpkg/log.go | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/iclient/iclientpkg/log.go b/iclient/iclientpkg/log.go index 36dd17eb..96f85b24 100644 --- a/iclient/iclientpkg/log.go +++ b/iclient/iclientpkg/log.go @@ -7,16 +7,19 @@ import ( "fmt" "log" "os" + "runtime" "time" ) func logFatal(err error) { logf("FATAL", "%v", err) + logStack() os.Exit(1) } func logFatalf(format string, args ...interface{}) { logf("FATAL", format, args...) + logStack() os.Exit(1) } @@ -56,6 +59,24 @@ func logTracef(format string, args ...interface{}) { } } +func logStack() { + const ( + runtimeStackBufSize = 65536 + ) + + var ( + runtimeStackBuf []byte + runtimeStackBufUsed int + runtimeStackString string + ) + + runtimeStackBuf = make([]byte, runtimeStackBufSize) + runtimeStackBufUsed = runtime.Stack(runtimeStackBuf, true) + runtimeStackString = string(runtimeStackBuf[:runtimeStackBufUsed]) + + logf("STACK", "\n%s", runtimeStackString) +} + func logf(level string, format string, args ...interface{}) { var ( enhancedArgs []interface{} diff --git a/imgr/imgrpkg/log.go b/imgr/imgrpkg/log.go index c5ed5169..50a1d0ff 100644 --- a/imgr/imgrpkg/log.go +++ b/imgr/imgrpkg/log.go @@ -6,16 +6,19 @@ package imgrpkg import ( "fmt" "os" + "runtime" "time" ) func logFatal(err error) { logf("FATAL", "%v", err) + logStack() os.Exit(1) } func logFatalf(format string, args ...interface{}) { logf("FATAL", format, args...) + logStack() os.Exit(1) } @@ -55,6 +58,24 @@ func logTracef(format string, args ...interface{}) { } } +func logStack() { + const ( + runtimeStackBufSize = 65536 + ) + + var ( + runtimeStackBuf []byte + runtimeStackBufUsed int + runtimeStackString string + ) + + runtimeStackBuf = make([]byte, runtimeStackBufSize) + runtimeStackBufUsed = runtime.Stack(runtimeStackBuf, true) + runtimeStackString = string(runtimeStackBuf[:runtimeStackBufUsed]) + + logf("STACK", "\n%s", runtimeStackString) +} + func logf(level string, format string, args ...interface{}) { var ( enhancedArgs []interface{} From d2297b08de99d6c0b6c8382757ea2aac18149f65 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 23 Dec 2021 00:03:23 -0800 Subject: [PATCH 218/258] Redirected LogToConsole output to go to StdOut rather than StdErr --- iclient/iclientpkg/log.go | 10 +++++----- imgr/imgrpkg/log.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/iclient/iclientpkg/log.go b/iclient/iclientpkg/log.go index 96f85b24..593c7ccb 100644 --- a/iclient/iclientpkg/log.go +++ b/iclient/iclientpkg/log.go @@ -90,10 +90,10 @@ func logf(level string, format string, args ...interface{}) { logMsg = fmt.Sprintf(enhancedFormat, enhancedArgs[:]...) - if nil == globals.logFile { - if "" != globals.config.LogFilePath { + if globals.logFile == nil { + if globals.config.LogFilePath != "" { globals.logFile, err = os.OpenFile(globals.config.LogFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666) - if nil == err { + if err == nil { _, _ = globals.logFile.WriteString(logMsg + "\n") } else { globals.logFile = nil @@ -103,12 +103,12 @@ func logf(level string, format string, args ...interface{}) { globals.logFile.WriteString(logMsg + "\n") } if globals.config.LogToConsole { - fmt.Fprintln(os.Stderr, logMsg) + fmt.Println(logMsg) } } func logSIGHUP() { - if nil != globals.logFile { + if globals.logFile != nil { _ = globals.logFile.Close() globals.logFile = nil } diff --git a/imgr/imgrpkg/log.go b/imgr/imgrpkg/log.go index 50a1d0ff..32ad002b 100644 --- a/imgr/imgrpkg/log.go +++ b/imgr/imgrpkg/log.go @@ -89,10 +89,10 @@ func logf(level string, format string, args ...interface{}) { logMsg = fmt.Sprintf(enhancedFormat, enhancedArgs[:]...) - if nil == globals.logFile { - if "" != globals.config.LogFilePath { + if globals.logFile == nil { + if globals.config.LogFilePath != "" { globals.logFile, err = os.OpenFile(globals.config.LogFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666) - if nil == err { + if err == nil { _, _ = globals.logFile.WriteString(logMsg + "\n") } else { globals.logFile = nil @@ -102,12 +102,12 @@ func logf(level string, format string, args ...interface{}) { globals.logFile.WriteString(logMsg + "\n") } if globals.config.LogToConsole { - fmt.Fprintln(os.Stderr, logMsg) + fmt.Println(logMsg) } } func logSIGHUP() { - if nil != globals.logFile { + if globals.logFile != nil { _ = globals.logFile.Close() globals.logFile = nil } From 871b5b86b222c4a15c8ecb3a556bbad43b5fcb4b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 23 Dec 2021 01:31:35 -0800 Subject: [PATCH 219/258] Fixed bad use of *list.List.PushBackList() --- imgr/imgrpkg/swift-client.go | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index c42100d4..3b6912b4 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -129,7 +129,7 @@ func (volume *volumeStruct) swiftObjectDeleteOnce(objectURL string) (authOK bool authOK, err = swiftObjectDeleteOnce(objectURL, mount.authToken) if nil == err { if authOK { - volume.healthyMountList.PushBackList(toRetryMountList) + volume.appendToHealthyMountList(toRetryMountList) mount.listElement = volume.healthyMountList.PushBack(mount) return } else { @@ -147,7 +147,7 @@ func (volume *volumeStruct) swiftObjectDeleteOnce(objectURL string) (authOK bool authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { - volume.healthyMountList.PushBackList(toRetryMountList) + volume.appendToHealthyMountList(toRetryMountList) if volume.authToken != "" { authOK, err = swiftObjectDeleteOnce(objectURL, volume.authToken) @@ -311,7 +311,7 @@ func (volume *volumeStruct) swiftObjectGetOnce(objectURL string, rangeHeaderValu buf, authOK, err = swiftObjectGetOnce(objectURL, mount.authToken, rangeHeaderValue) if nil == err { if authOK { - volume.healthyMountList.PushBackList(toRetryMountList) + volume.appendToHealthyMountList(toRetryMountList) mount.listElement = volume.healthyMountList.PushBack(mount) return } else { @@ -329,7 +329,7 @@ func (volume *volumeStruct) swiftObjectGetOnce(objectURL string, rangeHeaderValu authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { - volume.healthyMountList.PushBackList(toRetryMountList) + volume.appendToHealthyMountList(toRetryMountList) if volume.authToken != "" { buf, authOK, err = swiftObjectGetOnce(objectURL, volume.authToken, rangeHeaderValue) @@ -652,7 +652,7 @@ func (volume *volumeStruct) swiftObjectPutOnce(objectURL string, body io.ReadSee authOK, err = swiftObjectPutOnce(objectURL, mount.authToken, body) if nil == err { if authOK { - volume.healthyMountList.PushBackList(toRetryMountList) + volume.appendToHealthyMountList(toRetryMountList) mount.listElement = volume.healthyMountList.PushBack(mount) return } else { @@ -670,7 +670,7 @@ func (volume *volumeStruct) swiftObjectPutOnce(objectURL string, body io.ReadSee authOK = false // Auth failed, err = nil // but we will still indicate the func succeeded } else { - volume.healthyMountList.PushBackList(toRetryMountList) + volume.appendToHealthyMountList(toRetryMountList) if volume.authToken != "" { authOK, err = swiftObjectPutOnce(objectURL, volume.authToken, body) @@ -764,3 +764,27 @@ func (volume *volumeStruct) swiftObjectPut(objectNumber uint64, body io.ReadSeek return } + +func (volume *volumeStruct) appendToHealthyMountList(toRetryMountList *list.List) { + var ( + mount *mountStruct + mountListElement *list.Element + ok bool + ) + + for { + mountListElement = toRetryMountList.Front() + if mountListElement == nil { + return + } + + _ = toRetryMountList.Remove(mountListElement) + + mount, ok = mountListElement.Value.(*mountStruct) + if !ok { + logFatalf("mountListElement.Value.(*mountStruct) returned !ok") + } + + mount.listElement = volume.healthyMountList.PushBack(mount) + } +} From e318aadaef7df39d137118af432d45d98ba109db Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 23 Dec 2021 03:06:20 -0800 Subject: [PATCH 220/258] More properly cleaned up CheckPoint activity during imgrpkg.Stop() Outstanding problem is that imgrpkg/swift-client.go currently assumes it is called between globals.Lock() and globals.Unlock() as it is certainly "walking" the volume.healthyMountList in each of the receiver versions of swiftObject{Delete|Get|Put}Once() func's. --- iclient/iclientpkg/rpc.go | 7 ++-- imgr/imgrpkg/retry-rpc.go | 2 +- imgr/imgrpkg/swift-client.go | 17 +++++----- imgr/imgrpkg/volume.go | 63 ++++++++++++++++++++++++++++++++++-- iswift/iswiftpkg/impl.go | 2 +- 5 files changed, 76 insertions(+), 15 deletions(-) diff --git a/iclient/iclientpkg/rpc.go b/iclient/iclientpkg/rpc.go index fd330f87..e99e9727 100644 --- a/iclient/iclientpkg/rpc.go +++ b/iclient/iclientpkg/rpc.go @@ -42,11 +42,10 @@ func startRPCHandler() (err error) { return } - customTransport = &http.Transport{ // Up-to-date as of Golang 1.11 + customTransport = &http.Transport{ // Up-to-date as of Golang 1.17 Proxy: defaultTransport.Proxy, DialContext: defaultTransport.DialContext, - Dial: defaultTransport.Dial, - DialTLS: defaultTransport.DialTLS, + DialTLSContext: defaultTransport.DialTLSContext, TLSClientConfig: defaultTransport.TLSClientConfig, TLSHandshakeTimeout: globals.config.SwiftTimeout, DisableKeepAlives: false, @@ -60,6 +59,8 @@ func startRPCHandler() (err error) { TLSNextProto: defaultTransport.TLSNextProto, ProxyConnectHeader: defaultTransport.ProxyConnectHeader, MaxResponseHeaderBytes: defaultTransport.MaxResponseHeaderBytes, + WriteBufferSize: 0, + ReadBufferSize: 0, } globals.httpClient = &http.Client{ diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 2418e782..f996b32a 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -198,7 +198,7 @@ retryGenerateMountID: volume.checkPointControlWG.Add(1) - go volume.checkPointDaemon(volume.checkPointControlChan) + go volume.checkPointDaemon() } globals.Unlock() diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index 3b6912b4..f0347b04 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -23,15 +23,14 @@ func startSwiftClient() (err error) { defaultTransport, ok = http.DefaultTransport.(*http.Transport) if !ok { - err = fmt.Errorf("http.DefaultTransport.(*http.Transport) returned !ok\n") + err = fmt.Errorf("http.DefaultTransport.(*http.Transport) returned !ok") return } - customTransport = &http.Transport{ // Up-to-date as of Golang 1.11 + customTransport = &http.Transport{ // Up-to-date as of Golang 1.17 Proxy: defaultTransport.Proxy, DialContext: defaultTransport.DialContext, - Dial: defaultTransport.Dial, - DialTLS: defaultTransport.DialTLS, + DialTLSContext: defaultTransport.DialTLSContext, TLSClientConfig: defaultTransport.TLSClientConfig, TLSHandshakeTimeout: globals.config.SwiftTimeout, DisableKeepAlives: false, @@ -45,6 +44,8 @@ func startSwiftClient() (err error) { TLSNextProto: defaultTransport.TLSNextProto, ProxyConnectHeader: defaultTransport.ProxyConnectHeader, MaxResponseHeaderBytes: defaultTransport.MaxResponseHeaderBytes, + WriteBufferSize: 0, + ReadBufferSize: 0, } globals.httpClient = &http.Client{ @@ -72,24 +73,24 @@ func swiftObjectDeleteOnce(objectURL string, authToken string) (authOK bool, err return } - if "" != authToken { + if authToken != "" { httpRequest.Header["X-Auth-Token"] = []string{authToken} } httpResponse, err = globals.httpClient.Do(httpRequest) if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", objectURL, err) + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v", objectURL, err) return } _, err = ioutil.ReadAll(httpResponse.Body) if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) return } err = httpResponse.Body.Close() if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) return } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 2d44e1bb..424d784a 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -8,7 +8,9 @@ import ( "container/list" "fmt" "io" + "runtime" "strings" + "sync" "time" "github.com/NVIDIA/sortedmap" @@ -27,6 +29,61 @@ func startVolumeManagement() (err error) { } func stopVolumeManagement() (err error) { + var ( + checkPointControlWG *sync.WaitGroup + checkPointControlWGSlice []*sync.WaitGroup + ok bool + volume *volumeStruct + volumeAsValue sortedmap.Value + volumeMapIndex int + volumeMapLen int + ) + + // TODO: For now, let's just stop all of the CheckPoint activity + + globals.Lock() + + volumeMapLen, err = globals.volumeMap.Len() + if err != nil { + globals.Unlock() + return + } + + checkPointControlWGSlice = make([]*sync.WaitGroup, 0, volumeMapLen) + + for volumeMapIndex = 0; volumeMapIndex < volumeMapLen; volumeMapIndex++ { + _, volumeAsValue, ok, err = globals.volumeMap.GetByIndex(volumeMapIndex) + if err != nil { + globals.Unlock() + return + } + if !ok { + err = fmt.Errorf("globals.volumeMap.GetByIndex(volumeMapIndex: %v) (with volumeMapLen: %v) returned !ok", volumeMapIndex, volumeMapLen) + globals.Unlock() + return + } + + volume, ok = volumeAsValue.(*volumeStruct) + if !ok { + err = fmt.Errorf("volumeAsValue.(*volumeStruct) returned !ok") + globals.Unlock() + return + } + + if volume.checkPointControlChan != nil { + close(volume.checkPointControlChan) + checkPointControlWGSlice = append(checkPointControlWGSlice, &volume.checkPointControlWG) + } + } + + globals.Unlock() + + for _, checkPointControlWG = range checkPointControlWGSlice { + checkPointControlWG.Wait() + } + + // TODO: For now, just clear out volume-related globals fields + globals.inodeTableCache = nil globals.inodeLeaseLRU = nil globals.volumeMap = nil @@ -655,7 +712,7 @@ func putVolume(name string, storageURL string, authToken string) (err error) { return } -func (volume *volumeStruct) checkPointDaemon(checkPointControlChan chan chan error) { +func (volume *volumeStruct) checkPointDaemon() { var ( checkPointIntervalTimer *time.Timer checkPointResponseChan chan error @@ -672,7 +729,7 @@ func (volume *volumeStruct) checkPointDaemon(checkPointControlChan chan chan err if nil != err { logWarnf("checkPointIntervalTimer-triggered doCheckPoint() failed: %v", err) } - case checkPointResponseChan, more = <-checkPointControlChan: + case checkPointResponseChan, more = <-volume.checkPointControlChan: if !checkPointIntervalTimer.Stop() { <-checkPointIntervalTimer.C } @@ -685,7 +742,9 @@ func (volume *volumeStruct) checkPointDaemon(checkPointControlChan chan chan err checkPointResponseChan <- err } else { + volume.activeDeleteObjectWG.Wait() volume.checkPointControlWG.Done() + runtime.Goexit() } } } diff --git a/iswift/iswiftpkg/impl.go b/iswift/iswiftpkg/impl.go index 5bf7bc60..401a54a4 100644 --- a/iswift/iswiftpkg/impl.go +++ b/iswift/iswiftpkg/impl.go @@ -247,7 +247,7 @@ func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, reques _ = request.Body.Close() xAuthUser = request.Header.Get("X-Auth-User") xAuthKey = request.Header.Get("X-Auth-Key") - if ("" == xAuthUser) || ("" == xAuthKey) { + if (xAuthUser == "") || (xAuthKey == "") { responseWriter.WriteHeader(http.StatusUnauthorized) return } From 25e960ef2877684164eee0809a0fd3e843f67336 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 23 Dec 2021 03:59:32 -0800 Subject: [PATCH 221/258] Converted imgrpkg/swift-client.go func's to support locked or unlocked mode For now, "FLIP" sections indicate TODO work to actually support the unlocked mode case --- imgr/imgrpkg/http-server.go | 16 +++++----- imgr/imgrpkg/swift-client.go | 35 +++++++++++++-------- imgr/imgrpkg/volume.go | 61 +++++++++++++++++++++--------------- 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index fb0cd673..0fe9aad6 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -410,7 +410,7 @@ func (inodeTableWrapper *httpServerInodeTableWrapperStruct) GetNode(objectNumber authOK bool ) - nodeByteSlice, authOK, err = inodeTableWrapper.volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + nodeByteSlice, authOK, err = inodeTableWrapper.volume.swiftObjectGetRange(true, objectNumber, objectOffset, objectLength) if nil != err { err = fmt.Errorf("volume.swiftObjectGetRange() failed: %v", err) return @@ -488,7 +488,7 @@ func (directoryWrapper *httpServerDirectoryWrapperStruct) GetNode(objectNumber u authOK bool ) - nodeByteSlice, authOK, err = directoryWrapper.volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + nodeByteSlice, authOK, err = directoryWrapper.volume.swiftObjectGetRange(true, objectNumber, objectOffset, objectLength) if nil != err { err = fmt.Errorf("volume.swiftObjectGetRange() failed: %v", err) return @@ -566,7 +566,7 @@ func (extentMapWrapper *httpServerExtentMapWrapperStruct) GetNode(objectNumber u authOK bool ) - nodeByteSlice, authOK, err = extentMapWrapper.volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + nodeByteSlice, authOK, err = extentMapWrapper.volume.swiftObjectGetRange(true, objectNumber, objectOffset, objectLength) if nil != err { err = fmt.Errorf("volume.swiftObjectGetRange() failed: %v", err) return @@ -784,7 +784,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeAsStruct.authToken = requestAuthToken } - checkPointV1, err = volumeAsStruct.fetchCheckPoint() + checkPointV1, err = volumeAsStruct.fetchCheckPointWhileLocked() if nil != err { volumeAsStruct.authToken = volumeAuthToken globals.Unlock() @@ -792,7 +792,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ return } - superBlockV1, err = volumeAsStruct.fetchSuperBlock(checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength) + superBlockV1, err = volumeAsStruct.fetchSuperBlockWhileLocked(checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength) if nil != err { volumeAsStruct.authToken = volumeAuthToken globals.Unlock() @@ -948,7 +948,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ volumeAsStruct.authToken = requestAuthToken } - checkPointV1, err = volumeAsStruct.fetchCheckPoint() + checkPointV1, err = volumeAsStruct.fetchCheckPointWhileLocked() if nil != err { volumeAsStruct.authToken = volumeAuthToken globals.Unlock() @@ -956,7 +956,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ return } - superBlockV1, err = volumeAsStruct.fetchSuperBlock(checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength) + superBlockV1, err = volumeAsStruct.fetchSuperBlockWhileLocked(checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength) if nil != err { volumeAsStruct.authToken = volumeAuthToken globals.Unlock() @@ -996,7 +996,7 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logFatalf("inodeTableEntryValueV1AsValue.(*ilayout.InodeTableEntryValueV1Struct) returned !ok") } - inodeHeadV1, err = volumeAsStruct.fetchInodeHead(inodeTableEntryValueV1.InodeHeadObjectNumber, inodeTableEntryValueV1.InodeHeadLength) + inodeHeadV1, err = volumeAsStruct.fetchInodeHeadWhileLocked(inodeTableEntryValueV1.InodeHeadObjectNumber, inodeTableEntryValueV1.InodeHeadLength) if nil != err { volumeAsStruct.authToken = volumeAuthToken globals.Unlock() diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index f0347b04..ca3fcdb1 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -107,13 +107,16 @@ func swiftObjectDeleteOnce(objectURL string, authToken string) (authOK bool, err return } -func (volume *volumeStruct) swiftObjectDeleteOnce(objectURL string) (authOK bool, err error) { +func (volume *volumeStruct) swiftObjectDeleteOnce(locked bool, objectURL string) (authOK bool, err error) { var ( mount *mountStruct mountListElement *list.Element ok bool toRetryMountList *list.List ) + if !locked { + logFatalf("FLIP: (&volumeStruct).swiftObjectDeleteOnce(locked == false,) not yet supported") + } toRetryMountList = list.New() @@ -203,7 +206,7 @@ func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) return } -func (volume *volumeStruct) swiftObjectDelete(objectNumber uint64) (authOK bool, err error) { +func (volume *volumeStruct) swiftObjectDelete(locked bool, objectNumber uint64) (authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -220,7 +223,7 @@ func (volume *volumeStruct) swiftObjectDelete(objectNumber uint64) (authOK bool, nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - authOK, err = volume.swiftObjectDeleteOnce(objectURL) + authOK, err = volume.swiftObjectDeleteOnce(locked, objectURL) if nil == err { return } @@ -289,13 +292,16 @@ func swiftObjectGetOnce(objectURL string, authToken string, rangeHeaderValue str return } -func (volume *volumeStruct) swiftObjectGetOnce(objectURL string, rangeHeaderValue string) (buf []byte, authOK bool, err error) { +func (volume *volumeStruct) swiftObjectGetOnce(locked bool, objectURL string, rangeHeaderValue string) (buf []byte, authOK bool, err error) { var ( mount *mountStruct mountListElement *list.Element ok bool toRetryMountList *list.List ) + if !locked { + logFatalf("FLIP: (&volumeStruct).swiftObjectGetOnce(locked == false,,) not yet supported") + } toRetryMountList = list.New() @@ -385,7 +391,7 @@ func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (b return } -func (volume *volumeStruct) swiftObjectGet(objectNumber uint64) (buf []byte, authOK bool, err error) { +func (volume *volumeStruct) swiftObjectGet(locked bool, objectNumber uint64) (buf []byte, authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -402,7 +408,7 @@ func (volume *volumeStruct) swiftObjectGet(objectNumber uint64) (buf []byte, aut nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - buf, authOK, err = volume.swiftObjectGetOnce(objectURL, "") + buf, authOK, err = volume.swiftObjectGetOnce(locked, objectURL, "") if nil == err { return } @@ -462,7 +468,7 @@ func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint6 return } -func (volume *volumeStruct) swiftObjectGetRange(objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, authOK bool, err error) { +func (volume *volumeStruct) swiftObjectGetRange(locked bool, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -482,7 +488,7 @@ func (volume *volumeStruct) swiftObjectGetRange(objectNumber uint64, objectOffse nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - buf, authOK, err = volume.swiftObjectGetOnce(objectURL, rangeHeaderValue) + buf, authOK, err = volume.swiftObjectGetOnce(locked, objectURL, rangeHeaderValue) if nil == err { return } @@ -542,7 +548,7 @@ func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64 return } -func (volume *volumeStruct) swiftObjectGetTail(objectNumber uint64, objectLength uint64) (buf []byte, authOK bool, err error) { +func (volume *volumeStruct) swiftObjectGetTail(locked bool, objectNumber uint64, objectLength uint64) (buf []byte, authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -562,7 +568,7 @@ func (volume *volumeStruct) swiftObjectGetTail(objectNumber uint64, objectLength nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - buf, authOK, err = volume.swiftObjectGetOnce(objectURL, rangeHeaderValue) + buf, authOK, err = volume.swiftObjectGetOnce(locked, objectURL, rangeHeaderValue) if nil == err { return } @@ -630,13 +636,16 @@ func swiftObjectPutOnce(objectURL string, authToken string, body io.ReadSeeker) return } -func (volume *volumeStruct) swiftObjectPutOnce(objectURL string, body io.ReadSeeker) (authOK bool, err error) { +func (volume *volumeStruct) swiftObjectPutOnce(locked bool, objectURL string, body io.ReadSeeker) (authOK bool, err error) { var ( mount *mountStruct mountListElement *list.Element ok bool toRetryMountList *list.List ) + if !locked { + logFatalf("FLIP: (&volumeStruct).swiftObjectPutOnce(locked == false,,) not yet supported") + } toRetryMountList = list.New() @@ -726,7 +735,7 @@ func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, bo return } -func (volume *volumeStruct) swiftObjectPut(objectNumber uint64, body io.ReadSeeker) (authOK bool, err error) { +func (volume *volumeStruct) swiftObjectPut(locked bool, objectNumber uint64, body io.ReadSeeker) (authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -743,7 +752,7 @@ func (volume *volumeStruct) swiftObjectPut(objectNumber uint64, body io.ReadSeek nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - authOK, err = volume.swiftObjectPutOnce(objectURL, body) + authOK, err = volume.swiftObjectPutOnce(locked, objectURL, body) if nil == err { if authOK { return diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 424d784a..d3493fce 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -878,14 +878,14 @@ func (volume *volumeStruct) doCheckPoint() (err error) { volume.checkPoint.SuperBlockObjectNumber = volume.checkPointObjectNumber volume.checkPoint.SuperBlockLength = uint64(len(superBlockV1Buf)) - authOK, err = volume.swiftObjectPut(volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) + authOK, err = volume.swiftObjectPut(true, volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) if nil != err { - err = fmt.Errorf("volume.swiftObjectPut(volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) failed: %v", err) + err = fmt.Errorf("volume.swiftObjectPut(locked==true, volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) failed: %v", err) logFatal(err) } if !authOK { globals.Unlock() - err = fmt.Errorf("volume.swiftObjectPut(volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) returned !authOK") + err = fmt.Errorf("volume.swiftObjectPut(locked==true, volume.checkPointObjectNumber, bytes.NewReader(volume.checkPointPutObjectBuffer.Bytes())) returned !authOK") return } @@ -897,14 +897,14 @@ func (volume *volumeStruct) doCheckPoint() (err error) { logFatal(err) } - authOK, err = volume.swiftObjectPut(ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) + authOK, err = volume.swiftObjectPut(true, ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) if nil != err { - err = fmt.Errorf("volume.swiftObjectPut(ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) failed: %v", err) + err = fmt.Errorf("volume.swiftObjectPut(locked==true, ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) failed: %v", err) logFatal(err) } if !authOK { globals.Unlock() - err = fmt.Errorf("volume.swiftObjectPut(ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) returned !authOK") + err = fmt.Errorf("volume.swiftObjectPut(locked==true, ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) returned !authOK") return } @@ -944,14 +944,25 @@ func (volume *volumeStruct) doObjectDelete(activeDeleteObjectNumberListElement * logFatalf("activeDeleteObjectNumberListElement.Value.(uint64) returned !ok") } - globals.Unlock() + // FLIP: Here is the top limit of the swapped code + + // globals.Unlock() + + // authOK, err = volume.swiftObjectDelete(false, objectNumber) + // if nil != err { + // logFatalf("volume.swiftObjectDelete(locked: false, objectNumber: %016X) failed: %v", objectNumber, err) + // } - authOK, err = volume.swiftObjectDelete(objectNumber) + // globals.Lock() + + // FLIP: flip below to the above (i.e. locked: false use) + + authOK, err = volume.swiftObjectDelete(true, objectNumber) if nil != err { - logFatalf("volume.swiftObjectDelete(objectNumber: %016X) failed: %v", objectNumber, err) + logFatalf("volume.swiftObjectDelete(locked: true, objectNumber: %016X) failed: %v", objectNumber, err) } - globals.Lock() + // FLIP: Here is the bottom limit of the swapped code volume.activeDeleteObjectNumberList.Remove(activeDeleteObjectNumberListElement) @@ -1021,12 +1032,12 @@ func (volume *volumeStruct) GetNode(objectNumber uint64, objectOffset uint64, ob authOK bool ) - nodeByteSlice, authOK, err = volume.swiftObjectGetRange(objectNumber, objectOffset, objectLength) + nodeByteSlice, authOK, err = volume.swiftObjectGetRange(true, objectNumber, objectOffset, objectLength) if nil != err { - err = fmt.Errorf("volume.swiftObjectGetRange(objectNumber==0x%016X, objectOffset==0x%016X, objectLength==0x%016X) failed: %v", objectNumber, objectOffset, objectLength, err) + err = fmt.Errorf("volume.swiftObjectGetRange(locked==true, objectNumber==0x%016X, objectOffset==0x%016X, objectLength==0x%016X) failed: %v", objectNumber, objectOffset, objectLength, err) logError(err) } else if !authOK { - err = fmt.Errorf("volume.swiftObjectGetRange(objectNumber==0x%016X, objectOffset==0x%016X, objectLength==0x%016X) returned !authOK", objectNumber, objectOffset, objectLength) + err = fmt.Errorf("volume.swiftObjectGetRange(locked==true, objectNumber==0x%016X, objectOffset==0x%016X, objectLength==0x%016X) returned !authOK", objectNumber, objectOffset, objectLength) logWarn(err) } @@ -1180,13 +1191,13 @@ func (volume *volumeStruct) UnpackValue(payloadData []byte) (value sortedmap.Val return } -func (volume *volumeStruct) fetchCheckPoint() (checkPointV1 *ilayout.CheckPointV1Struct, err error) { +func (volume *volumeStruct) fetchCheckPointWhileLocked() (checkPointV1 *ilayout.CheckPointV1Struct, err error) { var ( authOK bool checkPointV1Buf []byte ) - checkPointV1Buf, authOK, err = volume.swiftObjectGet(ilayout.CheckPointObjectNumber) + checkPointV1Buf, authOK, err = volume.swiftObjectGet(true, ilayout.CheckPointObjectNumber) if nil != err { err = fmt.Errorf("volume.swiftObjectGet() failed: %v", err) return @@ -1201,13 +1212,13 @@ func (volume *volumeStruct) fetchCheckPoint() (checkPointV1 *ilayout.CheckPointV return } -func (volume *volumeStruct) fetchSuperBlock(superBlockObjectNumber uint64, superBlockLength uint64) (superBlockV1 *ilayout.SuperBlockV1Struct, err error) { +func (volume *volumeStruct) fetchSuperBlockWhileLocked(superBlockObjectNumber uint64, superBlockLength uint64) (superBlockV1 *ilayout.SuperBlockV1Struct, err error) { var ( authOK bool superBlockV1Buf []byte ) - superBlockV1Buf, authOK, err = volume.swiftObjectGetTail(superBlockObjectNumber, superBlockLength) + superBlockV1Buf, authOK, err = volume.swiftObjectGetTail(true, superBlockObjectNumber, superBlockLength) if nil != err { err = fmt.Errorf("volume.swiftObjectGetTail() failed: %v", err) return @@ -1222,13 +1233,13 @@ func (volume *volumeStruct) fetchSuperBlock(superBlockObjectNumber uint64, super return } -func (volume *volumeStruct) fetchInodeHead(inodeHeadObjectNumber uint64, inodeHeadLength uint64) (inodeHeadV1 *ilayout.InodeHeadV1Struct, err error) { +func (volume *volumeStruct) fetchInodeHeadWhileLocked(inodeHeadObjectNumber uint64, inodeHeadLength uint64) (inodeHeadV1 *ilayout.InodeHeadV1Struct, err error) { var ( authOK bool inodeHeadV1Buf []byte ) - inodeHeadV1Buf, authOK, err = volume.swiftObjectGetTail(inodeHeadObjectNumber, inodeHeadLength) + inodeHeadV1Buf, authOK, err = volume.swiftObjectGetTail(true, inodeHeadObjectNumber, inodeHeadLength) if nil != err { err = fmt.Errorf("volume.swiftObjectGetTail() failed: %v", err) return @@ -1282,19 +1293,19 @@ func (volume *volumeStruct) fetchNonceRangeWhileLocked() (nextNonce uint64, numN logFatalf("nonceUpdatedCheckPoint.MarshalCheckPointV1() failed: %v", err) } - authOK, err = volume.swiftObjectPut(ilayout.CheckPointObjectNumber, strings.NewReader(nonceUpdatedCheckPointAsString)) + authOK, err = volume.swiftObjectPut(true, ilayout.CheckPointObjectNumber, strings.NewReader(nonceUpdatedCheckPointAsString)) if nil == err { if authOK { volume.checkPoint = nonceUpdatedCheckPoint } else { nextNonce = 0 numNoncesFetched = 0 - err = fmt.Errorf("volume.swiftObjectPut(ilayout.CheckPointObjectNumber, strings.NewReader(nonceUpdatedCheckPointAsString)) returned !authOK") + err = fmt.Errorf("volume.swiftObjectPut(locked==true, ilayout.CheckPointObjectNumber, strings.NewReader(nonceUpdatedCheckPointAsString)) returned !authOK") } } else { nextNonce = 0 numNoncesFetched = 0 - err = fmt.Errorf("volume.swiftObjectPut(ilayout.CheckPointObjectNumber, strings.NewReader(nonceUpdatedCheckPointAsString)) failed: %v", err) + err = fmt.Errorf("volume.swiftObjectPut(locked==true, ilayout.CheckPointObjectNumber, strings.NewReader(nonceUpdatedCheckPointAsString)) failed: %v", err) } return @@ -1326,12 +1337,12 @@ func (volume *volumeStruct) removeInodeWhileLocked(inodeNumber uint64) { logFatalf("inodeTableEntryValueRaw.(ilayout.InodeTableEntryValueV1Struct) returned !ok") } - inodeHeadV1Buf, authOK, err = volume.swiftObjectGetTail(inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadLength) + inodeHeadV1Buf, authOK, err = volume.swiftObjectGetTail(true, inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadLength) if nil != err { - logFatalf("volume.swiftObjectGetTail(inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadObjectNumber) failed: %v", err) + logFatalf("volume.swiftObjectGetTail(locked==true, inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadObjectNumber) failed: %v", err) } if !authOK { - logFatalf("volume.swiftObjectGetTail(inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadObjectNumber) returned !authOK") + logFatalf("volume.swiftObjectGetTail(locked==true, inodeTableEntryValue.InodeHeadObjectNumber, inodeTableEntryValue.InodeHeadObjectNumber) returned !authOK") } inodeHeadV1, err = ilayout.UnmarshalInodeHeadV1(inodeHeadV1Buf) From fcd5a1a637bbd65906e28e2cdfbd1b9c3106a6c9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 29 Dec 2021 13:55:07 -0800 Subject: [PATCH 222/258] Resolved need for holding globals.Lock() while doing Swift ops --- imgr/imgrpkg/globals.go | 16 +- imgr/imgrpkg/retry-rpc.go | 43 ++++- imgr/imgrpkg/swift-client.go | 338 ++++++++++++++++++++--------------- imgr/imgrpkg/volume.go | 19 +- 4 files changed, 243 insertions(+), 173 deletions(-) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 28269215..69ff2794 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -190,8 +190,19 @@ type inodeLeaseStruct struct { interruptTimer *time.Timer // if .C != nil, timing when to issue next Interrupt... or expire a Lease } +// on*MountList indicates which volumeStruct.{healthy|leasesExpired|authTokenExpired}MountList the mountStruct is on +// +// Note that both of mountStruct.{leases|authToken}Expired may be true simultaneously. +// In this case, mountStruct.mountListMembership should be == onAuthTokenExpiredMountList. + +const ( + onHealthyMountList = uint8(iota) // if mountStruct.mountListElement is on volumeStruct.healthyMountList + onLeasesExpiredMountList // if mountStruct.mountListElement is on volumeStruct.leasesExpiredMountList + onAuthTokenExpiredMountList // if mountStruct.mountListElement is on volumeStruct.authTokenExpiredMountList + onNoMountList // if mountStruct.mountListElement is not an any of volumeStruct.{healthy|leasesExpired|authTokenExpired}MountList +) + type mountStruct struct { - // reentrancy covered by volumeStruct's sync.RWMutex volume *volumeStruct // volume.{R|}Lock() also protects each mountStruct mountID string // retryRPCClientID uint64 // @@ -201,7 +212,8 @@ type mountStruct struct { authTokenExpired bool // if true, authToken has been rejected... needing a renewMount() to update authToken string // lastAuthTime time.Time // used to periodically check TTL of authToken - listElement *list.Element // LRU element on either volumeStruct.{healthy|leasesExpired|authTokenExpired}MountList + mountListElement *list.Element // LRU element on either volumeStruct.{healthy|leasesExpired|authTokenExpired}MountList + mountListMembership uint8 // == one of on{No|healthy|leasesExpired|authTokenExpired}MountList inodeOpenMap map[uint64]uint64 // key == inodeNumber; value == open count for this mountStruct for this inodeNumber } diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index f996b32a..65a07cbf 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -157,11 +157,12 @@ retryGenerateMountID: authTokenExpired: false, authToken: mountRequest.AuthToken, lastAuthTime: startTime, + mountListMembership: onHealthyMountList, inodeOpenMap: make(map[uint64]uint64), } volume.mountMap[mountIDAsString] = mount - mount.listElement = volume.healthyMountList.PushBack(mount) + mount.mountListElement = volume.healthyMountList.PushBack(mount) globals.mountMap[mountIDAsString] = mount if nil == volume.checkPointControlChan { @@ -233,18 +234,44 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * _, err = swiftObjectGet(mount.volume.storageURL, mount.authToken, ilayout.CheckPointObjectNumber) if nil == err { - if mount.leasesExpired { - mount.volume.leasesExpiredMountList.MoveToBack(mount.listElement) - } else { - if mount.authTokenExpired { - _ = mount.volume.authTokenExpiredMountList.Remove(mount.listElement) - mount.listElement = mount.volume.healthyMountList.PushBack(mount) + mount.authTokenExpired = false + + switch mount.mountListMembership { + case onHealthyMountList: + mount.volume.healthyMountList.MoveToBack(mount.mountListElement) + case onLeasesExpiredMountList: + mount.volume.leasesExpiredMountList.MoveToBack(mount.mountListElement) + case onAuthTokenExpiredMountList: + _ = mount.volume.authTokenExpiredMountList.Remove(mount.mountListElement) + if mount.leasesExpired { + mount.mountListElement = mount.volume.leasesExpiredMountList.PushBack(mount) + mount.mountListMembership = onLeasesExpiredMountList } else { - mount.volume.healthyMountList.MoveToBack(mount.listElement) + mount.mountListElement = mount.volume.healthyMountList.PushBack(mount) + mount.mountListMembership = onHealthyMountList } + default: + logFatalf("mount.mountListMembership (%v) not one of on{Healthy|LeasesExpired|AuthTokenExpired}MountList") } } else { err = fmt.Errorf("%s %s", EAuthTokenRejected, renewMountRequest.AuthToken) + + mount.authTokenExpired = true + + switch mount.mountListMembership { + case onHealthyMountList: + _ = mount.volume.healthyMountList.Remove(mount.mountListElement) + mount.mountListElement = mount.volume.authTokenExpiredMountList.PushBack(mount) + mount.mountListMembership = onAuthTokenExpiredMountList + case onLeasesExpiredMountList: + _ = mount.volume.leasesExpiredMountList.Remove(mount.mountListElement) + mount.mountListElement = mount.volume.authTokenExpiredMountList.PushBack(mount) + mount.mountListMembership = onAuthTokenExpiredMountList + case onAuthTokenExpiredMountList: + mount.volume.authTokenExpiredMountList.MoveToBack(mount.mountListElement) + default: + logFatalf("mount.mountListMembership (%v) not one of on{Healthy|LeasesExpired|AuthTokenExpired}MountList") + } } globals.Unlock() diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index ca3fcdb1..6eb07bce 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -107,64 +107,86 @@ func swiftObjectDeleteOnce(objectURL string, authToken string) (authOK bool, err return } -func (volume *volumeStruct) swiftObjectDeleteOnce(locked bool, objectURL string) (authOK bool, err error) { +func (volume *volumeStruct) swiftObjectDeleteOnce(alreadyLocked bool, objectURL string) (authOK bool, err error) { var ( - mount *mountStruct - mountListElement *list.Element - ok bool - toRetryMountList *list.List + authToken string + mount *mountStruct + mountListElement *list.Element + ok bool + usingVolumeAuthToken bool ) - if !locked { - logFatalf("FLIP: (&volumeStruct).swiftObjectDeleteOnce(locked == false,) not yet supported") - } - toRetryMountList = list.New() + if !alreadyLocked { + globals.Lock() + } mountListElement = volume.healthyMountList.Front() - for nil != mountListElement { - _ = volume.healthyMountList.Remove(mountListElement) + if mountListElement == nil { + if volume.authToken == "" { + if !alreadyLocked { + globals.Unlock() + } + authOK = false // Auth failed, + err = nil // but we still indicate the func succeeded + return + } + authToken = volume.authToken + usingVolumeAuthToken = true + } else { mount, ok = mountListElement.Value.(*mountStruct) if !ok { logFatalf("mountListElement.Value.(*mountStruct) returned !ok") } - authOK, err = swiftObjectDeleteOnce(objectURL, mount.authToken) - if nil == err { - if authOK { - volume.appendToHealthyMountList(toRetryMountList) - mount.listElement = volume.healthyMountList.PushBack(mount) - return - } else { - mount.authTokenExpired = true - mount.listElement = volume.authTokenExpiredMountList.PushBack(mount) - } - } else { - mount.listElement = toRetryMountList.PushBack(mount) - } + // We know that mount.mountListMembership == onHealthyMountList + + volume.healthyMountList.MoveToBack(mount.mountListElement) - mountListElement = volume.healthyMountList.Front() + authToken = mount.authToken + usingVolumeAuthToken = true } - if (toRetryMountList.Len() == 0) && (volume.authToken == "") { - authOK = false // Auth failed, - err = nil // but we will still indicate the func succeeded - } else { - volume.appendToHealthyMountList(toRetryMountList) + if !alreadyLocked { + globals.Unlock() + } - if volume.authToken != "" { - authOK, err = swiftObjectDeleteOnce(objectURL, volume.authToken) - if (nil == err) && !authOK { - logWarnf("swiftObjectDeleteOnce(,volume.authToken) !authOK for volume %s...clearing volume.authToken", volume.name) + authOK, err = swiftObjectDeleteOnce(objectURL, authToken) + if err == nil { + if !authOK { + if !alreadyLocked { + globals.Lock() + } + + if usingVolumeAuthToken { + logWarnf("swiftObjectDeleteOnce(,volume.authToken) returned !authOK for volume %s...clearing volume.authToken", volume.name) volume.authToken = "" + } else { + mount.authTokenExpired = true + + // It's possible that mount has "moved" from volume.healthyMountList + + switch mount.mountListMembership { + case onHealthyMountList: + _ = mount.volume.healthyMountList.Remove(mount.mountListElement) + mount.mountListElement = mount.volume.authTokenExpiredMountList.PushBack(mount) + mount.mountListMembership = onAuthTokenExpiredMountList + case onLeasesExpiredMountList: + _ = mount.volume.leasesExpiredMountList.Remove(mount.mountListElement) + mount.mountListElement = mount.volume.authTokenExpiredMountList.PushBack(mount) + mount.mountListMembership = onAuthTokenExpiredMountList + case onAuthTokenExpiredMountList: + volume.authTokenExpiredMountList.MoveToBack(mount.mountListElement) + default: + logFatalf("mount.mountListMembership (%v) not one of on{Healthy|LeasesExpired|AuthTokenExpired}MountList") + } } - return + if !alreadyLocked { + globals.Unlock() + } } - - authOK = true - err = fmt.Errorf("authToken list not empty - retry possible") } return @@ -206,7 +228,7 @@ func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) return } -func (volume *volumeStruct) swiftObjectDelete(locked bool, objectNumber uint64) (authOK bool, err error) { +func (volume *volumeStruct) swiftObjectDelete(alreadyLocked bool, objectNumber uint64) (authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -223,7 +245,7 @@ func (volume *volumeStruct) swiftObjectDelete(locked bool, objectNumber uint64) nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - authOK, err = volume.swiftObjectDeleteOnce(locked, objectURL) + authOK, err = volume.swiftObjectDeleteOnce(alreadyLocked, objectURL) if nil == err { return } @@ -292,64 +314,86 @@ func swiftObjectGetOnce(objectURL string, authToken string, rangeHeaderValue str return } -func (volume *volumeStruct) swiftObjectGetOnce(locked bool, objectURL string, rangeHeaderValue string) (buf []byte, authOK bool, err error) { +func (volume *volumeStruct) swiftObjectGetOnce(alreadyLocked bool, objectURL string, rangeHeaderValue string) (buf []byte, authOK bool, err error) { var ( - mount *mountStruct - mountListElement *list.Element - ok bool - toRetryMountList *list.List + authToken string + mount *mountStruct + mountListElement *list.Element + ok bool + usingVolumeAuthToken bool ) - if !locked { - logFatalf("FLIP: (&volumeStruct).swiftObjectGetOnce(locked == false,,) not yet supported") - } - toRetryMountList = list.New() + if !alreadyLocked { + globals.Lock() + } mountListElement = volume.healthyMountList.Front() - for nil != mountListElement { - _ = volume.healthyMountList.Remove(mountListElement) + if mountListElement == nil { + if volume.authToken == "" { + if !alreadyLocked { + globals.Unlock() + } + authOK = false // Auth failed, + err = nil // but we still indicate the func succeeded + return + } + authToken = volume.authToken + usingVolumeAuthToken = true + } else { mount, ok = mountListElement.Value.(*mountStruct) if !ok { logFatalf("mountListElement.Value.(*mountStruct) returned !ok") } - buf, authOK, err = swiftObjectGetOnce(objectURL, mount.authToken, rangeHeaderValue) - if nil == err { - if authOK { - volume.appendToHealthyMountList(toRetryMountList) - mount.listElement = volume.healthyMountList.PushBack(mount) - return - } else { - mount.authTokenExpired = true - mount.listElement = volume.authTokenExpiredMountList.PushBack(mount) - } - } else { - mount.listElement = toRetryMountList.PushBack(mount) - } + // We know that mount.mountListMembership == onHealthyMountList + + volume.healthyMountList.MoveToBack(mount.mountListElement) - mountListElement = volume.healthyMountList.Front() + authToken = mount.authToken + usingVolumeAuthToken = true } - if (toRetryMountList.Len() == 0) && (volume.authToken == "") { - authOK = false // Auth failed, - err = nil // but we will still indicate the func succeeded - } else { - volume.appendToHealthyMountList(toRetryMountList) + if !alreadyLocked { + globals.Unlock() + } - if volume.authToken != "" { - buf, authOK, err = swiftObjectGetOnce(objectURL, volume.authToken, rangeHeaderValue) - if (nil == err) && !authOK { - logWarnf("swiftObjectGetOnce(,volume.authToken,) !authOK for volume %s...clearing volume.authToken", volume.name) + buf, authOK, err = swiftObjectGetOnce(objectURL, authToken, rangeHeaderValue) + if err == nil { + if !authOK { + if !alreadyLocked { + globals.Lock() + } + + if usingVolumeAuthToken { + logWarnf("swiftObjectGetOnce(,volume.authToken,) returned !authOK for volume %s...clearing volume.authToken", volume.name) volume.authToken = "" + } else { + mount.authTokenExpired = true + + // It's possible that mount has "moved" from volume.healthyMountList + + switch mount.mountListMembership { + case onHealthyMountList: + _ = mount.volume.healthyMountList.Remove(mount.mountListElement) + mount.mountListElement = mount.volume.authTokenExpiredMountList.PushBack(mount) + mount.mountListMembership = onAuthTokenExpiredMountList + case onLeasesExpiredMountList: + _ = mount.volume.leasesExpiredMountList.Remove(mount.mountListElement) + mount.mountListElement = mount.volume.authTokenExpiredMountList.PushBack(mount) + mount.mountListMembership = onAuthTokenExpiredMountList + case onAuthTokenExpiredMountList: + volume.authTokenExpiredMountList.MoveToBack(mount.mountListElement) + default: + logFatalf("mount.mountListMembership (%v) not one of on{Healthy|LeasesExpired|AuthTokenExpired}MountList") + } } - return + if !alreadyLocked { + globals.Unlock() + } } - - authOK = true - err = fmt.Errorf("authToken list not empty - retry possible") } return @@ -391,7 +435,7 @@ func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (b return } -func (volume *volumeStruct) swiftObjectGet(locked bool, objectNumber uint64) (buf []byte, authOK bool, err error) { +func (volume *volumeStruct) swiftObjectGet(alreadyLocked bool, objectNumber uint64) (buf []byte, authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -408,7 +452,7 @@ func (volume *volumeStruct) swiftObjectGet(locked bool, objectNumber uint64) (bu nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - buf, authOK, err = volume.swiftObjectGetOnce(locked, objectURL, "") + buf, authOK, err = volume.swiftObjectGetOnce(alreadyLocked, objectURL, "") if nil == err { return } @@ -468,7 +512,7 @@ func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint6 return } -func (volume *volumeStruct) swiftObjectGetRange(locked bool, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, authOK bool, err error) { +func (volume *volumeStruct) swiftObjectGetRange(alreadyLocked bool, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -488,7 +532,7 @@ func (volume *volumeStruct) swiftObjectGetRange(locked bool, objectNumber uint64 nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - buf, authOK, err = volume.swiftObjectGetOnce(locked, objectURL, rangeHeaderValue) + buf, authOK, err = volume.swiftObjectGetOnce(alreadyLocked, objectURL, rangeHeaderValue) if nil == err { return } @@ -548,7 +592,7 @@ func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64 return } -func (volume *volumeStruct) swiftObjectGetTail(locked bool, objectNumber uint64, objectLength uint64) (buf []byte, authOK bool, err error) { +func (volume *volumeStruct) swiftObjectGetTail(alreadyLocked bool, objectNumber uint64, objectLength uint64) (buf []byte, authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -568,7 +612,7 @@ func (volume *volumeStruct) swiftObjectGetTail(locked bool, objectNumber uint64, nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - buf, authOK, err = volume.swiftObjectGetOnce(locked, objectURL, rangeHeaderValue) + buf, authOK, err = volume.swiftObjectGetOnce(alreadyLocked, objectURL, rangeHeaderValue) if nil == err { return } @@ -636,64 +680,86 @@ func swiftObjectPutOnce(objectURL string, authToken string, body io.ReadSeeker) return } -func (volume *volumeStruct) swiftObjectPutOnce(locked bool, objectURL string, body io.ReadSeeker) (authOK bool, err error) { +func (volume *volumeStruct) swiftObjectPutOnce(alreadyLocked bool, objectURL string, body io.ReadSeeker) (authOK bool, err error) { var ( - mount *mountStruct - mountListElement *list.Element - ok bool - toRetryMountList *list.List + authToken string + mount *mountStruct + mountListElement *list.Element + ok bool + usingVolumeAuthToken bool ) - if !locked { - logFatalf("FLIP: (&volumeStruct).swiftObjectPutOnce(locked == false,,) not yet supported") - } - toRetryMountList = list.New() + if !alreadyLocked { + globals.Lock() + } mountListElement = volume.healthyMountList.Front() - for nil != mountListElement { - _ = volume.healthyMountList.Remove(mountListElement) + if mountListElement == nil { + if volume.authToken == "" { + if !alreadyLocked { + globals.Unlock() + } + authOK = false // Auth failed, + err = nil // but we still indicate the func succeeded + return + } + authToken = volume.authToken + usingVolumeAuthToken = true + } else { mount, ok = mountListElement.Value.(*mountStruct) if !ok { logFatalf("mountListElement.Value.(*mountStruct) returned !ok") } - authOK, err = swiftObjectPutOnce(objectURL, mount.authToken, body) - if nil == err { - if authOK { - volume.appendToHealthyMountList(toRetryMountList) - mount.listElement = volume.healthyMountList.PushBack(mount) - return - } else { - mount.authTokenExpired = true - mount.listElement = volume.authTokenExpiredMountList.PushBack(mount) - } - } else { - mount.listElement = toRetryMountList.PushBack(mount) - } + // We know that mount.mountListMembership == onHealthyMountList + + volume.healthyMountList.MoveToBack(mount.mountListElement) - mountListElement = volume.healthyMountList.Front() + authToken = mount.authToken + usingVolumeAuthToken = true } - if (toRetryMountList.Len() == 0) && (volume.authToken == "") { - authOK = false // Auth failed, - err = nil // but we will still indicate the func succeeded - } else { - volume.appendToHealthyMountList(toRetryMountList) + if !alreadyLocked { + globals.Unlock() + } - if volume.authToken != "" { - authOK, err = swiftObjectPutOnce(objectURL, volume.authToken, body) - if (nil == err) && !authOK { - logWarnf("swiftObjectPutOnce(,volume.authToken,) !authOK for volume %s...clearing volume.authToken", volume.name) + authOK, err = swiftObjectPutOnce(objectURL, authToken, body) + if err == nil { + if !authOK { + if !alreadyLocked { + globals.Lock() + } + + if usingVolumeAuthToken { + logWarnf("swiftObjectPutOnce(,volume.authToken,) returned !authOK for volume %s...clearing volume.authToken", volume.name) volume.authToken = "" + } else { + mount.authTokenExpired = true + + // It's possible that mount has "moved" from volume.healthyMountList + + switch mount.mountListMembership { + case onHealthyMountList: + _ = mount.volume.healthyMountList.Remove(mount.mountListElement) + mount.mountListElement = mount.volume.authTokenExpiredMountList.PushBack(mount) + mount.mountListMembership = onAuthTokenExpiredMountList + case onLeasesExpiredMountList: + _ = mount.volume.leasesExpiredMountList.Remove(mount.mountListElement) + mount.mountListElement = mount.volume.authTokenExpiredMountList.PushBack(mount) + mount.mountListMembership = onAuthTokenExpiredMountList + case onAuthTokenExpiredMountList: + volume.authTokenExpiredMountList.MoveToBack(mount.mountListElement) + default: + logFatalf("mount.mountListMembership (%v) not one of on{Healthy|LeasesExpired|AuthTokenExpired}MountList") + } } - return + if !alreadyLocked { + globals.Unlock() + } } - - authOK = true - err = fmt.Errorf("authToken list not empty - retry possible") } return @@ -735,7 +801,7 @@ func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, bo return } -func (volume *volumeStruct) swiftObjectPut(locked bool, objectNumber uint64, body io.ReadSeeker) (authOK bool, err error) { +func (volume *volumeStruct) swiftObjectPut(alreadyLocked bool, objectNumber uint64, body io.ReadSeeker) (authOK bool, err error) { var ( nextSwiftRetryDelay time.Duration numSwiftRetries uint32 @@ -752,7 +818,7 @@ func (volume *volumeStruct) swiftObjectPut(locked bool, objectNumber uint64, bod nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - authOK, err = volume.swiftObjectPutOnce(locked, objectURL, body) + authOK, err = volume.swiftObjectPutOnce(alreadyLocked, objectURL, body) if nil == err { if authOK { return @@ -774,27 +840,3 @@ func (volume *volumeStruct) swiftObjectPut(locked bool, objectNumber uint64, bod return } - -func (volume *volumeStruct) appendToHealthyMountList(toRetryMountList *list.List) { - var ( - mount *mountStruct - mountListElement *list.Element - ok bool - ) - - for { - mountListElement = toRetryMountList.Front() - if mountListElement == nil { - return - } - - _ = toRetryMountList.Remove(mountListElement) - - mount, ok = mountListElement.Value.(*mountStruct) - if !ok { - logFatalf("mountListElement.Value.(*mountStruct) returned !ok") - } - - mount.listElement = volume.healthyMountList.PushBack(mount) - } -} diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index d3493fce..326aea92 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -944,25 +944,14 @@ func (volume *volumeStruct) doObjectDelete(activeDeleteObjectNumberListElement * logFatalf("activeDeleteObjectNumberListElement.Value.(uint64) returned !ok") } - // FLIP: Here is the top limit of the swapped code - - // globals.Unlock() - - // authOK, err = volume.swiftObjectDelete(false, objectNumber) - // if nil != err { - // logFatalf("volume.swiftObjectDelete(locked: false, objectNumber: %016X) failed: %v", objectNumber, err) - // } - - // globals.Lock() - - // FLIP: flip below to the above (i.e. locked: false use) + globals.Unlock() - authOK, err = volume.swiftObjectDelete(true, objectNumber) + authOK, err = volume.swiftObjectDelete(false, objectNumber) if nil != err { - logFatalf("volume.swiftObjectDelete(locked: true, objectNumber: %016X) failed: %v", objectNumber, err) + logFatalf("volume.swiftObjectDelete(locked: false, objectNumber: %016X) failed: %v", objectNumber, err) } - // FLIP: Here is the bottom limit of the swapped code + globals.Lock() volume.activeDeleteObjectNumberList.Remove(activeDeleteObjectNumberListElement) From a6a9263f59da834e11c10ddb148d30da941361cb Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 29 Dec 2021 14:44:58 -0800 Subject: [PATCH 223/258] Initial (major) pruning --- cleanproxyfs/Makefile | 6 - cleanproxyfs/dummy_test.go | 11 - cleanproxyfs/main.go | 275 - confgen/Makefile | 6 - confgen/api.go | 91 - confgen/api_internal.go | 1089 ---- confgen/api_test.go | 100 - confgen/confgen/Makefile | 6 - confgen/confgen/dummy_test.go | 11 - confgen/confgen/main.go | 109 - confgen/mac.conf | 4 - confgen/saio.conf | 4 - .../proxyfs-etcd-endpoints.conf | 2 - .../proxyfs-shares.conf | 235 - .../sample-proxyfs-configuration/proxyfs.conf | 136 - .../samba/smb.conf | 126 - confgen/template.go | 80 - confgen/templates/smb_globals.tmpl | 89 - confgen/templates/smb_shares.tmpl | 37 - dlm/Makefile | 6 - dlm/api.go | 166 - dlm/config.go | 75 - dlm/llm.go | 401 -- dlm/llm_test.go | 691 --- emswift/Makefile | 6 - emswift/dummy_test.go | 11 - emswift/emswift.conf | 18 - emswift/emswiftpkg/Makefile | 6 - emswift/emswiftpkg/api.go | 23 - emswift/emswiftpkg/impl.go | 1586 ------ emswift/emswiftpkg/pkg_test.go | 1529 ------ emswift/main.go | 67 - etcdclient/Makefile | 6 - etcdclient/api.go | 60 - etcdclient/dummy_test.go | 11 - evtlog/Makefile | 6 - evtlog/api.go | 380 -- evtlog/api_internal.go | 1219 ---- evtlog/api_test.go | 180 - evtlog/benchmark_test.go | 403 -- evtlog/config.go | 280 - evtlog/pfsevtlogd/Makefile | 6 - evtlog/pfsevtlogd/dummy_test.go | 11 - evtlog/pfsevtlogd/main.go | 135 - fs/Makefile | 6 - fs/api.go | 312 -- fs/api_internal.go | 4449 --------------- fs/api_test.go | 1358 ----- fs/config.go | 377 -- fs/metadata_stress_test.go | 709 --- fs/resolve_path.go | 1023 ---- fs/resolve_path_test.go | 424 -- fs/setup_teardown_test.go | 205 - fs/treewalk.go | 1448 ----- fs/utils.go | 98 - fsworkout/Makefile | 6 - fsworkout/dummy_test.go | 11 - fsworkout/main.go | 352 -- fuse/Makefile | 6 - fuse/config.go | 334 -- fuse/dir.go | 357 -- fuse/dummy_test.go | 9 - fuse/file.go | 196 - fuse/fuse.go | 71 - fuse/symlink.go | 125 - go.mod | 44 +- go.sum | 249 +- halter/Makefile | 6 - halter/api.go | 124 - halter/api_test.go | 195 - halter/config.go | 82 - headhunter/Makefile | 6 - headhunter/api.go | 126 - headhunter/api_internal.go | 1771 ------ headhunter/api_test.go | 312 -- headhunter/checkpoint.go | 3119 ----------- headhunter/config.go | 882 --- headhunter/sortedmap_helper.go | 216 - headhunter/stress_test.go | 746 --- httpserver/.gitignore | 13 - httpserver/Makefile | 21 - httpserver/config.go | 317 -- httpserver/html_templates.go | 1255 ----- httpserver/request_handler.go | 3082 ----------- httpserver/request_handler_test.go | 67 - httpserver/setup_teardown_test.go | 135 - httpserver/static-content.go | 22 - httpserver/static-content/bootstrap.min.css | 7 - httpserver/static-content/bootstrap.min.js | 7 - httpserver/static-content/jquery.min.js | 2 - httpserver/static-content/jsontree.js | 182 - .../font/css/open-iconic-bootstrap.min.css | 1 - .../open-iconic/font/fonts/open-iconic.eot | Bin 28196 -> 0 bytes .../open-iconic/font/fonts/open-iconic.otf | Bin 20996 -> 0 bytes .../open-iconic/font/fonts/open-iconic.svg | 543 -- .../open-iconic/font/fonts/open-iconic.ttf | Bin 28028 -> 0 bytes .../open-iconic/font/fonts/open-iconic.woff | Bin 14984 -> 0 bytes httpserver/static-content/popper.min.js | 5 - httpserver/static-content/styles.css | 71 - inode/.gitignore | 1 - inode/Makefile | 9 - inode/api.go | 260 - inode/api_test.go | 2328 -------- inode/benchmark_test.go | 154 - inode/cache.go | 133 - inode/coalesce_test.go | 441 -- inode/config.go | 815 --- inode/config_test.go | 47 - inode/cron.go | 645 --- inode/cron_test.go | 728 --- inode/dir.go | 1429 ----- inode/dir_stress_test.go | 155 - inode/file.go | 1345 ----- inode/file_extent_test.go | 640 --- inode/file_flusher.go | 549 -- inode/file_stress_test.go | 117 - inode/gc_test.go | 115 - inode/inode.go | 2439 -------- inode/locker.go | 120 - inode/locker_test.go | 92 - inode/payload.go | 234 - inode/setup_teardown_test.go | 220 - inode/symlink.go | 67 - inode/validate_test.go | 157 - inode/volume.go | 163 - inodeworkout/Makefile | 6 - inodeworkout/dummy_test.go | 11 - inodeworkout/main.go | 427 -- jrpcfs/Makefile | 6 - jrpcfs/api.go | 1010 ---- jrpcfs/config.go | 500 -- jrpcfs/encoding_test.go | 1140 ---- jrpcfs/filesystem.go | 2251 -------- jrpcfs/io.go | 484 -- jrpcfs/io_stats | 138 - jrpcfs/lease.go | 2112 ------- jrpcfs/lease_test.go | 2158 -------- jrpcfs/middleware.go | 421 -- jrpcfs/middleware_test.go | 2079 ------- jrpcfs/ping.go | 24 - jrpcfs/retryrpc.go | 64 - jrpcfs/setup_teardown_test.go | 463 -- jrpcfs/test/Makefile | 15 - jrpcfs/test/client_test.c | 223 - jrpcfs/test/client_test.py | 63 - jrpcfs/test/example.go | 115 - liveness/Makefile | 6 - liveness/api.go | 54 - liveness/api_internal.go | 256 - liveness/config.go | 887 --- liveness/messages.go | 597 -- liveness/polling.go | 1014 ---- liveness/polling_test.go | 306 -- liveness/states.go | 855 --- make-static-content/.gitignore | 1 - make-static-content/Makefile | 9 - make-static-content/dummy_test.go | 11 - make-static-content/main.go | 128 - meta_middleware/meta_middleware/__init__.py | 4 - meta_middleware/meta_middleware/middleware.py | 46 - meta_middleware/setup.py | 25 - meta_middleware/test-requirements.txt | 2 - meta_middleware/tests/__init__.py | 6 - meta_middleware/tox.ini | 23 - mkproxyfs/Makefile | 6 - mkproxyfs/api.go | 352 -- mkproxyfs/dummy_test.go | 11 - mkproxyfs/mkproxyfs/Makefile | 6 - mkproxyfs/mkproxyfs/dummy_test.go | 11 - mkproxyfs/mkproxyfs/main.go | 64 - pfs-crash/Makefile | 6 - pfs-crash/b_tree_load.sh | 19 - pfs-crash/dd4Kx256.sh | 12 - pfs-crash/dummy_test.go | 11 - pfs-crash/fio.sh | 9 - pfs-crash/main.go | 869 --- pfs-crash/touch.sh | 12 - pfs-fsck/Makefile | 6 - pfs-fsck/dummy_test.go | 11 - pfs-fsck/main.go | 1091 ---- pfs-restart-test/Makefile | 6 - pfs-restart-test/dummy_test.go | 11 - pfs-restart-test/macramswift0.conf | 10 - pfs-restart-test/main.go | 664 --- pfs-restart-test/saioproxyfsd0.conf | 10 - pfs-restart-test/saioramswift0.conf | 10 - pfs-stress/Makefile | 6 - pfs-stress/dummy_test.go | 11 - pfs-stress/main.go | 385 -- pfs-stress/pfs-stress.conf | 10 - pfs-swift-load/.gitignore | 1 - pfs-swift-load/Makefile | 6 - pfs-swift-load/dummy_test.go | 11 - pfs-swift-load/main.go | 1000 ---- pfs-swift-load/pfs-swift-load-plot | 95 - pfs-swift-load/pfs-swift-load.conf | 49 - pfs-swift-load/requirements.txt | 3 - pfs_middleware/pfs_middleware/__init__.py | 21 - .../pfs_middleware/bimodal_checker.py | 218 - pfs_middleware/pfs_middleware/middleware.py | 2174 -------- pfs_middleware/pfs_middleware/pfs_errno.py | 26 - pfs_middleware/pfs_middleware/rpc.py | 541 -- pfs_middleware/pfs_middleware/s3_compat.py | 61 - pfs_middleware/pfs_middleware/swift_code.py | 1081 ---- pfs_middleware/pfs_middleware/utils.py | 138 - pfs_middleware/setup.py | 31 - pfs_middleware/test-requirements.txt | 1 - pfs_middleware/tests/__init__.py | 7 - pfs_middleware/tests/helpers.py | 153 - pfs_middleware/tests/test_bimodal_checker.py | 319 -- pfs_middleware/tests/test_pfs_middleware.py | 4882 ----------------- pfs_middleware/tests/test_rpc.py | 146 - pfs_middleware/tests/test_s3_compat.py | 84 - pfs_middleware/tests/test_utils.py | 42 - pfs_middleware/tox.ini | 31 - pfsagentConfig/Makefile | 6 - pfsagentConfig/dataStructures.go | 90 - pfsagentConfig/menu.go | 40 - pfsagentConfig/pfsagentConfig/Makefile | 6 - pfsagentConfig/pfsagentConfig/dummy_test.go | 11 - pfsagentConfig/pfsagentConfig/help.go | 43 - pfsagentConfig/pfsagentConfig/main.go | 63 - pfsagentConfig/stateMachine.go | 419 -- pfsagentConfig/strings.go | 79 - pfsagentConfig/swiftStuff.go | 117 - pfsagentConfig/util_test.go | 11 - pfsagentConfig/utils.go | 138 - pfsagentConfig/validators.go | 128 - pfsagentd/.gitignore | 1 - pfsagentd/LeaseManagement.uml | 57 - pfsagentd/LeaseRequestClient.uml | 44 - pfsagentd/LeaseRequestServer.uml | 47 - pfsagentd/LockManagement.uml | 29 - .../LockManagementWithExclusiveLease.uml | 36 - pfsagentd/LockManagementWithSharedLease.uml | 20 - pfsagentd/LockRequest.uml | 15 - pfsagentd/Makefile | 6 - pfsagentd/README.md | 191 - pfsagentd/container/build/Dockerfile | 38 - pfsagentd/container/insert/.gitignore | 5 - .../container/insert/MakePFSAgentDockerfile | 206 - .../insert/MakePFSAgentDockerfile.conf_SAMPLE | 17 - pfsagentd/file_inode.go | 1938 ------- pfsagentd/fission.go | 2575 --------- pfsagentd/functional_test.go | 389 -- pfsagentd/globals.go | 799 --- pfsagentd/html_templates.go | 25 - pfsagentd/http_server.go | 431 -- pfsagentd/jrpc.go | 173 - pfsagentd/lease.go | 982 ---- pfsagentd/log.go | 78 - pfsagentd/main.go | 105 - pfsagentd/pfsagent.conf | 48 - pfsagentd/pfsagentd-init/Makefile | 6 - pfsagentd/pfsagentd-init/README.md | 24 - pfsagentd/pfsagentd-init/dummy_test.go | 11 - pfsagentd/pfsagentd-init/main.go | 397 -- .../pfsagentd-swift-auth-plugin/Makefile | 6 - .../pfsagentd-swift-auth-plugin/README.md | 49 - .../pfsagentd-swift-auth-plugin/dummy_test.go | 11 - pfsagentd/pfsagentd-swift-auth-plugin/main.go | 147 - pfsagentd/request.go | 566 -- pfsagentd/setup_teardown_test.go | 375 -- pfsagentd/swift_proxy_emulator_test.go | 411 -- pfsagentd/unit_test.go | 399 -- pfsconfjson/Makefile | 6 - pfsconfjson/dummy_test.go | 11 - pfsconfjson/main.go | 53 - pfsconfjsonpacked/Makefile | 6 - pfsconfjsonpacked/dummy_test.go | 11 - pfsconfjsonpacked/main.go | 43 - pfsworkout/Makefile | 6 - pfsworkout/dummy_test.go | 11 - pfsworkout/main.go | 1240 ----- proxyfsd/Makefile | 6 - proxyfsd/cluster_1_peer.conf | 22 - proxyfsd/cluster_3_peers.conf | 32 - proxyfsd/daemon.go | 152 - proxyfsd/daemon_test.go | 468 -- proxyfsd/debug.conf | 13 - proxyfsd/default.conf | 277 - proxyfsd/file_server.conf | 147 - proxyfsd/file_server_mac_3_peers.conf | 155 - proxyfsd/httpserver.conf | 6 - proxyfsd/logging.conf | 29 - proxyfsd/mac_cluster_1_peer.conf | 22 - proxyfsd/mac_cluster_3_peers.conf | 39 - proxyfsd/macproxyfsd0.conf | 15 - proxyfsd/macproxyfsd1.conf | 15 - proxyfsd/macproxyfsd2.conf | 15 - proxyfsd/macproxyfsd3.conf | 15 - proxyfsd/proxyfsd/Makefile | 6 - proxyfsd/proxyfsd/dummy_test.go | 11 - proxyfsd/proxyfsd/main.go | 68 - proxyfsd/proxyfsd0.conf | 15 - proxyfsd/proxyfsd1.conf | 15 - proxyfsd/proxyfsd2.conf | 15 - proxyfsd/proxyfsd3.conf | 15 - proxyfsd/rpc_server.conf | 17 - proxyfsd/saio_cluster_1_peer.conf | 22 - proxyfsd/saio_logging.conf | 24 - proxyfsd/saioproxyfsd0.conf | 19 - proxyfsd/stats.conf | 5 - proxyfsd/statslogger.conf | 8 - proxyfsd/swift_client.conf | 33 - proxyfsd/trackedlock.conf | 19 - ramswift/Makefile | 6 - ramswift/chaos_settings.conf | 22 - ramswift/daemon.go | 1667 ------ ramswift/daemon_test.go | 1274 ----- ramswift/macramswift0.conf | 10 - ramswift/ramswift/Makefile | 6 - ramswift/ramswift/dummy_test.go | 11 - ramswift/ramswift/main.go | 25 - ramswift/ramswift0.conf | 10 - ramswift/saioramswift0.conf | 10 - ramswift/swift_info.conf | 20 - readdir-stress/.gitignore | 1 - readdir-stress/Makefile | 28 - readdir-stress/readdir-stress.c | 327 -- saio/Vagrantfile | 36 - saio/bin/flush_caches | 6 - saio/bin/provision_middleware | 9 - saio/bin/reset_etcd | 6 - saio/bin/start_etcd | 18 - saio/bin/start_proxyfs_and_swift | 71 - saio/bin/start_swift_only | 23 - saio/bin/stop_etcd | 6 - saio/bin/unmount_and_stop_pfs | 93 - saio/container/.gitignore | 2 - saio/container/Dockerfile | 205 - saio/container/Makefile | 17 - saio/container/docker_startup.sh | 32 - .../container/etc/swift/account-server/1.conf | 27 - .../etc/swift/container-reconciler.conf | 47 - .../etc/swift/container-server/1.conf | 29 - .../etc/swift/container-sync-realms.conf | 4 - saio/container/etc/swift/object-expirer.conf | 59 - saio/container/etc/swift/object-server/1.conf | 29 - saio/container/etc/swift/proxy-server.conf | 93 - .../proxy-noauth.conf.d/20_settings.conf | 45 - saio/container/etc/swift/swift.conf | 17 - saio/container/etc/swift/test.conf | 112 - saio/container/proxyfs.conf | 232 - saio/etc/exports | 1 - saio/etc/samba/smb.conf | 25 - saio/etc/swift/account-server/1.conf | 27 - saio/etc/swift/account-server/2.conf | 27 - saio/etc/swift/account-server/3.conf | 27 - saio/etc/swift/account-server/4.conf | 27 - saio/etc/swift/container-reconciler.conf | 47 - saio/etc/swift/container-server/1.conf | 29 - saio/etc/swift/container-server/2.conf | 29 - saio/etc/swift/container-server/3.conf | 29 - saio/etc/swift/container-server/4.conf | 29 - saio/etc/swift/container-sync-realms.conf | 4 - saio/etc/swift/object-expirer.conf | 59 - saio/etc/swift/object-server/1.conf | 29 - saio/etc/swift/object-server/2.conf | 29 - saio/etc/swift/object-server/3.conf | 29 - saio/etc/swift/object-server/4.conf | 29 - saio/etc/swift/proxy-server.conf | 93 - .../proxy-noauth.conf.d/20_settings.conf | 45 - saio/etc/swift/swift.conf | 29 - saio/etc/swift/test.conf | 112 - saio/home/swift/bin/remakerings | 81 - saio/home/swift/bin/resetswift | 27 - saio/proxyfs.conf | 38 - saio/usr/lib/systemd/system/pfsagentd.service | 20 - saio/usr/lib/systemd/system/proxyfsd.service | 10 - saio/usr/local/go/src/runtime/runtime-gdb.py | 541 -- saio/vagrant_provision.sh | 411 -- sait/Vagrantfile | 82 - sait/bin/provision_middleware | 9 - sait/bin/start_proxyfs_and_swift | 90 - sait/bin/start_swift_only | 20 - sait/bin/unmount_and_stop_pfs | 32 - sait/etc/swift/container-reconciler.conf | 47 - sait/etc/swift/container-sync-realms.conf | 4 - sait/etc/swift/object-expirer.conf | 59 - sait/etc/swift/proxy-server.conf | 93 - .../proxy-noauth.conf.d/20_settings.conf | 45 - sait/etc/swift/swift.conf | 17 - sait/home/swift/bin/remakerings | 31 - sait/home/swift/bin/resetswift | 30 - sait/proxyfs.conf | 43 - sait/sait1/etc/swift/account-server/1.conf | 27 - sait/sait1/etc/swift/container-server/1.conf | 29 - sait/sait1/etc/swift/object-server/1.conf | 29 - sait/sait1/proxyfs.conf | 4 - .../usr/lib/systemd/system/proxyfsd.service | 10 - sait/sait2/etc/swift/account-server/2.conf | 27 - sait/sait2/etc/swift/container-server/2.conf | 29 - sait/sait2/etc/swift/object-server/2.conf | 29 - sait/sait2/proxyfs.conf | 4 - .../usr/lib/systemd/system/proxyfsd.service | 10 - sait/sait3/etc/swift/account-server/3.conf | 27 - sait/sait3/etc/swift/container-server/3.conf | 29 - sait/sait3/etc/swift/object-server/3.conf | 29 - sait/sait3/proxyfs.conf | 4 - .../usr/lib/systemd/system/proxyfsd.service | 10 - sait/usr/local/go/src/runtime/runtime-gdb.py | 541 -- sait/vagrant_provision.sh | 410 -- samba-dc/Vagrantfile | 28 - samba-dc/vagrant_provision.sh | 173 - stats/Makefile | 6 - stats/api.go | 74 - stats/api_internal.go | 305 - stats/api_test.go | 464 -- stats/config.go | 183 - stats/sender.go | 126 - stats/strings.go | 250 - statslogger/Makefile | 6 - statslogger/config.go | 409 -- statslogger/config_test.go | 953 ---- statslogger/simplestats.go | 60 - swiftclient/Makefile | 6 - swiftclient/account.go | 474 -- swiftclient/api.go | 178 - swiftclient/api_test.go | 1732 ------ swiftclient/config.go | 387 -- swiftclient/container.go | 475 -- swiftclient/http_status.go | 134 - swiftclient/object.go | 1746 ------ swiftclient/retry.go | 148 - swiftclient/utils.go | 1131 ---- swiftclient/utils_test.go | 130 - utf/utf.go | 95 - utf/utf_test.go | 95 - win10/Vagrantfile | 45 - 430 files changed, 21 insertions(+), 114341 deletions(-) delete mode 100644 cleanproxyfs/Makefile delete mode 100644 cleanproxyfs/dummy_test.go delete mode 100644 cleanproxyfs/main.go delete mode 100644 confgen/Makefile delete mode 100644 confgen/api.go delete mode 100644 confgen/api_internal.go delete mode 100644 confgen/api_test.go delete mode 100644 confgen/confgen/Makefile delete mode 100644 confgen/confgen/dummy_test.go delete mode 100644 confgen/confgen/main.go delete mode 100644 confgen/mac.conf delete mode 100644 confgen/saio.conf delete mode 100644 confgen/sample-proxyfs-configuration/proxyfs-etcd-endpoints.conf delete mode 100644 confgen/sample-proxyfs-configuration/proxyfs-shares.conf delete mode 100644 confgen/sample-proxyfs-configuration/proxyfs.conf delete mode 100644 confgen/sample-proxyfs-configuration/samba/smb.conf delete mode 100644 confgen/template.go delete mode 100644 confgen/templates/smb_globals.tmpl delete mode 100644 confgen/templates/smb_shares.tmpl delete mode 100644 dlm/Makefile delete mode 100644 dlm/api.go delete mode 100644 dlm/config.go delete mode 100644 dlm/llm.go delete mode 100644 dlm/llm_test.go delete mode 100644 emswift/Makefile delete mode 100644 emswift/dummy_test.go delete mode 100644 emswift/emswift.conf delete mode 100644 emswift/emswiftpkg/Makefile delete mode 100644 emswift/emswiftpkg/api.go delete mode 100644 emswift/emswiftpkg/impl.go delete mode 100644 emswift/emswiftpkg/pkg_test.go delete mode 100644 emswift/main.go delete mode 100644 etcdclient/Makefile delete mode 100644 etcdclient/api.go delete mode 100644 etcdclient/dummy_test.go delete mode 100644 evtlog/Makefile delete mode 100644 evtlog/api.go delete mode 100644 evtlog/api_internal.go delete mode 100644 evtlog/api_test.go delete mode 100644 evtlog/benchmark_test.go delete mode 100644 evtlog/config.go delete mode 100644 evtlog/pfsevtlogd/Makefile delete mode 100644 evtlog/pfsevtlogd/dummy_test.go delete mode 100644 evtlog/pfsevtlogd/main.go delete mode 100644 fs/Makefile delete mode 100644 fs/api.go delete mode 100644 fs/api_internal.go delete mode 100644 fs/api_test.go delete mode 100644 fs/config.go delete mode 100644 fs/metadata_stress_test.go delete mode 100644 fs/resolve_path.go delete mode 100644 fs/resolve_path_test.go delete mode 100644 fs/setup_teardown_test.go delete mode 100644 fs/treewalk.go delete mode 100644 fs/utils.go delete mode 100644 fsworkout/Makefile delete mode 100644 fsworkout/dummy_test.go delete mode 100644 fsworkout/main.go delete mode 100644 fuse/Makefile delete mode 100644 fuse/config.go delete mode 100644 fuse/dir.go delete mode 100644 fuse/dummy_test.go delete mode 100644 fuse/file.go delete mode 100644 fuse/fuse.go delete mode 100644 fuse/symlink.go delete mode 100644 halter/Makefile delete mode 100644 halter/api.go delete mode 100644 halter/api_test.go delete mode 100644 halter/config.go delete mode 100644 headhunter/Makefile delete mode 100644 headhunter/api.go delete mode 100644 headhunter/api_internal.go delete mode 100644 headhunter/api_test.go delete mode 100644 headhunter/checkpoint.go delete mode 100644 headhunter/config.go delete mode 100644 headhunter/sortedmap_helper.go delete mode 100644 headhunter/stress_test.go delete mode 100644 httpserver/.gitignore delete mode 100644 httpserver/Makefile delete mode 100644 httpserver/config.go delete mode 100644 httpserver/html_templates.go delete mode 100644 httpserver/request_handler.go delete mode 100644 httpserver/request_handler_test.go delete mode 100644 httpserver/setup_teardown_test.go delete mode 100644 httpserver/static-content.go delete mode 100644 httpserver/static-content/bootstrap.min.css delete mode 100644 httpserver/static-content/bootstrap.min.js delete mode 100644 httpserver/static-content/jquery.min.js delete mode 100644 httpserver/static-content/jsontree.js delete mode 100755 httpserver/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css delete mode 100755 httpserver/static-content/open-iconic/font/fonts/open-iconic.eot delete mode 100755 httpserver/static-content/open-iconic/font/fonts/open-iconic.otf delete mode 100755 httpserver/static-content/open-iconic/font/fonts/open-iconic.svg delete mode 100755 httpserver/static-content/open-iconic/font/fonts/open-iconic.ttf delete mode 100755 httpserver/static-content/open-iconic/font/fonts/open-iconic.woff delete mode 100644 httpserver/static-content/popper.min.js delete mode 100644 httpserver/static-content/styles.css delete mode 100644 inode/.gitignore delete mode 100644 inode/Makefile delete mode 100644 inode/api.go delete mode 100644 inode/api_test.go delete mode 100644 inode/benchmark_test.go delete mode 100644 inode/cache.go delete mode 100644 inode/coalesce_test.go delete mode 100644 inode/config.go delete mode 100644 inode/config_test.go delete mode 100644 inode/cron.go delete mode 100644 inode/cron_test.go delete mode 100644 inode/dir.go delete mode 100644 inode/dir_stress_test.go delete mode 100644 inode/file.go delete mode 100644 inode/file_extent_test.go delete mode 100644 inode/file_flusher.go delete mode 100644 inode/file_stress_test.go delete mode 100644 inode/gc_test.go delete mode 100644 inode/inode.go delete mode 100644 inode/locker.go delete mode 100644 inode/locker_test.go delete mode 100644 inode/payload.go delete mode 100644 inode/setup_teardown_test.go delete mode 100644 inode/symlink.go delete mode 100644 inode/validate_test.go delete mode 100644 inode/volume.go delete mode 100644 inodeworkout/Makefile delete mode 100644 inodeworkout/dummy_test.go delete mode 100644 inodeworkout/main.go delete mode 100644 jrpcfs/Makefile delete mode 100644 jrpcfs/api.go delete mode 100644 jrpcfs/config.go delete mode 100644 jrpcfs/encoding_test.go delete mode 100644 jrpcfs/filesystem.go delete mode 100644 jrpcfs/io.go delete mode 100755 jrpcfs/io_stats delete mode 100644 jrpcfs/lease.go delete mode 100644 jrpcfs/lease_test.go delete mode 100644 jrpcfs/middleware.go delete mode 100644 jrpcfs/middleware_test.go delete mode 100644 jrpcfs/ping.go delete mode 100644 jrpcfs/retryrpc.go delete mode 100644 jrpcfs/setup_teardown_test.go delete mode 100644 jrpcfs/test/Makefile delete mode 100644 jrpcfs/test/client_test.c delete mode 100644 jrpcfs/test/client_test.py delete mode 100644 jrpcfs/test/example.go delete mode 100644 liveness/Makefile delete mode 100644 liveness/api.go delete mode 100644 liveness/api_internal.go delete mode 100644 liveness/config.go delete mode 100644 liveness/messages.go delete mode 100644 liveness/polling.go delete mode 100644 liveness/polling_test.go delete mode 100644 liveness/states.go delete mode 100644 make-static-content/.gitignore delete mode 100644 make-static-content/Makefile delete mode 100644 make-static-content/dummy_test.go delete mode 100644 make-static-content/main.go delete mode 100644 meta_middleware/meta_middleware/__init__.py delete mode 100644 meta_middleware/meta_middleware/middleware.py delete mode 100644 meta_middleware/setup.py delete mode 100644 meta_middleware/test-requirements.txt delete mode 100644 meta_middleware/tests/__init__.py delete mode 100644 meta_middleware/tox.ini delete mode 100644 mkproxyfs/Makefile delete mode 100644 mkproxyfs/api.go delete mode 100644 mkproxyfs/dummy_test.go delete mode 100644 mkproxyfs/mkproxyfs/Makefile delete mode 100644 mkproxyfs/mkproxyfs/dummy_test.go delete mode 100644 mkproxyfs/mkproxyfs/main.go delete mode 100644 pfs-crash/Makefile delete mode 100755 pfs-crash/b_tree_load.sh delete mode 100755 pfs-crash/dd4Kx256.sh delete mode 100644 pfs-crash/dummy_test.go delete mode 100755 pfs-crash/fio.sh delete mode 100644 pfs-crash/main.go delete mode 100755 pfs-crash/touch.sh delete mode 100644 pfs-fsck/Makefile delete mode 100644 pfs-fsck/dummy_test.go delete mode 100644 pfs-fsck/main.go delete mode 100644 pfs-restart-test/Makefile delete mode 100644 pfs-restart-test/dummy_test.go delete mode 100644 pfs-restart-test/macramswift0.conf delete mode 100644 pfs-restart-test/main.go delete mode 100644 pfs-restart-test/saioproxyfsd0.conf delete mode 100644 pfs-restart-test/saioramswift0.conf delete mode 100644 pfs-stress/Makefile delete mode 100644 pfs-stress/dummy_test.go delete mode 100644 pfs-stress/main.go delete mode 100644 pfs-stress/pfs-stress.conf delete mode 100644 pfs-swift-load/.gitignore delete mode 100644 pfs-swift-load/Makefile delete mode 100644 pfs-swift-load/dummy_test.go delete mode 100644 pfs-swift-load/main.go delete mode 100755 pfs-swift-load/pfs-swift-load-plot delete mode 100644 pfs-swift-load/pfs-swift-load.conf delete mode 100644 pfs-swift-load/requirements.txt delete mode 100644 pfs_middleware/pfs_middleware/__init__.py delete mode 100644 pfs_middleware/pfs_middleware/bimodal_checker.py delete mode 100644 pfs_middleware/pfs_middleware/middleware.py delete mode 100644 pfs_middleware/pfs_middleware/pfs_errno.py delete mode 100644 pfs_middleware/pfs_middleware/rpc.py delete mode 100644 pfs_middleware/pfs_middleware/s3_compat.py delete mode 100644 pfs_middleware/pfs_middleware/swift_code.py delete mode 100644 pfs_middleware/pfs_middleware/utils.py delete mode 100644 pfs_middleware/setup.py delete mode 100644 pfs_middleware/test-requirements.txt delete mode 100644 pfs_middleware/tests/__init__.py delete mode 100644 pfs_middleware/tests/helpers.py delete mode 100644 pfs_middleware/tests/test_bimodal_checker.py delete mode 100644 pfs_middleware/tests/test_pfs_middleware.py delete mode 100644 pfs_middleware/tests/test_rpc.py delete mode 100644 pfs_middleware/tests/test_s3_compat.py delete mode 100644 pfs_middleware/tests/test_utils.py delete mode 100644 pfs_middleware/tox.ini delete mode 100644 pfsagentConfig/Makefile delete mode 100644 pfsagentConfig/dataStructures.go delete mode 100644 pfsagentConfig/menu.go delete mode 100644 pfsagentConfig/pfsagentConfig/Makefile delete mode 100644 pfsagentConfig/pfsagentConfig/dummy_test.go delete mode 100644 pfsagentConfig/pfsagentConfig/help.go delete mode 100644 pfsagentConfig/pfsagentConfig/main.go delete mode 100644 pfsagentConfig/stateMachine.go delete mode 100644 pfsagentConfig/strings.go delete mode 100644 pfsagentConfig/swiftStuff.go delete mode 100644 pfsagentConfig/util_test.go delete mode 100644 pfsagentConfig/utils.go delete mode 100644 pfsagentConfig/validators.go delete mode 100644 pfsagentd/.gitignore delete mode 100644 pfsagentd/LeaseManagement.uml delete mode 100644 pfsagentd/LeaseRequestClient.uml delete mode 100644 pfsagentd/LeaseRequestServer.uml delete mode 100644 pfsagentd/LockManagement.uml delete mode 100644 pfsagentd/LockManagementWithExclusiveLease.uml delete mode 100644 pfsagentd/LockManagementWithSharedLease.uml delete mode 100644 pfsagentd/LockRequest.uml delete mode 100644 pfsagentd/Makefile delete mode 100644 pfsagentd/README.md delete mode 100644 pfsagentd/container/build/Dockerfile delete mode 100644 pfsagentd/container/insert/.gitignore delete mode 100755 pfsagentd/container/insert/MakePFSAgentDockerfile delete mode 100644 pfsagentd/container/insert/MakePFSAgentDockerfile.conf_SAMPLE delete mode 100644 pfsagentd/file_inode.go delete mode 100644 pfsagentd/fission.go delete mode 100644 pfsagentd/functional_test.go delete mode 100644 pfsagentd/globals.go delete mode 100644 pfsagentd/html_templates.go delete mode 100644 pfsagentd/http_server.go delete mode 100644 pfsagentd/jrpc.go delete mode 100644 pfsagentd/lease.go delete mode 100644 pfsagentd/log.go delete mode 100644 pfsagentd/main.go delete mode 100644 pfsagentd/pfsagent.conf delete mode 100644 pfsagentd/pfsagentd-init/Makefile delete mode 100644 pfsagentd/pfsagentd-init/README.md delete mode 100644 pfsagentd/pfsagentd-init/dummy_test.go delete mode 100644 pfsagentd/pfsagentd-init/main.go delete mode 100644 pfsagentd/pfsagentd-swift-auth-plugin/Makefile delete mode 100644 pfsagentd/pfsagentd-swift-auth-plugin/README.md delete mode 100644 pfsagentd/pfsagentd-swift-auth-plugin/dummy_test.go delete mode 100644 pfsagentd/pfsagentd-swift-auth-plugin/main.go delete mode 100644 pfsagentd/request.go delete mode 100644 pfsagentd/setup_teardown_test.go delete mode 100644 pfsagentd/swift_proxy_emulator_test.go delete mode 100644 pfsagentd/unit_test.go delete mode 100644 pfsconfjson/Makefile delete mode 100644 pfsconfjson/dummy_test.go delete mode 100644 pfsconfjson/main.go delete mode 100644 pfsconfjsonpacked/Makefile delete mode 100644 pfsconfjsonpacked/dummy_test.go delete mode 100644 pfsconfjsonpacked/main.go delete mode 100644 pfsworkout/Makefile delete mode 100644 pfsworkout/dummy_test.go delete mode 100644 pfsworkout/main.go delete mode 100644 proxyfsd/Makefile delete mode 100644 proxyfsd/cluster_1_peer.conf delete mode 100644 proxyfsd/cluster_3_peers.conf delete mode 100644 proxyfsd/daemon.go delete mode 100644 proxyfsd/daemon_test.go delete mode 100644 proxyfsd/debug.conf delete mode 100644 proxyfsd/default.conf delete mode 100644 proxyfsd/file_server.conf delete mode 100644 proxyfsd/file_server_mac_3_peers.conf delete mode 100644 proxyfsd/httpserver.conf delete mode 100644 proxyfsd/logging.conf delete mode 100644 proxyfsd/mac_cluster_1_peer.conf delete mode 100644 proxyfsd/mac_cluster_3_peers.conf delete mode 100644 proxyfsd/macproxyfsd0.conf delete mode 100644 proxyfsd/macproxyfsd1.conf delete mode 100644 proxyfsd/macproxyfsd2.conf delete mode 100644 proxyfsd/macproxyfsd3.conf delete mode 100644 proxyfsd/proxyfsd/Makefile delete mode 100644 proxyfsd/proxyfsd/dummy_test.go delete mode 100644 proxyfsd/proxyfsd/main.go delete mode 100644 proxyfsd/proxyfsd0.conf delete mode 100644 proxyfsd/proxyfsd1.conf delete mode 100644 proxyfsd/proxyfsd2.conf delete mode 100644 proxyfsd/proxyfsd3.conf delete mode 100644 proxyfsd/rpc_server.conf delete mode 100644 proxyfsd/saio_cluster_1_peer.conf delete mode 100644 proxyfsd/saio_logging.conf delete mode 100644 proxyfsd/saioproxyfsd0.conf delete mode 100644 proxyfsd/stats.conf delete mode 100644 proxyfsd/statslogger.conf delete mode 100644 proxyfsd/swift_client.conf delete mode 100644 proxyfsd/trackedlock.conf delete mode 100644 ramswift/Makefile delete mode 100644 ramswift/chaos_settings.conf delete mode 100644 ramswift/daemon.go delete mode 100644 ramswift/daemon_test.go delete mode 100644 ramswift/macramswift0.conf delete mode 100644 ramswift/ramswift/Makefile delete mode 100644 ramswift/ramswift/dummy_test.go delete mode 100644 ramswift/ramswift/main.go delete mode 100644 ramswift/ramswift0.conf delete mode 100644 ramswift/saioramswift0.conf delete mode 100644 ramswift/swift_info.conf delete mode 100644 readdir-stress/.gitignore delete mode 100644 readdir-stress/Makefile delete mode 100644 readdir-stress/readdir-stress.c delete mode 100644 saio/Vagrantfile delete mode 100755 saio/bin/flush_caches delete mode 100755 saio/bin/provision_middleware delete mode 100755 saio/bin/reset_etcd delete mode 100755 saio/bin/start_etcd delete mode 100755 saio/bin/start_proxyfs_and_swift delete mode 100755 saio/bin/start_swift_only delete mode 100755 saio/bin/stop_etcd delete mode 100755 saio/bin/unmount_and_stop_pfs delete mode 100644 saio/container/.gitignore delete mode 100644 saio/container/Dockerfile delete mode 100644 saio/container/Makefile delete mode 100644 saio/container/docker_startup.sh delete mode 100644 saio/container/etc/swift/account-server/1.conf delete mode 100644 saio/container/etc/swift/container-reconciler.conf delete mode 100644 saio/container/etc/swift/container-server/1.conf delete mode 100644 saio/container/etc/swift/container-sync-realms.conf delete mode 100644 saio/container/etc/swift/object-expirer.conf delete mode 100644 saio/container/etc/swift/object-server/1.conf delete mode 100644 saio/container/etc/swift/proxy-server.conf delete mode 100644 saio/container/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf delete mode 100644 saio/container/etc/swift/swift.conf delete mode 100644 saio/container/etc/swift/test.conf delete mode 100644 saio/container/proxyfs.conf delete mode 100644 saio/etc/exports delete mode 100644 saio/etc/samba/smb.conf delete mode 100644 saio/etc/swift/account-server/1.conf delete mode 100644 saio/etc/swift/account-server/2.conf delete mode 100644 saio/etc/swift/account-server/3.conf delete mode 100644 saio/etc/swift/account-server/4.conf delete mode 100644 saio/etc/swift/container-reconciler.conf delete mode 100644 saio/etc/swift/container-server/1.conf delete mode 100644 saio/etc/swift/container-server/2.conf delete mode 100644 saio/etc/swift/container-server/3.conf delete mode 100644 saio/etc/swift/container-server/4.conf delete mode 100644 saio/etc/swift/container-sync-realms.conf delete mode 100644 saio/etc/swift/object-expirer.conf delete mode 100644 saio/etc/swift/object-server/1.conf delete mode 100644 saio/etc/swift/object-server/2.conf delete mode 100644 saio/etc/swift/object-server/3.conf delete mode 100644 saio/etc/swift/object-server/4.conf delete mode 100644 saio/etc/swift/proxy-server.conf delete mode 100644 saio/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf delete mode 100644 saio/etc/swift/swift.conf delete mode 100644 saio/etc/swift/test.conf delete mode 100755 saio/home/swift/bin/remakerings delete mode 100755 saio/home/swift/bin/resetswift delete mode 100644 saio/proxyfs.conf delete mode 100644 saio/usr/lib/systemd/system/pfsagentd.service delete mode 100644 saio/usr/lib/systemd/system/proxyfsd.service delete mode 100644 saio/usr/local/go/src/runtime/runtime-gdb.py delete mode 100644 saio/vagrant_provision.sh delete mode 100644 sait/Vagrantfile delete mode 100755 sait/bin/provision_middleware delete mode 100755 sait/bin/start_proxyfs_and_swift delete mode 100755 sait/bin/start_swift_only delete mode 100755 sait/bin/unmount_and_stop_pfs delete mode 100644 sait/etc/swift/container-reconciler.conf delete mode 100644 sait/etc/swift/container-sync-realms.conf delete mode 100644 sait/etc/swift/object-expirer.conf delete mode 100644 sait/etc/swift/proxy-server.conf delete mode 100644 sait/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf delete mode 100644 sait/etc/swift/swift.conf delete mode 100755 sait/home/swift/bin/remakerings delete mode 100755 sait/home/swift/bin/resetswift delete mode 100644 sait/proxyfs.conf delete mode 100644 sait/sait1/etc/swift/account-server/1.conf delete mode 100644 sait/sait1/etc/swift/container-server/1.conf delete mode 100644 sait/sait1/etc/swift/object-server/1.conf delete mode 100644 sait/sait1/proxyfs.conf delete mode 100644 sait/sait1/usr/lib/systemd/system/proxyfsd.service delete mode 100644 sait/sait2/etc/swift/account-server/2.conf delete mode 100644 sait/sait2/etc/swift/container-server/2.conf delete mode 100644 sait/sait2/etc/swift/object-server/2.conf delete mode 100644 sait/sait2/proxyfs.conf delete mode 100644 sait/sait2/usr/lib/systemd/system/proxyfsd.service delete mode 100644 sait/sait3/etc/swift/account-server/3.conf delete mode 100644 sait/sait3/etc/swift/container-server/3.conf delete mode 100644 sait/sait3/etc/swift/object-server/3.conf delete mode 100644 sait/sait3/proxyfs.conf delete mode 100644 sait/sait3/usr/lib/systemd/system/proxyfsd.service delete mode 100644 sait/usr/local/go/src/runtime/runtime-gdb.py delete mode 100644 sait/vagrant_provision.sh delete mode 100644 samba-dc/Vagrantfile delete mode 100755 samba-dc/vagrant_provision.sh delete mode 100644 stats/Makefile delete mode 100644 stats/api.go delete mode 100644 stats/api_internal.go delete mode 100644 stats/api_test.go delete mode 100644 stats/config.go delete mode 100644 stats/sender.go delete mode 100644 stats/strings.go delete mode 100644 statslogger/Makefile delete mode 100644 statslogger/config.go delete mode 100644 statslogger/config_test.go delete mode 100644 statslogger/simplestats.go delete mode 100644 swiftclient/Makefile delete mode 100644 swiftclient/account.go delete mode 100644 swiftclient/api.go delete mode 100644 swiftclient/api_test.go delete mode 100644 swiftclient/config.go delete mode 100644 swiftclient/container.go delete mode 100644 swiftclient/http_status.go delete mode 100644 swiftclient/object.go delete mode 100644 swiftclient/retry.go delete mode 100644 swiftclient/utils.go delete mode 100644 swiftclient/utils_test.go delete mode 100644 utf/utf.go delete mode 100644 utf/utf_test.go delete mode 100644 win10/Vagrantfile diff --git a/cleanproxyfs/Makefile b/cleanproxyfs/Makefile deleted file mode 100644 index 381b9e6e..00000000 --- a/cleanproxyfs/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/cleanproxyfs - -include ../GoMakefile diff --git a/cleanproxyfs/dummy_test.go b/cleanproxyfs/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/cleanproxyfs/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/cleanproxyfs/main.go b/cleanproxyfs/main.go deleted file mode 100644 index fbcac855..00000000 --- a/cleanproxyfs/main.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// The cleanproxyfs program deletes the headhunter databases and deletes log -// segments from Swift, thereby creating a "clean slate" for continued testing or -// development. -package main - -import ( - "fmt" - "io/ioutil" - "net/http" - "os" - "strings" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/transitions" - "github.com/NVIDIA/proxyfs/utils" -) - -const ( - swiftAccountCheckpointHeaderName = "X-Account-Meta-Checkpoint" -) - -func main() { - cumulativeFailures := 0 - verbose := false - - args := os.Args[1:] - - // Check for verbose option in args[0] - if (0 < len(args)) && ("-v" == args[0]) { - verbose = true - args = args[1:] - } - - // Read in the program's args[0]-specified (and required) .conf file - if 0 == len(args) { - fmt.Fprint(os.Stderr, "no .conf file specified\n") - os.Exit(1) - } - - confMap, confErr := conf.MakeConfMapFromFile(args[0]) - if nil != confErr { - fmt.Fprintf(os.Stderr, "failed to load config: %v\n", confErr) - os.Exit(1) - } - - // Update confMap with any extra os.Args supplied - confErr = confMap.UpdateFromStrings(args[1:]) - if nil != confErr { - fmt.Fprintf(os.Stderr, "failed to load config overrides: %v\n", confErr) - os.Exit(1) - } - - // Upgrade confMap if necessary (remove when appropriate) - // Note that this follows UpdateFromStrings() - // hence overrides should be relative to the pre-upgraded .conf format - confErr = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != confErr { - fmt.Fprintf(os.Stderr, "failed to perform transitions.UpgradeConfMapIfNeeded(): %v\n", confErr) - os.Exit(1) - } - - // Fetch WhoAmI (qualifies which VolumeList elements are applicable) - whoAmI, confErr := confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != confErr { - fmt.Fprintf(os.Stderr, "confMap did not contain Cluster.WhoAmI\n") - os.Exit(1) - } - - // Compute activeVolumeNameList from each "active" VolumeGroup's VolumeList - activeVolumeNameList := []string{} - volumeGroupNameList, confErr := confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != confErr { - fmt.Fprintf(os.Stderr, "confMap did not contain FSGlobals.VolumeGroupList\n") - os.Exit(1) - } - for _, volumeGroupName := range volumeGroupNameList { - volumeGroupSectionName := "VolumeGroup:" + volumeGroupName - primaryPeerList, confErr := confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "PrimaryPeer") - if nil != confErr { - fmt.Fprintf(os.Stderr, "confMap did not contain %v.PrimaryPeer\n", volumeGroupSectionName) - os.Exit(1) - } - if 0 == len(primaryPeerList) { - continue - } else if 1 == len(primaryPeerList) { - if whoAmI == primaryPeerList[0] { - volumeNameList, confErr := confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "VolumeList") - if nil != confErr { - fmt.Fprintf(os.Stderr, "confMap did not contain %v.VolumeGroupList\n", volumeGroupSectionName) - os.Exit(1) - } - activeVolumeNameList = append(activeVolumeNameList, volumeNameList...) - } else { - fmt.Fprintf(os.Stderr, "confMap contained multiple values for %v.PrimaryPeer: %v\n", volumeGroupSectionName, primaryPeerList) - os.Exit(1) - } - } - } - - // Clean out referenced Swift Accounts - httpClient := &http.Client{} - - noAuthTCPPort, confErr := confMap.FetchOptionValueString("SwiftClient", "NoAuthTCPPort") - if nil != confErr { - fmt.Fprintf(os.Stderr, "confMap did not contain Swift.NoAuthTCPPort\n") - os.Exit(1) - } - - urlPrefix := "http://127.0.0.1:" + noAuthTCPPort + "/v1/" - - for _, volumeName := range activeVolumeNameList { - volumeSectionName := "Volume:" + volumeName - accountName, confErr := confMap.FetchOptionValueString(volumeSectionName, "AccountName") - if nil != confErr { - fmt.Fprintf(os.Stderr, "confMap did not contain %v.AccountName\n", volumeSectionName) - os.Exit(1) - } - - accountURL := urlPrefix + accountName - - if verbose { - fmt.Fprintf(os.Stdout, "about to clear out Volume %v @ Account %v\n", volumeName, accountURL) - } - - accountGetResponse, accountGetErr := httpClient.Get(accountURL) - if nil != accountGetErr { - fmt.Fprintf(os.Stderr, "httpClient.Get(%v) returned unexpected error: %v\n", accountURL, accountGetErr) - os.Exit(1) - } - - if http.StatusNoContent == accountGetResponse.StatusCode { - // Nothing to do here - } else if http.StatusOK == accountGetResponse.StatusCode { - accountGetResponseBodyByteSlice, accountGetResponseBodyReadAllErr := ioutil.ReadAll(accountGetResponse.Body) - if nil != accountGetResponseBodyReadAllErr { - fmt.Fprintf(os.Stderr, "ioutil.ReadAll(httpClient.Get(%v).Body)returned unexpected error: %v\n", accountURL, accountGetResponseBodyReadAllErr) - os.Exit(1) - } - accountGetResponseBodyString := utils.ByteSliceToString(accountGetResponseBodyByteSlice) - accountGetResponseBodyStringSlice := strings.Split(accountGetResponseBodyString, "\n") - - for _, containerName := range accountGetResponseBodyStringSlice[:len(accountGetResponseBodyStringSlice)-1] { - containerURL := accountURL + "/" + containerName - - if verbose { - fmt.Fprintf(os.Stdout, "about to clear out and remove %v\n", containerURL) - } - - containerGetResponse, containerGetErr := httpClient.Get(containerURL) - if nil != containerGetErr { - fmt.Fprintf(os.Stderr, "httpClient.Get(%v) returned unexpected error: %v\n", containerURL, containerGetErr) - os.Exit(1) - } - - if http.StatusNoContent == containerGetResponse.StatusCode { - // Nothing to do here - } else if http.StatusOK == containerGetResponse.StatusCode { - containerGetResponseBodyByteSlice, containerGetResponseBodyReadAllErr := ioutil.ReadAll(containerGetResponse.Body) - if nil != containerGetResponseBodyReadAllErr { - fmt.Fprintf(os.Stderr, "ioutil.ReadAll(httpClient.Get(%v).Body) returned unexpected error: %v\n", containerURL, containerGetResponseBodyReadAllErr) - os.Exit(1) - } - containerGetResponseBodyString := utils.ByteSliceToString(containerGetResponseBodyByteSlice) - containerGetResponseBodyStringSlice := strings.Split(containerGetResponseBodyString, "\n") - - for _, objectName := range containerGetResponseBodyStringSlice[:len(containerGetResponseBodyStringSlice)-1] { - objectURL := containerURL + "/" + objectName - - objectDeleteRequest, objectDeleteRequestErr := http.NewRequest("DELETE", objectURL, nil) - if nil != objectDeleteRequestErr { - fmt.Fprintf(os.Stderr, "http.NewRequest(\"DELETE\", %v, nil) returned unexpected error: %v\n", objectURL, objectDeleteRequestErr) - os.Exit(1) - } - objectDeleteResponse, objectDeleteResponseErr := httpClient.Do(objectDeleteRequest) - if nil != objectDeleteResponseErr { - fmt.Fprintf(os.Stderr, "httpClient.Do(DELETE %v) returned unexpected error: %v\n", objectURL, objectDeleteResponseErr) - os.Exit(1) - } - if http.StatusNoContent != objectDeleteResponse.StatusCode { - if verbose { - fmt.Fprintf(os.Stderr, "httpClient.Do(DELETE %v) returned unexpected StatusCode: %v\n", objectURL, objectDeleteResponse.StatusCode) - } - } - objectDeleteResponseBodyCloseErr := objectDeleteResponse.Body.Close() - if nil != objectDeleteResponseBodyCloseErr { - fmt.Fprintf(os.Stderr, "httpClient.Do(DELETE %v).Body.Close() returned unexpected error: %v\n", objectURL, objectDeleteResponseBodyCloseErr) - os.Exit(1) - } - } - } else { - if verbose { - fmt.Fprintf(os.Stderr, "httpClient.Get(%v) returned unexpected StatusCode: %v\n", containerURL, containerGetResponse.StatusCode) - } - } - - containerGetResponseBodyCloseErr := containerGetResponse.Body.Close() - if nil != containerGetResponseBodyCloseErr { - fmt.Fprintf(os.Stderr, "httpClient.Get(%v).Body.Close() returned unexpected error: %v\n", containerURL, containerGetResponseBodyCloseErr) - os.Exit(1) - } - - containerDeleteRequest, containerDeleteRequestErr := http.NewRequest("DELETE", containerURL, nil) - if nil != containerDeleteRequestErr { - fmt.Fprintf(os.Stderr, "http.NewRequest(\"DELETE\", %v, nil) returned unexpected error: %v\n", containerURL, containerDeleteRequestErr) - os.Exit(1) - } - containerDeleteResponse, containerDeleteResponseErr := httpClient.Do(containerDeleteRequest) - if nil != containerDeleteResponseErr { - fmt.Fprintf(os.Stderr, "httpClient.Do(DELETE %v) returned unexpected error: %v\n", containerURL, containerDeleteResponseErr) - os.Exit(1) - } - switch containerDeleteResponse.StatusCode { - case http.StatusNoContent: - // Nothing to do here - case http.StatusConflict: - if verbose { - fmt.Fprintf(os.Stderr, "%v still not empty - re-run required\n", containerURL) - } - cumulativeFailures++ - default: - if verbose { - fmt.Fprintf(os.Stderr, "httpClient.Do(DELETE %v) returned unexpected StatusCode: %v\n", containerURL, containerDeleteResponse.StatusCode) - } - } - containerDeleteResponseBodyCloseErr := containerDeleteResponse.Body.Close() - if nil != containerDeleteResponseBodyCloseErr { - fmt.Fprintf(os.Stderr, "httpClient.Do(DELETE %v).Body.Close() returned unexpected error: %v\n", containerURL, containerDeleteResponseBodyCloseErr) - os.Exit(1) - } - - replayLogFileName, confErr := confMap.FetchOptionValueString(volumeSectionName, "ReplayLogFileName") - if nil == confErr { - if "" != replayLogFileName { - removeReplayLogFileErr := os.Remove(replayLogFileName) - if nil != removeReplayLogFileErr { - if !os.IsNotExist(removeReplayLogFileErr) { - fmt.Fprintf(os.Stderr, "os.Remove(replayLogFileName == \"%v\") returned unexpected error: %v\n", replayLogFileName, removeReplayLogFileErr) - os.Exit(1) - } - } - } - } - } - } else { - if verbose { - fmt.Fprintf(os.Stderr, "httpClient.Get(%v) returned unexpected StatusCode: %v\n", accountURL, accountGetResponse.StatusCode) - } - } - - accountGetResponseBodyCloseErr := accountGetResponse.Body.Close() - if nil != accountGetResponseBodyCloseErr { - fmt.Fprintf(os.Stderr, "httpClient.Get(%v).Body.Close() returned unexpected error: %v\n", accountURL, accountGetResponseBodyCloseErr) - os.Exit(1) - } - } - - // Finally, remove Logging.LogFilePath (if any) - logFilePath, confErr := confMap.FetchOptionValueString("Logging", "LogFilePath") - if nil == confErr { - if verbose { - fmt.Fprintf(os.Stdout, "about to clear out %v\n", logFilePath) - } - - osRemoveAllErr := os.RemoveAll(logFilePath) - if nil != osRemoveAllErr { - fmt.Fprintf(os.Stderr, "os.RemoveAll(logFilePath) failed: %v\n", osRemoveAllErr) - os.Exit(cumulativeFailures + 1) - } - } - - os.Exit(cumulativeFailures) -} diff --git a/confgen/Makefile b/confgen/Makefile deleted file mode 100644 index d6391127..00000000 --- a/confgen/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/confgen - -include ../GoMakefile diff --git a/confgen/api.go b/confgen/api.go deleted file mode 100644 index 01d48553..00000000 --- a/confgen/api.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package confgen provides a mechanism by which a supplied configuration is converted into a -// set of configuration files to be supplied to ProxyFS (proxyfsd), Samba (smbd et. al.), and -// NFS (nfsd). In (at least) the case of Samba, multiple configuration files will be produced -// with a per-IP path. -// -// Only the configuration files for the Cluster.WhoAmI-identified Peer will be produced though, -// in the case of the ProxyFS configuration file, all VolumeGroups will be included given that -// ProxyFS needs to know the mapping of VolumeGroup to Peer. This allows one ProxyFS instance -// to redirect a JSONRPC client to the appropriate ProxyFS instance servicing the refernected -// Volume. The Cluster.WhoAmI can either already be present in the `confFilePath` or be provided -// as an element of the `confOverrides` argument in each of the APIs below. -// -// As would be typical in a deployment where the placement of VolumeGroups on Peers in dynamic, -// the `confFilePath` would provide `VirtualIPAddr` values as opposed to fixed `PrimaryPeer` -// assignments. It is the callers responsibility to compute the assignments of VolumeGroups -// to Peers and either modify the provided `confFilePath` to include PrimaryPeer values or to -// supply those via the `confOverrides` (e.g. "VolumeGroup:CommonVolumeGroup.PrimaryPeer=Peer0"). -package confgen - -// EnvMap allows the caller to provide environment-specific paths for various programs invoked -// by this API as well as customize the scripts produced by this API. -type EnvMap map[string]string - -const ( - // LinuxUserCommentDefault is the default value of environment variable LinuxUserCommentEnv. - LinuxUserCommentDefault = "user-created-for-samba" - // LinuxUserCommentEnv specifies a comment to be applied to each Linux user created - // to be referenced by the SMB user system as provided by SAMBA(7). - LinuxUserCommentEnv = "LINUX_USER_COMMENT" - - // PathToNetDefault is the default value of environment variable PathToNetEnv. - PathToNetDefault = "/usr/bin/net" - // PathToNetEnv is the name of the environment variable used to specify the path to the - // NET(8) tool used to administer a SAMBA(7) installation. - PathToNetEnv = "PATH_TO_NET" - - // PathToKRB5ConfDirDefault is the default value of environment variable PathToKRB5ConfDirEnv. - PathToKRB5ConfDirDefault = "/etc/krb5.conf.d" - // PathToKRB5ConfDirEnv is the name of the environment variable used to specify the path to - // the KRB5 configuration directory where realm declarations are placed. This method is used - // to avoid having to merge all realm declarations into a common KRB5 configuration file. - PathToKRB5ConfDirEnv = "PATH_TO_KRB5_CONF_DIR" - - // PathToPDBEditDefault is the default value of environment variable PathToPDBEditEnv. - PathToPDBEditDefault = "/usr/bin/pdbedit" - // PathToPDBEditEnv is the name of the environment variable used to specify the path to - // the PDBEDIT(8) tool used to administer a SAMBA(7) installation. - PathToPDBEditEnv = "PATH_TO_PDBEDIT" - - // PathToPerVirtualIPAddrDirDefault is the default value of environment variable PathToPerVirtualIPAddrDirEnv. - PathToPerVirtualIPAddrDirDefault = "/var/lib/vips" - // PathToPerVirtualIPAddrDirEnv is the name of the environment variable used to specify the - // path to a directory containing a set of subdirectories each named for the corresponding - // VirtualIPAddr to which they pertain. In particular, each such subdirectory will contain - // a "samba" subdirectory containing all files referenced by that VirtualIPAddr's instance - // of SAMBA(7). - PathToPerVirtualIPAddrDirEnv = "PATH_TO_PER_VIRTUAL_IP_ADDR_DIR" - - // PathToSMBDDefault is the default value of environment variable PathToSMBDEnv. - PathToSMBDDefault = "/usr/bin/smbd" - // PathToSMBDEnv is the name of the environment variable used to specify the path to the - // SMBD(8) program in a SAMBA(7) installation used to provide SMB file serving to clients. - PathToSMBDEnv = "PATH_TO_SMBD" - - // PathToSMBPasswdDefault is the default value of environment variable PathToSMBPasswdEnv. - PathToSMBPasswdDefault = "/usr/bin/smbpasswd" - // PathToSMBPasswdEnv is the name of the environment variable used to specify the path to - // the SMBPASSWD(8) tool used to add an SMB user or update an SMB user's password in a - // SAMBA(7) installation. - PathToSMBPasswdEnv = "PATH_TO_SMBPASSWD" -) - -// ComputeInitial takes a supplied ConfFile, overlays ConfOverrides, and computes an initial -// set of configuration files that are used by a per-IPAddr set of Samba instances as well as -// NFSd & ProxyFS. -func ComputeInitial(envMap EnvMap, confFilePath string, confOverrides []string, initialDirPath string) (err error) { - err = computeInitial(envMap, confFilePath, confOverrides, initialDirPath) - return -} - -// ComputePhases takes a supplied initial set of conf files (such as produced by ComputeInitial() -// above) along with a new ConfFile and new ConfOverrides and produces two sets of conf files -// used in a 2-phase migration from the initial config to a new config. Presumably, the 2nd phase -// produced will be used as the initial config in the next call to ComputePhases(). -func ComputePhases(envMap EnvMap, initialDirPath string, confFilePath string, confOverrides []string, phaseOneDirPath string, phaseTwoDirPath string) (err error) { - err = computePhases(envMap, initialDirPath, confFilePath, confOverrides, phaseOneDirPath, phaseTwoDirPath) - return -} diff --git a/confgen/api_internal.go b/confgen/api_internal.go deleted file mode 100644 index 2a635fa5..00000000 --- a/confgen/api_internal.go +++ /dev/null @@ -1,1089 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package confgen - -import ( - "fmt" - "io/ioutil" - "net" - "os" - "strconv" - "strings" - - "github.com/NVIDIA/proxyfs/conf" -) - -type envSettingsStruct struct { - linuxUserComment string - pathToNet string - pathToKRB5ConfDir string - pathToPDBEdit string - pathToPerVirtualIPAddrDir string - pathToSMBD string - pathToSMBPasswd string -} - -const ( - useraddPath = "/usr/sbin/useradd" - userdelPath = "/usr/sbin/userdel" - volumeDummyPath = "/opt/ss/var/lib/dummy_path" -) - -const ( - exportsFileName = "exports" // Used to identify NFS exports - fuseSetupFileName = "fuse_setup.bash" // Used to create FUSE export directories - proxyFSConfFileName = "proxyfs.conf" // Used to pass to mkproxyfs & proxyfsd - realmsSourceDirName = "realms" // Used to hold files destined for realmsDestinationDirName - smbConfFileName = "smb.conf" // Used for passing to various SAMBA(7) components - smbCommonConfFileName = "smb_common.conf" // Used for generating a smbPassdbFileName cloned for each vips/{VirtualIPAddr} - smbPassdbFileName = "passdb.tdb" // Used in vips/{VirtualIPAddr} to hold "local" SMB Passwords - smbUsersSetupFileName = "smb_users_setup.bash" // Used to {create|update|destroy} SMB & Linux users - vipsDirName = "vips" // Used to hold a set of VirtualIPAddr-named subdirectories - proxyfsConfDir0 = "/usr/lib/proxyfsd" // proxyfs conf files (includingtemplates directory) are here - proxyfsConfDir1 = "/opt/ss/lib/proxyfsd" // or here - proxyfsConfDir2 = "./templates" // or relative to the current directory! - // where each holds files specific to that VirtualIPAddr -) - -const ( - confDirPerm = os.FileMode(0777) // Let umask "restrict" this as desired - confFilePerm = os.FileMode(0666) // Let umask "restrict" this as desired - exportsFilePerm = os.FileMode(0666) // Let umask "restrict" this as desired - fuseDirPerm = os.FileMode(0000) // Fail all non-root access to missing FUSE exports - scriptPerm = os.FileMode(0777) // Let umask "restrict" this as desired - smbConfPerm = os.FileMode(0644) // Let umask "restrict" this as desired -) - -type smbUserMap map[string]string // Key=SMBUserName; Value=SMBUserPassword decoded via base64.StdEncoding.DecodeString() - -type uint64Set map[uint64]struct{} -type stringSet map[string]struct{} - -const ( - nfsSubtreeCheck = "no_subtree_check" // Force this for every NFS Export - nfsSync = "sync" // Force this for every NFS Export -) - -type NFSClient struct { - clientName string - ClientPattern string - AccessMode string - RootSquash string - Secure string -} - -type NFSClientList []*NFSClient -type NFSClientMap map[string]*NFSClient // Key=NFSClient.clientName - -type Volume struct { - VolumeName string // Must be unique - VolumeGroup *VolumeGroup // - FSID uint64 // Must be unique - FUSEMountPointName string // Must be unique unless == "" (no FUSE mount point...and cannot be NFS exported) - nfsClientList NFSClientList // Must be empty (no NFS Export) if FUSEMountPointName == "" - nfsClientMap NFSClientMap // Must be empty (no NFS Export) if FUSEMountPointName == "" - AccountName string // Must be unique - SMB SMBVolume -} - -type volumeMap map[string]*Volume // Key=volume.volumeName - -// SMBVolume contains per volume SMB settings -type SMBVolume struct { - AuditLogging bool - Browseable bool - EncryptionRequired bool - Path string - ShareName string // Must be unique unless == "" (no SMB Share) - StrictSync bool - ValidADGroup []string - ValidADUsers []string - ValidUsers []string -} - -// SMBVG contains per Volume Group SMB settings -type SMBVG struct { - ADBackEnd string - ADEnabled bool - ADIDMapDefaultMin int - ADIDMapDefaultMax int - ADIDMapWorkgroupMin int - ADIDMapWorkgroupMax int - ADIDMgmt bool - ADIDSchema string - AuditLogging bool // True if any volume in volume group has it enabled - BrowserAnnounce string - FastTCPPort int - MapToGuest string - ADRealm string - RPCServerLSARPC string - Security string - ServerMinProtocol string - TCPPort int - WorkGroup string -} - -// VolumeGroup contains VolumeGroup conf settings -type VolumeGroup struct { - PrimaryPeer string // - SMB SMBVG // SMB specific settings of the VG - VirtualHostName string // Must be unique - VirtualIPAddr net.IP // Must be unique - VirtualIPMask *net.IPNet // not necessarily unique - VolumeGroupName string // Must be unique - VolumeMap volumeMap // -} - -type volumeGroupMap map[string]*VolumeGroup // Key=VolumeGroup.volumeGroupName - -func computeInitial(envMap EnvMap, confFilePath string, confOverrides []string, initialDirPath string) (err error) { - var ( - envSettings *envSettingsStruct - exportsFile *os.File - fuseSetupFile *os.File - initialConfMap conf.ConfMap - localSMBVolumeGroupMap volumeGroupMap - localVolumeMap volumeMap - nfsClient *NFSClient - smbUsersSetupFile *os.File - toCreateSMBUserMap smbUserMap - toCreateSMBUserName string - toCreateSMBUserPassword string - toCreateSMBUserPasswordEscaped string // == strings.ReplaceAll(toCreateSMBUserPassword, "\\", "\\\\") - volume *Volume - ) - - // Fetch environ settings - - envSettings = fetchEnvironSettings(envMap) - - // Load supplied config with overrides - - initialConfMap, err = conf.MakeConfMapFromFile(confFilePath) - if nil != err { - return - } - - err = initialConfMap.UpdateFromStrings(confOverrides) - if nil != err { - return - } - - // Store Initial Config - - err = os.Mkdir(initialDirPath, confDirPerm) - if nil != err { - return - } - - err = ioutil.WriteFile(initialDirPath+"/"+proxyFSConfFileName, []byte(initialConfMap.Dump()), confFilePerm) - if nil != err { - return - } - - // Fetch pertinent data from Initial Config - - _, localSMBVolumeGroupMap, localVolumeMap, _, _, err = fetchVolumeInfo(initialConfMap) - if nil != err { - return - } - - _, toCreateSMBUserMap, _, _, err = computeSMBUserListChange(make(smbUserMap), initialConfMap) - if nil != err { - return - } - - // Compute common exports file - NFSd will (unfortunately) serve ALL VirtualIPAddrs - - exportsFile, err = os.OpenFile(initialDirPath+"/"+exportsFileName, os.O_CREATE|os.O_WRONLY, exportsFilePerm) - if nil != err { - return - } - - for _, volume = range localVolumeMap { - if 0 < len(volume.nfsClientList) { - _, err = exportsFile.WriteString(fmt.Sprintf("\"%s\"", volume.FUSEMountPointName)) - if nil != err { - return - } - for _, nfsClient = range volume.nfsClientList { - _, err = exportsFile.WriteString(fmt.Sprintf(" %s(%s,%s,fsid=%d,%s,%s,%s)", nfsClient.ClientPattern, nfsClient.AccessMode, nfsSync, volume.FSID, nfsClient.RootSquash, nfsClient.Secure, nfsSubtreeCheck)) - if nil != err { - return - } - } - _, err = exportsFile.WriteString("\n") - if nil != err { - return - } - } - } - - err = exportsFile.Close() - if nil != err { - return - } - - // Compute SMB Users script - - smbUsersSetupFile, err = os.OpenFile(initialDirPath+"/"+smbUsersSetupFileName, os.O_CREATE|os.O_WRONLY, scriptPerm) - if nil != err { - return - } - - _, err = smbUsersSetupFile.WriteString("#!/bin/bash\n") - if nil != err { - return - } - _, err = smbUsersSetupFile.WriteString("set -e\n") - if nil != err { - return - } - - for toCreateSMBUserName, toCreateSMBUserPassword = range toCreateSMBUserMap { - _, err = smbUsersSetupFile.WriteString(fmt.Sprintf("%s --comment %s --no-create-home %s\n", useraddPath, envSettings.linuxUserComment, toCreateSMBUserName)) - if nil != err { - return - } - - // TODO: Replace following line once Golang 1.12 (sporting new strings.ReplaceAll() func) is required with: - // - // toCreateSMBUserPasswordEscaped = strings.ReplaceAll(toCreateSMBUserPassword, "\\", "\\\\\\") - // - toCreateSMBUserPasswordEscaped = strings.Replace(toCreateSMBUserPassword, "\\", "\\\\\\", -1) - - _, err = smbUsersSetupFile.WriteString(fmt.Sprintf("echo -e \"%s\\n%s\" | %s -c %s -a %s\n", toCreateSMBUserPasswordEscaped, toCreateSMBUserPasswordEscaped, envSettings.pathToSMBPasswd, smbConfFileName, toCreateSMBUserName)) - if nil != err { - return - } - } - - err = smbUsersSetupFile.Close() - if nil != err { - return - } - - // Compute per-VitualIPAddr (Samba) - - err = os.Mkdir(initialDirPath+"/"+vipsDirName, confDirPerm) - if nil != err { - return - } - - // Create per VG smb.conf files - err = createSMBConf(initialDirPath, localSMBVolumeGroupMap) - if nil != err { - // TODO - logging - return - } - - // Compute FUSE MountPoint Directory script - - fuseSetupFile, err = os.OpenFile(initialDirPath+"/"+fuseSetupFileName, os.O_CREATE|os.O_WRONLY, scriptPerm) - if nil != err { - return - } - - _, err = fuseSetupFile.WriteString("#!/bin/bash\n") - if nil != err { - return - } - _, err = fuseSetupFile.WriteString("set -e\n") - if nil != err { - return - } - - for _, volume = range localVolumeMap { - if "" != volume.FUSEMountPointName { - // if the directory exists and the file system is mounted but - // proxyfs is not running, the '-d' test will return false and the - // mkdir will fail with EEXIST, so append "|| true" to the mkdir - // so the script won't fail - cmdString := "" - cmdString += fmt.Sprintf("if [ ! -d '%s' ]; then\n", volume.FUSEMountPointName) - cmdString += fmt.Sprintf(" mkdir -p -m 0%03o '%s' || true\n", - fuseDirPerm, volume.FUSEMountPointName) - cmdString += fmt.Sprintf("fi\n") - - _, err = fuseSetupFile.WriteString(cmdString) - if nil != err { - return - } - } - } - - err = fuseSetupFile.Close() - if nil != err { - return - } - - return nil // TODO -} - -func computePhases(envMap EnvMap, initialDirPath string, confFilePath string, confOverrides []string, phaseOneDirPath string, phaseTwoDirPath string) (err error) { - var ( - initialConfMap conf.ConfMap - phaseOneConfMap conf.ConfMap - phaseTwoConfMap conf.ConfMap - ) - - // Load config from initialDirPath - - initialConfMap, err = conf.MakeConfMapFromFile(initialDirPath + "/" + proxyFSConfFileName) - if nil != err { - return - } - - // Load supplied config with overrides that will be used for Phase Two - - phaseTwoConfMap, err = conf.MakeConfMapFromFile(confFilePath) - if nil != err { - return - } - - err = phaseTwoConfMap.UpdateFromStrings(confOverrides) - if nil != err { - return - } - - // Compute config that will be used for Phase One - - phaseOneConfMap = initialConfMap // TODO: for now, just use this one - - // Store Phase One Config - - err = os.Mkdir(phaseOneDirPath, confDirPerm) - if nil != err { - return - } - - err = ioutil.WriteFile(phaseOneDirPath+"/"+proxyFSConfFileName, []byte(phaseOneConfMap.Dump()), confFilePerm) - if nil != err { - return - } - - // Store Phase Two Config - - err = os.Mkdir(phaseTwoDirPath, confDirPerm) - if nil != err { - return - } - - err = ioutil.WriteFile(phaseTwoDirPath+"/"+proxyFSConfFileName, []byte(phaseTwoConfMap.Dump()), confFilePerm) - if nil != err { - return - } - - // Compute SMB Users - // computeSMBUserListChange(oldSMBUserMap smbUserMap, newConfMap conf.ConfMap) - // (newSMBUserMap smbUserMap, toCreateSMBUserMap smbUserMap, toUpdateSMBUserMap smbUserMap, toDeleteSMBUserMap smbUserMap, err error) - - return nil // TODO -} - -func fetchEnvironSettings(envMap EnvMap) (envSettings *envSettingsStruct) { - var ( - inEnv bool - ) - - envSettings = &envSettingsStruct{} - - envSettings.linuxUserComment, inEnv = envMap[LinuxUserCommentEnv] - if !inEnv { - envSettings.linuxUserComment = LinuxUserCommentDefault - } - - envSettings.pathToNet, inEnv = envMap[PathToNetEnv] - if !inEnv { - envSettings.pathToNet = PathToNetDefault - } - - envSettings.pathToKRB5ConfDir, inEnv = envMap[PathToKRB5ConfDirEnv] - if !inEnv { - envSettings.pathToKRB5ConfDir = PathToKRB5ConfDirDefault - } - - envSettings.pathToPDBEdit, inEnv = envMap[PathToPDBEditEnv] - if !inEnv { - envSettings.pathToPDBEdit = PathToPDBEditDefault - } - - envSettings.pathToPerVirtualIPAddrDir, inEnv = envMap[PathToPerVirtualIPAddrDirEnv] - if !inEnv { - envSettings.pathToPerVirtualIPAddrDir = PathToPerVirtualIPAddrDirDefault - } - - envSettings.pathToSMBD, inEnv = envMap[PathToSMBDEnv] - if !inEnv { - envSettings.pathToSMBD = PathToSMBDDefault - } - - envSettings.pathToSMBPasswd, inEnv = envMap[PathToSMBPasswdEnv] - if !inEnv { - envSettings.pathToSMBPasswd = PathToSMBPasswdDefault - } - - return -} - -func computeSMBUserListChange(oldSMBUserMap smbUserMap, newConfMap conf.ConfMap) (newSMBUserMap smbUserMap, toCreateSMBUserMap smbUserMap, toUpdateSMBUserMap smbUserMap, toDeleteSMBUserMap smbUserMap, err error) { - var ( - inNewSMBUserMap bool - inOldSMBUserMap bool - newSMBUserName string - newSMBUserNameList []string - newSMBUserPassword string - oldSMBUserName string - ) - - newSMBUserMap = make(smbUserMap) - toCreateSMBUserMap = make(smbUserMap) - toUpdateSMBUserMap = make(smbUserMap) - toDeleteSMBUserMap = make(smbUserMap) - - newSMBUserNameList, err = newConfMap.FetchOptionValueStringSlice("FSGlobals", "SMBUserList") - if nil != err { - newSMBUserNameList = make([]string, 0) - } - - for _, newSMBUserName = range newSMBUserNameList { - newSMBUserPassword, err = newConfMap.FetchOptionValueBase64String("SMBUsers", newSMBUserName) - if nil != err { - return - } - newSMBUserMap[newSMBUserName] = newSMBUserPassword - _, inOldSMBUserMap = oldSMBUserMap[newSMBUserName] - if inOldSMBUserMap { - toUpdateSMBUserMap[newSMBUserName] = newSMBUserPassword - } else { - toCreateSMBUserMap[newSMBUserName] = newSMBUserPassword - } - } - - for oldSMBUserName = range oldSMBUserMap { - _, inNewSMBUserMap = newSMBUserMap[oldSMBUserName] - if !inNewSMBUserMap { - toDeleteSMBUserMap[oldSMBUserName] = "" - } - } - - return -} - -func computeVolumeSetChange(oldConfMap conf.ConfMap, newConfMap conf.ConfMap) (toDeleteVolumeMap volumeMap, toCreateVolumeMap volumeMap, err error) { - var ( - newLocalVolumeMap volumeMap - newVolume *Volume - ok bool - oldLocalVolumeMap volumeMap - oldVolume *Volume - volumeName string - ) - - _, _, oldLocalVolumeMap, _, _, err = fetchVolumeInfo(oldConfMap) - if nil != err { - err = fmt.Errorf("In oldConfMap: %v", err) - return - } - _, _, newLocalVolumeMap, _, _, err = fetchVolumeInfo(newConfMap) - if nil != err { - err = fmt.Errorf("In newConfMap: %v", err) - return - } - - toDeleteVolumeMap = make(volumeMap) - - for volumeName, oldVolume = range oldLocalVolumeMap { - newVolume, ok = newLocalVolumeMap[volumeName] - if !ok || (oldVolume.VolumeGroup.VolumeGroupName != newVolume.VolumeGroup.VolumeGroupName) { - toDeleteVolumeMap[volumeName] = oldVolume - } - } - - toCreateVolumeMap = make(volumeMap) - - for volumeName, newVolume = range newLocalVolumeMap { - oldVolume, ok = oldLocalVolumeMap[volumeName] - if !ok || (oldVolume.VolumeGroup.VolumeGroupName != newVolume.VolumeGroup.VolumeGroupName) { - toCreateVolumeMap[volumeName] = newVolume - } - } - - return -} - -// fetchStringSet fetches 0 or 1 elements from a slice stored in a ConfMap and returns an error if -// the slice has more than 1 element. -func fetchStringSet(confMap conf.ConfMap, section string, value string, valueSet stringSet) (data string, err error) { - var ( - valueAsSlice []string - ok bool - ) - - valueAsSlice, err = confMap.FetchOptionValueStringSlice(section, value) - if err != nil { - return - } - - if 0 == len(valueAsSlice) { - data = "" - return - } - if 1 == len(valueAsSlice) { - data = valueAsSlice[0] - - // valueSet nil means caller is not interested in the value being unique - if nil != valueSet { - _, ok = valueSet[data] - if ok { - err = fmt.Errorf("Found duplicate [%s]%s (\"%s\")", section, value, data) - return - } - - valueSet[data] = struct{}{} - } - } else { - err = fmt.Errorf("Found multiple values for [%s]%s", section, value) - return - } - - return -} - -// populateVolumeSMB populates the Volume with SMB related info -func populateVolumeSMB(confMap conf.ConfMap, volumeSection string, volume *Volume, shareNameSet stringSet) (err error) { - - volume.SMB.AuditLogging, err = confMap.FetchOptionValueBool(volumeSection, "SMBAuditLogging") - if nil != err { - return - } - volume.SMB.Browseable, err = confMap.FetchOptionValueBool(volumeSection, "SMBBrowseable") - if nil != err { - return - } - - volume.SMB.EncryptionRequired, err = confMap.FetchOptionValueBool(volumeSection, "SMBEncryptionRequired") - if nil != err { - return - } - - volume.SMB.Path = volumeDummyPath - - volume.SMB.ShareName, err = fetchStringSet(confMap, volumeSection, "SMBShareName", shareNameSet) - if nil != err { - return - } - - volume.SMB.StrictSync, err = confMap.FetchOptionValueBool(volumeSection, "SMBStrictSync") - if nil != err { - return - } - - volume.SMB.ValidADGroup, err = confMap.FetchOptionValueStringSlice(volumeSection, "SMBValidADGroupList") - if nil != err { - return - } - - volume.SMB.ValidADUsers, err = confMap.FetchOptionValueStringSlice(volumeSection, "SMBValidADUserList") - if nil != err { - return - } - - volume.SMB.ValidUsers, err = confMap.FetchOptionValueStringSlice(volumeSection, "SMBValidUserList") - if nil != err { - return - } - - // TODO - figure out the ValidUserList, SMBMapToGuest, SMBNetBiosName, SMBUserList - // Make consistent with controller, etc - /* - "SMBValidUserList": [ - "InN3aWZ0IiwgImJvYiIsICJlZCIsICJibGFrZSI=" - ], - */ - - return -} - -// populateVolumeGroupSMB populates the VolumeGroup with SMB related -// info -func populateVolumeGroupSMB(confMap conf.ConfMap, volumeGroupSection string, tcpPort int, fastTCPPort int, volumeGroup *VolumeGroup, - workGroupSet stringSet) (err error) { - var ( - idUint32 uint32 - mapToGuest []string - ) - volumeGroup.SMB.TCPPort = tcpPort - volumeGroup.SMB.FastTCPPort = fastTCPPort - - // We do not verify that the backend is unique - volumeGroup.SMB.ADBackEnd, err = fetchStringSet(confMap, volumeGroupSection, "SMBADBackend", nil) - if nil != err { - return - } - - volumeGroup.SMB.ADIDMgmt, err = confMap.FetchOptionValueBool(volumeGroupSection, "SMBADIDMgmt") - if nil != err { - return - } - - // We do not verify that the schema is unique - volumeGroup.SMB.ADIDSchema, err = fetchStringSet(confMap, volumeGroupSection, "SMBADIDSchema", nil) - if nil != err { - return - } - - volumeGroup.SMB.ADEnabled, err = confMap.FetchOptionValueBool(volumeGroupSection, "SMBActiveDirectoryEnabled") - if nil != err { - return - } - - idUint32, err = confMap.FetchOptionValueUint32(volumeGroupSection, "SMBActiveDirectoryIDMapDefaultMin") - volumeGroup.SMB.ADIDMapDefaultMin = int(idUint32) - if nil != err { - return - } - idUint32, err = confMap.FetchOptionValueUint32(volumeGroupSection, "SMBActiveDirectoryIDMapDefaultMax") - volumeGroup.SMB.ADIDMapDefaultMax = int(idUint32) - if nil != err { - return - } - - idUint32, err = confMap.FetchOptionValueUint32(volumeGroupSection, "SMBActiveDirectoryIDMapWorkgroupMin") - volumeGroup.SMB.ADIDMapWorkgroupMin = int(idUint32) - if nil != err { - return - } - idUint32, err = confMap.FetchOptionValueUint32(volumeGroupSection, "SMBActiveDirectoryIDMapWorkgroupMax") - volumeGroup.SMB.ADIDMapWorkgroupMax = int(idUint32) - if nil != err { - return - } - - // We do not verify that the realm is unique - volumeGroup.SMB.ADRealm, err = fetchStringSet(confMap, volumeGroupSection, "SMBActiveDirectoryRealm", nil) - if nil != err { - return - } - - // We do not verify that browser announce is unique - volumeGroup.SMB.BrowserAnnounce, err = fetchStringSet(confMap, volumeGroupSection, "SMBBrowserAnnounce", nil) - if nil != err { - return - } - - mapToGuest, err = confMap.FetchOptionValueStringSlice(volumeGroupSection, "SMBMapToGuest") - if nil != err { - return - } - - // No easy way to pass string with " " between words. Join the elements and store. - volumeGroup.SMB.MapToGuest = strings.Join(mapToGuest, " ") - - volumeGroup.SMB.RPCServerLSARPC, err = fetchStringSet(confMap, volumeGroupSection, "SMBRPCServerLSARPC", nil) - if nil != err { - return - } - - volumeGroup.SMB.Security, err = fetchStringSet(confMap, volumeGroupSection, "SMBSecurity", nil) - if nil != err { - return - } - - volumeGroup.SMB.ServerMinProtocol, err = fetchStringSet(confMap, volumeGroupSection, "SMBServerMinProtocol", nil) - if nil != err { - return - } - - volumeGroup.SMB.WorkGroup, err = fetchStringSet(confMap, volumeGroupSection, "SMBWorkgroup", workGroupSet) - if nil != err { - return - } - - return -} - -// populateVolumeGroup is a helper function to populate the volume group information -func populateVolumeGroup(confMap conf.ConfMap, globalVolumeGroupMap volumeGroupMap, - volumeGroupList []string) (err error) { - var ( - accountNameSet stringSet - fastTCPPort int - fsidSet uint64Set - fuseMountPointNameSet stringSet - nfsClient *NFSClient - nfsClientSection string - nfsExportClientMapList []string - nfsExportClientMapListElement string - nfsExportClientMapSet stringSet - ok bool - shareNameSet stringSet - shared bool - tcpPort int - virtualHostNameSet stringSet - virtualIPAddrSet stringSet - virtualIPAddr string - volume *Volume - volumeGroup *VolumeGroup - volumeGroupName string - volumeGroupSection string - volumeList []string - volumeName string - volumeNameSet stringSet - volumeSection string - workGroupSet stringSet - ) - - // Fetch tcpPort and fastTCPPort number from config file. - // - // We store this in each SMBVG since the per volume group template used for generating - // smb.conf files needs it. - portString, confErr := confMap.FetchOptionValueString("JSONRPCServer", "TCPPort") - if nil != confErr { - err = fmt.Errorf("failed to get JSONRPCServer.TCPPort from config file") - return - } - tcpPort, err = strconv.Atoi(portString) - if nil != err { - return - } - fastPortString, confErr := confMap.FetchOptionValueString("JSONRPCServer", "FastTCPPort") - if nil != confErr { - err = fmt.Errorf("failed to get JSONRPCServer.TCPFastPort from config file") - return - } - fastTCPPort, err = strconv.Atoi(fastPortString) - if nil != err { - return - } - - accountNameSet = make(stringSet) - fsidSet = make(uint64Set) - fuseMountPointNameSet = make(stringSet) - shareNameSet = make(stringSet) - virtualHostNameSet = make(stringSet) - virtualIPAddrSet = make(stringSet) - volumeNameSet = make(stringSet) - workGroupSet = make(stringSet) - - for _, volumeGroupName = range volumeGroupList { - var ( - haveVolumeWithAuditLogging bool - ) - _, ok = globalVolumeGroupMap[volumeGroupName] - if ok { - err = fmt.Errorf("Found duplicate volumeGroupName (\"%s\") in [FSGlobals]VolumeGroupList", volumeGroupName) - return - } - - volumeGroup = &VolumeGroup{VolumeGroupName: volumeGroupName, VolumeMap: make(volumeMap)} - - volumeGroupSection = "VolumeGroup:" + volumeGroupName - - volumeList, err = confMap.FetchOptionValueStringSlice(volumeGroupSection, "VolumeList") - if nil != err { - return - } - - // Fetch the virtual IP address for the group. If the volume group is - // shared via NFS or SMB then it must have a virtual IP, otherwise it - // should not. - virtualIPAddr, err = fetchStringSet(confMap, volumeGroupSection, "VirtualIPAddr", virtualIPAddrSet) - if nil != err { - return - } - - shared, err = IsVolumeGroupSharedViaSMB(confMap, volumeGroupName) - if nil != err { - err = fmt.Errorf("IsVolumeGroupSharedViaSMB() failed for volume group '%s': %v", - volumeGroupName, err) - return - } - if !shared { - shared, err = IsVolumeGroupSharedViaNFS(confMap, volumeGroupName) - if nil != err { - err = fmt.Errorf("IsVolumeGroupSharedViaNFS() failed for volume group '%s': %v", - volumeGroupName, err) - return - } - } - - // The virtual IP must be be a valid (parsable) CIDR notation IP address. - // Hold onto the netmask separtely. This code also has the effect of - // converting an IP address to canonical form (RFC-4291 or RFC-4632). - if shared { - volumeGroup.VirtualIPAddr, volumeGroup.VirtualIPMask, err = net.ParseCIDR(virtualIPAddr) - if nil != err { - err = fmt.Errorf("ParseCIDR() for config file variable [%s][%s] value '%s' failed: %v", - volumeGroupSection, "VirtualIPAddr", virtualIPAddr, err) - return - } - - volumeGroup.VirtualHostName, err = fetchStringSet(confMap, volumeGroupSection, - "VirtualHostname", virtualHostNameSet) - if nil != err { - return - } - } - - // We do not check for duplicates of PrimaryPeer - emptySet := make(stringSet) - volumeGroup.PrimaryPeer, err = fetchStringSet(confMap, volumeGroupSection, "PrimaryPeer", emptySet) - if nil != err { - return - } - - // Fill in VG SMB information - err = populateVolumeGroupSMB(confMap, volumeGroupSection, tcpPort, fastTCPPort, volumeGroup, - workGroupSet) - if nil != err { - return - } - - // Grab the volumes in the VG - for _, volumeName = range volumeList { - _, ok = volumeNameSet[volumeName] - if ok { - err = fmt.Errorf("Found duplicate volumeName (\"%s\") in [%s]VolumeList", volumeName, volumeGroupSection) - return - } - - volume = &Volume{VolumeName: volumeName, VolumeGroup: volumeGroup} - - volumeSection = "Volume:" + volumeName - - volume.FSID, err = confMap.FetchOptionValueUint64(volumeSection, "FSID") - if nil != err { - return - } - _, ok = fsidSet[volume.FSID] - if ok { - err = fmt.Errorf("Found duplicate [%s]FSID (%d)", volumeSection, volume.FSID) - return - } - fsidSet[volume.FSID] = struct{}{} - - volume.FUSEMountPointName, err = fetchStringSet(confMap, volumeSection, "FUSEMountPointName", fuseMountPointNameSet) - if nil != err { - return - } - - nfsExportClientMapList, err = confMap.FetchOptionValueStringSlice(volumeSection, "NFSExportClientMapList") - if nil == err { - if (0 < len(nfsExportClientMapList)) && ("" == volume.FUSEMountPointName) { - err = fmt.Errorf("Found empty [%s]FUSEMountPointName but [%s]NFSExportClientMapList is non-empty", volumeSection, volumeSection) - return - } - - volume.nfsClientList = make(NFSClientList, 0, len(nfsExportClientMapList)) - volume.nfsClientMap = make(NFSClientMap) - - nfsExportClientMapSet = make(stringSet) - - for _, nfsExportClientMapListElement = range nfsExportClientMapList { - _, ok = nfsExportClientMapSet[nfsExportClientMapListElement] - if ok { - err = fmt.Errorf("Found duplicate nfsExportClientMapListElement (\"%s\") in [%s]NFSExportClientMapList", nfsExportClientMapListElement, volumeSection) - return - } - - nfsClient = &NFSClient{clientName: nfsExportClientMapListElement} - - nfsClientSection = "NFSClientMap:" + nfsExportClientMapListElement - - nfsClient.ClientPattern, err = confMap.FetchOptionValueString(nfsClientSection, "ClientPattern") - if nil != err { - return - } - nfsClient.AccessMode, err = confMap.FetchOptionValueString(nfsClientSection, "AccessMode") - if nil != err { - return - } - nfsClient.RootSquash, err = confMap.FetchOptionValueString(nfsClientSection, "RootSquash") - if nil != err { - return - } - nfsClient.Secure, err = confMap.FetchOptionValueString(nfsClientSection, "Secure") - if nil != err { - return - } - - volume.nfsClientList = append(volume.nfsClientList, nfsClient) - volume.nfsClientMap[nfsExportClientMapListElement] = nfsClient - - nfsExportClientMapSet[nfsExportClientMapListElement] = struct{}{} - } - } else { - volume.nfsClientList = make(NFSClientList, 0) - volume.nfsClientMap = make(NFSClientMap) - } - - err = populateVolumeSMB(confMap, volumeSection, volume, shareNameSet) - if nil != err { - return - } - - // If any volume has audit logging then the volume group - // has audit logging. - if volume.SMB.AuditLogging { - haveVolumeWithAuditLogging = volume.SMB.AuditLogging - } - - volume.AccountName, err = confMap.FetchOptionValueString(volumeSection, "AccountName") - if nil != err { - return - } - _, ok = accountNameSet[volume.AccountName] - if ok { - err = fmt.Errorf("Found duplicate AccountName (\"%s\") in [%s]FSID", volume.AccountName, volumeSection) - return - } - accountNameSet[volume.AccountName] = struct{}{} - - volumeGroup.VolumeMap[volumeName] = volume - } - volumeGroup.SMB.AuditLogging = haveVolumeWithAuditLogging - - globalVolumeGroupMap[volumeGroupName] = volumeGroup - } - - return -} - -func fetchVolumeInfo(confMap conf.ConfMap) (whoAmI string, localSMBVolumeGroupMap volumeGroupMap, - localVolumeMap volumeMap, globalVolumeGroupMap volumeGroupMap, globalVolumeMap volumeMap, - err error) { - var ( - shared bool - volume *Volume - volumeGroup *VolumeGroup - volumeGroupList []string - volumeGroupName string - volumeName string - ) - - globalVolumeGroupMap = make(volumeGroupMap) - - volumeGroupList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - return - } - - err = populateVolumeGroup(confMap, globalVolumeGroupMap, volumeGroupList) - if nil != err { - return - } - - localSMBVolumeGroupMap = make(volumeGroupMap) - localVolumeMap = make(volumeMap) - globalVolumeMap = make(volumeMap) - - whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - return - } - - for volumeGroupName, volumeGroup = range globalVolumeGroupMap { - if whoAmI == volumeGroup.PrimaryPeer { - for volumeName, volume = range volumeGroup.VolumeMap { - localVolumeMap[volumeName] = volume - } - - shared, err = IsVolumeGroupSharedViaSMB(confMap, volumeGroupName) - if nil != err { - return - } - if shared { - localSMBVolumeGroupMap[volumeGroupName] = volumeGroup - } - } - for volumeName, volume = range volumeGroup.VolumeMap { - globalVolumeMap[volumeName] = volume - } - } - - return -} - -func IsVolumeSharedViaSMB(confMap conf.ConfMap, volumeName string) (shared bool, err error) { - - volumeSection := "Volume:" + volumeName - shareName, err := fetchStringSet(confMap, volumeSection, "SMBShareName", nil) - if err != nil { - return - } - - if shareName != "" { - shared = true - } - return - -} - -func IsVolumeSharedViaNFS(confMap conf.ConfMap, volumeName string) (shared bool, err error) { - - volumeSection := "Volume:" + volumeName - clientList, err := fetchStringSet(confMap, volumeSection, "NFSExportClientMapList", nil) - if err != nil { - return - } - - if clientList != "" { - shared = true - } - return - -} - -// IsVolumeGroupSharedViaSMB Returns true if any volume in the volume group is -// shared using SMB. While we don't support having a volume be shared by both -// SMB and NFS, it seems like two different volumes in a volume group could be, -// so a volume group might be "shared" using both protocols. -// -func IsVolumeGroupSharedViaSMB(confMap conf.ConfMap, vgName string) (shared bool, err error) { - - volumeGroupSection := "VolumeGroup:" + vgName - volumeList, err := confMap.FetchOptionValueStringSlice(volumeGroupSection, "VolumeList") - if err != nil { - return - } - - for _, volumeName := range volumeList { - shared, err = IsVolumeSharedViaSMB(confMap, volumeName) - if err != nil { - return - } - - if shared { - return - } - } - return -} - -// IsVolumeGroupSharedViaNFS Returns true if any volume in the volume group is -// shared using NFS. While we don't support having a volume be shared by both -// SMB and NFS, it seems like two different volumes in a volume group could be, -// so a volume group might be "shared" using both protocols. -// -func IsVolumeGroupSharedViaNFS(confMap conf.ConfMap, vgName string) (shared bool, err error) { - - volumeGroupSection := "VolumeGroup:" + vgName - volumeList, err := confMap.FetchOptionValueStringSlice(volumeGroupSection, "VolumeList") - if err != nil { - return - } - - for _, volumeName := range volumeList { - shared, err = IsVolumeSharedViaNFS(confMap, volumeName) - if err != nil { - return - } - - if shared { - return - } - } - return -} diff --git a/confgen/api_test.go b/confgen/api_test.go deleted file mode 100644 index 5d734546..00000000 --- a/confgen/api_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package confgen - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/stretchr/testify/assert" -) - -func getConfMap(t *testing.T, filename string) (confMap conf.ConfMap, err error) { - assert := assert.New(t) - confMap, err = conf.MakeConfMapFromFile(filename) - assert.Nil(err, "MakeConfMapFromFile errored out") - assert.NotNil(confMap, "confMap should not be nil") - return -} - -// This test tests that the correct smb.conf files are created -// for each VG. -// -// 1. Read sample config using ConfigMap -// 2. Pass ConfigMap to both templates to generate the smb.conf files and prove -// correct -func TestConfigPath(t *testing.T) { - assert := assert.New(t) - - // Get the configuration from the config file - confMap, err := getConfMap(t, "sample-proxyfs-configuration/proxyfs.conf") - - // Grab volumes and volume group information - _, localVolumeGroupMap, _, _, _, err := fetchVolumeInfo(confMap) - assert.Nil(err, "fetchVolumeInfo should succeed") - - // Create temp directory for SMB VG configuration files - var tmpDir string - tmpDir, err = ioutil.TempDir(".", "tst-gen-files") - assert.Nil(err, "ioutil.TempDir returned error") - - err = os.Mkdir(tmpDir+"/vips", confDirPerm) - assert.Nil(err, "os.Mkdir returned error") - - err = createSMBConf(tmpDir, localVolumeGroupMap) - assert.Nil(err, "createSMBConf returned error") - - // TODO - verify new contents - - err = os.RemoveAll(tmpDir) - assert.Nil(err, "Remove of generated directory returned err error") -} - -// Test the IsVolumeShared*() and IsVolumeGroupShared* functions. -func TestIsSharing(t *testing.T) { - assert := assert.New(t) - - // Get the configuration from the config file - confMap, err := getConfMap(t, "sample-proxyfs-configuration/proxyfs.conf") - assert.Nil(err, "getConMap(sample-proxyfs-configuration/proxyfs.conf) should not fail") - - var shared bool - shared, err = IsVolumeSharedViaSMB(confMap, "volume3") - assert.Nil(err, "IsVolumeSharedViaSMB(volume3) should not fail") - assert.False(shared, "volume3 is not shared via SMB") - - shared, err = IsVolumeSharedViaNFS(confMap, "volume3") - assert.Nil(err, "IsVolumeSharedViaNFS(volume3) should not fail") - assert.True(shared, "volume3 is shared via NFS") - - shared, err = IsVolumeSharedViaSMB(confMap, "vol-vg32-2") - assert.Nil(err, "IsVolumeSharedViaSMB(vol-vg32-2) should not fail") - assert.True(shared, "vol-vg32-2 is shared via SMB") - - shared, err = IsVolumeGroupSharedViaSMB(confMap, "vg32-2") - assert.Nil(err, "IsVolumeSharedViaSMB(vg32-2) should not fail") - assert.True(shared, "vg32-2 is shared via SMB") - - shared, err = IsVolumeGroupSharedViaNFS(confMap, "vg32-2") - assert.Nil(err, "IsVolumeSharedViaNFS(vg32-2) should not fail") - assert.False(shared, "vg32-2 is not shared via NFS") - - shared, err = IsVolumeGroupSharedViaNFS(confMap, "VG1") - assert.Nil(err, "IsVolumeSharedViaNFS(VG1) should not fail") - assert.True(shared, "VG1 is shared via NFS") - - shared, err = IsVolumeGroupSharedViaNFS(confMap, "bazbaz") - assert.NotNil(err, "volume group 'bazbaz' does not exist") - - shared, err = IsVolumeSharedViaNFS(confMap, "bazbaz") - assert.NotNil(err, "volume 'bazbaz' does not exist") - - shared, err = IsVolumeSharedViaSMB(confMap, "bambam") - assert.NotNil(err, "volume 'bambam' does not exist") - - shared, err = IsVolumeGroupSharedViaSMB(confMap, "bambam") - assert.NotNil(err, "volume group 'bambam' does not exist") -} diff --git a/confgen/confgen/Makefile b/confgen/confgen/Makefile deleted file mode 100644 index 073b6b37..00000000 --- a/confgen/confgen/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/confgen/confgen - -include ../../GoMakefile diff --git a/confgen/confgen/dummy_test.go b/confgen/confgen/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/confgen/confgen/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/confgen/confgen/main.go b/confgen/confgen/main.go deleted file mode 100644 index a8f05ddf..00000000 --- a/confgen/confgen/main.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// This is a command-line wrapper around package confgen APIs. -// -// Note that the ConfFileOverrides` arguments are specified at the end of the command -// in order to allow for a list of zero or more items. As such, the `ConfFilePath` -// argument immediately preceeds `ConfFileOverrides`. This is the reverse of the -// argument order in the package confgen APIs that conversely follow a from->to -// convention. -package main - -import ( - "fmt" - "log" - "os" - "strings" - - "github.com/NVIDIA/proxyfs/confgen" -) - -func usage() { - fmt.Printf("%s -?\n", os.Args[0]) - fmt.Printf(" Prints this help text\n") - fmt.Printf("%s -I InitialDirPath ConfFilePath [ConfOverrides]*\n", os.Args[0]) - fmt.Printf(" Computes the initial set of conf files storing them in created dir \n") - fmt.Printf("%s -P InitialDirPath PhaseOneDirPath PhaseTwoDirPath ConfFilePath [ConfOverrides]*\n", os.Args[0]) - fmt.Printf(" Computes the two-phase migration from the conf files in \n") - fmt.Printf(" to reach the conf specified in + by first\n") - fmt.Printf(" evacuating Volumes and VolumeGroups that are moving away or being deleted\n") - fmt.Printf(" in Phase One and then populating Volumes and VolumeGroups that are arriving\n") - fmt.Printf(" (via creation of moving towards) in Phase Two.\n") -} - -func main() { - var ( - confFilePath string - confOverrides []string - env confgen.EnvMap - err error - initialDirPath string - phaseOneDirPath string - phaseTwoDirPath string - ) - - if 1 == len(os.Args) { - usage() - os.Exit(1) - } - - switch os.Args[1] { - case "-?": - usage() - os.Exit(0) - case "-I": - if 4 > len(os.Args) { - usage() - os.Exit(1) - } - - env = fetchEnv() - - initialDirPath = os.Args[2] - confFilePath = os.Args[3] - confOverrides = os.Args[4:] - - err = confgen.ComputeInitial(env, confFilePath, confOverrides, initialDirPath) - case "-P": - if 6 > len(os.Args) { - usage() - os.Exit(1) - } - - env = fetchEnv() - - initialDirPath = os.Args[2] - phaseOneDirPath = os.Args[3] - phaseTwoDirPath = os.Args[4] - confFilePath = os.Args[5] - confOverrides = os.Args[6:] - - err = confgen.ComputePhases(env, initialDirPath, confFilePath, confOverrides, phaseOneDirPath, phaseTwoDirPath) - default: - usage() - os.Exit(1) - } - - if nil != err { - log.Fatal(err) - } -} - -func fetchEnv() (env confgen.EnvMap) { - var ( - osEnvironString string - osEnvironStringSlice []string - osEnvironStringSplit []string - ) - - env = make(confgen.EnvMap) - osEnvironStringSlice = os.Environ() - - for _, osEnvironString = range osEnvironStringSlice { - osEnvironStringSplit = strings.SplitN(osEnvironString, "=", 2) - env[osEnvironStringSplit[0]] = osEnvironStringSplit[1] - } - - return -} diff --git a/confgen/mac.conf b/confgen/mac.conf deleted file mode 100644 index e07118ed..00000000 --- a/confgen/mac.conf +++ /dev/null @@ -1,4 +0,0 @@ -.include ../proxyfsd/macproxyfsd0.conf - -[VolumeGroup:CommonVolumeGroup] -VirtualIPAddr: 127.0.0.2 # To be distinct from [Peer:Peer0]PublicIPAddr where [Cluster]WhoAmI == Peer0 diff --git a/confgen/saio.conf b/confgen/saio.conf deleted file mode 100644 index c17e92d1..00000000 --- a/confgen/saio.conf +++ /dev/null @@ -1,4 +0,0 @@ -.include ../proxyfsd/saioproxyfsd0.conf - -[VolumeGroup:CommonVolumeGroup] -VirtualIPAddr: 127.0.0.2 # To be distinct from [Peer:Peer0]PublicIPAddr where [Cluster]WhoAmI == Peer0 diff --git a/confgen/sample-proxyfs-configuration/proxyfs-etcd-endpoints.conf b/confgen/sample-proxyfs-configuration/proxyfs-etcd-endpoints.conf deleted file mode 100644 index 58781fb1..00000000 --- a/confgen/sample-proxyfs-configuration/proxyfs-etcd-endpoints.conf +++ /dev/null @@ -1,2 +0,0 @@ -[FSGlobals] -EtcdEndpoints: 192.168.17.234:2379 192.168.17.177:2379 192.168.18.222:2379 diff --git a/confgen/sample-proxyfs-configuration/proxyfs-shares.conf b/confgen/sample-proxyfs-configuration/proxyfs-shares.conf deleted file mode 100644 index b3b0842d..00000000 --- a/confgen/sample-proxyfs-configuration/proxyfs-shares.conf +++ /dev/null @@ -1,235 +0,0 @@ -# A description of a volume of the file system... along with references to -# storage policies and flow control -# - -[VolumeGroup:vg32-2] -ReadCacheLineSize: 1048576 -ReadCacheWeight: 100 -PrimaryPeer: c67631b5-cb88-11e9-99da-0248604d6797 -VolumeList: vol-vg32-2 -VirtualIPAddr: 32.32.32.32/24 -VirtualHostname: VIP-VG32-2 - -SMBWorkgroup: -SMBActiveDirectoryEnabled: false -SMBActiveDirectoryRealm: -SMBActiveDirectoryIDMapDefaultMin: 501 -SMBActiveDirectoryIDMapDefaultMax: 3554431 -SMBActiveDirectoryIDMapWorkgroupMin: 500 -SMBActiveDirectoryIDMapWorkgroupMax: 3554431 -SMBMapToGuest: bad user -SMBServerMinProtocol: NT1 -SMBADBackend: rid -SMBADIDMgmt: false -SMBADIDSchema: -SMBRPCServerLSARPC: embedded -SMBBrowserAnnounce: 0.0.0.0 -SMBSecurity: user - - - -[Volume:vol-vg32-2] -AccountName: vgacct2 -PhysicalContainerLayoutList: back-policy1 -DefaultPhysicalContainerLayout: back-policy1 -VolumeGroup: vg32-2 -PrimaryPeer: c67631b5-cb88-11e9-99da-0248604d6797 -StandbyPeerList: c6e8e18f-cb88-11e9-92db-02a727b377d1 c63edaae-cb88-11e9-b49f-020e05f0ad07 -CheckpointContainerStoragePolicy: Standard-Replica -CheckpointInterval: 10000ms -CheckpointContainerName: .__checkpoint__ -NonceValuesToReserve: 10000 -FUSEMountPointName: /share/vol-vg32-2 -FSID: 11 -MaxEntriesPerDirNode: 32 -MaxExtentsPerFileNode: 32 -MaxInodesPerMetadataNode: 32 -MaxLogSegmentsPerMetadataNode: 64 -MaxDirFileNodesPerMetadataNode: 16 -MaxCreatedDeletedObjectsPerMetadataNode: 256 -SnapShotIDNumBits: 10 -MaxBytesInodeCache: 104876 -InodeCacheEvictInterval: 5000ms -MaxFlushSize: 10485760 -MaxFlushTime: 10000ms -ActiveLeaseEvictLowLimit: 5000 -ActiveLeaseEvictHighLimit: 5010 - - -CheckpointEtcdKeyName: ProxyFS:Volume:vol-vg32-2 -SMBBrowseable: true -SMBStrictSync: true -SMBAuditLogging: true -SMBEncryptionRequired: false -SMBValidUserList: swift, bob, ed, blake -SMBValidADUserList: -SMBValidADGroupList: -SMBShareName: vol-vg32-2 -NFSExportClientMapList: - - - - - - - -[VolumeGroup:vg32-1] -ReadCacheLineSize: 1048576 -ReadCacheWeight: 100 -PrimaryPeer: c63edaae-cb88-11e9-b49f-020e05f0ad07 -VolumeList: vol-vg32-1 -VirtualIPAddr: 192.168.17.234/24 -VirtualHostname: VIP-VG32-1 - -SMBWorkgroup: -SMBActiveDirectoryEnabled: false -SMBActiveDirectoryRealm: -SMBActiveDirectoryIDMapDefaultMin: 501 -SMBActiveDirectoryIDMapDefaultMax: 3554431 -SMBActiveDirectoryIDMapWorkgroupMin: 500 -SMBActiveDirectoryIDMapWorkgroupMax: 3554431 -SMBMapToGuest: bad user -SMBServerMinProtocol: NT1 -SMBADBackend: rid -SMBADIDMgmt: false -SMBADIDSchema: -SMBRPCServerLSARPC: embedded -SMBBrowserAnnounce: 0.0.0.0 -SMBSecurity: user - - - -[Volume:vol-vg32-1] -AccountName: vgacct1 -PhysicalContainerLayoutList: back-policy1 -DefaultPhysicalContainerLayout: back-policy1 -VolumeGroup: vg32-1 -PrimaryPeer: c63edaae-cb88-11e9-b49f-020e05f0ad07 -StandbyPeerList: c67631b5-cb88-11e9-99da-0248604d6797 c6e8e18f-cb88-11e9-92db-02a727b377d1 -CheckpointContainerStoragePolicy: Standard-Replica -CheckpointInterval: 10000ms -CheckpointContainerName: .__checkpoint__ -NonceValuesToReserve: 10000 -FUSEMountPointName: /share/vol-vg32-1 -FSID: 10 -MaxEntriesPerDirNode: 32 -MaxExtentsPerFileNode: 32 -MaxInodesPerMetadataNode: 32 -MaxLogSegmentsPerMetadataNode: 64 -MaxDirFileNodesPerMetadataNode: 16 -MaxCreatedDeletedObjectsPerMetadataNode: 256 -SnapShotIDNumBits: 10 -MaxBytesInodeCache: 104876 -InodeCacheEvictInterval: 5000ms -MaxFlushSize: 10485760 -MaxFlushTime: 10000ms -ActiveLeaseEvictLowLimit: 5000 -ActiveLeaseEvictHighLimit: 5010 - - -CheckpointEtcdKeyName: ProxyFS:Volume:vol-vg32-1 -SMBBrowseable: true -SMBStrictSync: true -SMBAuditLogging: true -SMBEncryptionRequired: false -SMBValidUserList: swift, bob, ed, blake -SMBValidADUserList: -SMBValidADGroupList: -SMBShareName: vol-vg32-1 -NFSExportClientMapList: - - - - - - - -[VolumeGroup:VG1] -ReadCacheLineSize: 1048576 -ReadCacheWeight: 100 -PrimaryPeer: c67631b5-cb88-11e9-99da-0248604d6797 -VolumeList: volume3 -VirtualIPAddr: 11.11.11.11/24 -VirtualHostname: VIP-VG1 - -SMBWorkgroup: -SMBActiveDirectoryEnabled: false -SMBActiveDirectoryRealm: -SMBActiveDirectoryIDMapDefaultMin: 501 -SMBActiveDirectoryIDMapDefaultMax: 3554431 -SMBActiveDirectoryIDMapWorkgroupMin: 500 -SMBActiveDirectoryIDMapWorkgroupMax: 3554431 -SMBMapToGuest: bad user -SMBServerMinProtocol: NT1 -SMBADBackend: rid -SMBADIDMgmt: false -SMBADIDSchema: -SMBRPCServerLSARPC: embedded -SMBBrowserAnnounce: 0.0.0.0 -SMBSecurity: user - - - -[Volume:volume3] -AccountName: volume3acct -PhysicalContainerLayoutList: back-policy1 -DefaultPhysicalContainerLayout: back-policy1 -VolumeGroup: VG1 -PrimaryPeer: c67631b5-cb88-11e9-99da-0248604d6797 -StandbyPeerList: c6e8e18f-cb88-11e9-92db-02a727b377d1 c63edaae-cb88-11e9-b49f-020e05f0ad07 -CheckpointContainerStoragePolicy: Standard-Replica -CheckpointInterval: 10000ms -CheckpointContainerName: .__checkpoint__ -NonceValuesToReserve: 10000 -FUSEMountPointName: /share/volume3 -FSID: 3 -MaxEntriesPerDirNode: 32 -MaxExtentsPerFileNode: 32 -MaxInodesPerMetadataNode: 32 -MaxLogSegmentsPerMetadataNode: 64 -MaxDirFileNodesPerMetadataNode: 16 -MaxCreatedDeletedObjectsPerMetadataNode: 256 -SnapShotIDNumBits: 10 -MaxBytesInodeCache: 104876 -InodeCacheEvictInterval: 5000ms -MaxFlushSize: 10485760 -MaxFlushTime: 10000ms -ActiveLeaseEvictLowLimit: 5000 -ActiveLeaseEvictHighLimit: 5010 - - -CheckpointEtcdKeyName: ProxyFS:Volume:volume3 -SMBBrowseable: true -SMBStrictSync: true -SMBAuditLogging: true -SMBEncryptionRequired: false -SMBValidUserList: blake -SMBValidADUserList: -SMBValidADGroupList: -SMBShareName: -NFSExportClientMapList: share1_client1 - - -[NFSClientMap:share1_client1] -AccessMode: rw -ClientPattern: 192.168.0.0/16 -RootSquash: no_root_squash -Secure: insecure - - - - -# Describes the set of volumes of the file system listed above -[FSGlobals] -VolumeGroupList: vg32-2 vg32-1 VG1 -ShareConfigVersion: 1572046570.2 - -SMBUserList: swift bob ed blake - - - -[SMBUsers] -swift: c3dpZnQ= -bob: Ym9i -ed: ZWQ= -blake: Ymxha2U= diff --git a/confgen/sample-proxyfs-configuration/proxyfs.conf b/confgen/sample-proxyfs-configuration/proxyfs.conf deleted file mode 100644 index 9ad6c982..00000000 --- a/confgen/sample-proxyfs-configuration/proxyfs.conf +++ /dev/null @@ -1,136 +0,0 @@ -# Default .conf file - -# All ProxyFS nodes in cluster (by uuid) - -[Peer:c63edaae-cb88-11e9-b49f-020e05f0ad07] -PublicIPAddr: 192.168.17.234 -PrivateIPAddr: 192.168.17.234 -ReadCacheQuotaFraction: 0.20 - -[Peer:c67631b5-cb88-11e9-99da-0248604d6797] -PublicIPAddr: 192.168.17.177 -PrivateIPAddr: 192.168.17.177 -ReadCacheQuotaFraction: 0.20 - -[Peer:c6e8e18f-cb88-11e9-92db-02a727b377d1] -PublicIPAddr: 192.168.18.222 -PrivateIPAddr: 192.168.18.222 -ReadCacheQuotaFraction: 0.20 - -# Identifies what "peers" make up the cluster and which one "we" are -[Cluster] -WhoAmI: c67631b5-cb88-11e9-99da-0248604d6797 -Peers: c63edaae-cb88-11e9-b49f-020e05f0ad07 c67631b5-cb88-11e9-99da-0248604d6797 c6e8e18f-cb88-11e9-92db-02a727b377d1 -Arbiters: c63edaae-cb88-11e9-b49f-020e05f0ad07 c67631b5-cb88-11e9-99da-0248604d6797 c6e8e18f-cb88-11e9-92db-02a727b377d1 -ServerGuid: c67631b5-cb88-11e9-99da-0248604d6797 -AcctHash: de374096638e77912ff8ebf617b98fc7 -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 -ConfigVersion: 1570666863 - -# Specifies the path particulars to the "NoAuth" WSGI pipeline -[SwiftClient] -NoAuthIPAddr: 127.0.0.1 -NoAuthTCPPort: 8090 - -RetryDelay: 1s -RetryExpBackoff: 1.5 -RetryLimit: 11 - -RetryDelayObject: 1s -RetryExpBackoffObject: 1.95 -RetryLimitObject: 8 - -ChunkedConnectionPoolSize: 512 -NonChunkedConnectionPoolSize: 128 - -SwiftReconNoWriteThreshold: 80 -SwiftReconNoWriteErrno: ENOSPC -SwiftReconReadOnlyThreshold: 90 -SwiftReconReadOnlyErrno: EROFS -SwiftConfDir: /etc/swift -SwiftReconChecksPerConfCheck: 10 - - -# A set of storage policies into which the chunks of files and directories will go - -[PhysicalContainerLayout:back-policy1] -# Index is maintained both compat with 0.54.1.2 and so we can track policy through name changes -ContainerStoragePolicyIndex: 0 -ContainerStoragePolicy: Standard-Replica -ContainerNamePrefix: Standard-Replica_ -ContainersPerPeer: 1000 -MaxObjectsPerContainer: 1000000 - -# RPC path from file system clients (both Samba and "normal" WSGI stack)... needs to be shared with them -[JSONRPCServer] -TCPPort: 12345 -FastTCPPort: 32345 -DataPathLogging: false -Debug: false -RetryRPCPort: 32356 -RetryRPCTTLCompleted: 10m -RetryRPCAckTrim: 100ms -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCertFilePath: -RetryRPCKeyFilePath: -MinLeaseDuration: 250ms -LeaseInterruptInterval: 250ms -LeaseInterruptLimit: 20 - -[RPC] # Note: This is very soon to be deprecated... so just hard-code these values until then -NoAuthTCPSocket=true -# To use MockSwift instead of real swift, comment out the above line and uncomment the one below. -#MockSocket=true - -# Log reporting parameters -[Logging] -LogFilePath: /opt/ss/var/log/proxyfsd.log -TraceLevelLogging: none -DebugLevelLogging: none -# when true, lot to stderr even when LogFilePath is set- -LogToConsole: false - -[Stats] -UDPPort: 8133 -BufferLength: 100 -MaxLatency: 1000ms - -[StatsLogger] -Period: 600s - -[HTTPServer] -TCPPort: 15346 - -[FSGlobals] -InodeRecCacheEvictLowLimit: 100000 -InodeRecCacheEvictHighLimit: 100010 -LogSegmentRecCacheEvictLowLimit: 100000 -LogSegmentRecCacheEvictHighLimit: 100010 -BPlusTreeObjectCacheEvictLowLimit: 100000 -BPlusTreeObjectCacheEvictHighLimit: 100010 -DirEntryCacheEvictLowLimit: 100000 -DirEntryCacheEvictHighLimit: 100010 -FileExtentMapEvictLowLimit: 100000 -FileExtentMapEvictHighLimit: 100010 -CreatedDeletedObjectsCacheEvictLowLimit: 100000 -CreatedDeletedObjectsCacheEvictHighLimit: 100010 -HAMode: on - -EtcdEnabled: true -EtcdAutoSyncInterval: 60000ms -EtcdCertDir: /etc/ssl/etcd/ssl/ -EtcdDialTimeout: 10000ms -EtcdOpTimeout: 20000ms - -.include ./proxyfs-etcd-endpoints.conf -.include ./proxyfs-shares.conf diff --git a/confgen/sample-proxyfs-configuration/samba/smb.conf b/confgen/sample-proxyfs-configuration/samba/smb.conf deleted file mode 100644 index b7fc840a..00000000 --- a/confgen/sample-proxyfs-configuration/samba/smb.conf +++ /dev/null @@ -1,126 +0,0 @@ -[global] - -# ----------------------- Network-Related Options ------------------------- - - server string = Samba Server Version %v - map to guest = bad user - - netbios name = ip-192-168-17-177 - server min protocol = NT1 - - -# --------------------------- Temporary Options ------------------------- -# Until we resolve permissions -force user = root - -# --------------------------- Logging Options ----------------------------- - - # log files split per-machine: - log file = /opt/ss/var/log/samba/log.%m - # maximum size of 50KB per log file, then rotate: - max log size = 50 - - log level = full_audit:1 - - -# ----------------------- Standalone Server Options ------------------------ - security = user - passdb backend = tdbsam:/opt/ss/lib64/samba/passdb.tdb - restrict anonymous = 2 - public = no - guest account = nobody - rpc_server:lsarpc = embedded - - -# ----------------------- Domain Members Options ------------------------ -;realm = -;winbind nss info = -winbind use default domain = yes -winbind refresh tickets = yes -winbind enum users = yes -winbind enum groups = yes -winbind expand groups = 5 -winbind nested groups = yes - -;idmap config *:backend = tdb -;idmap config *:range = 501 - 3554431 - -;idmap config :backend = rid -;idmap config :default = yes -;idmap config :range = 500 - 3554431 - -;template shell = /sbin/nologin -;domain master = no -;preferred master = no - -;kerberos method = secrets and keytab - - -#----------------------------- Name Resolution ------------------------------- - -;wins server = -;remote announce = 0.0.0.0/ - -# --------------------------- ProxyFS Options --------------------------- - - proxyfs:PrivateIPAddr = 192.168.17.177 - proxyfs:TCPPort = 12345 - proxyfs:FastTCPPort = 32345 - -#============================ Share Definitions ============================== - - - -[volume3] - comment = ProxyFS volume volume3 - path = /opt/ss/var/lib/dummy_path - proxyfs:volume = volume3 - valid users = "blake" - writable = yes - printable = no - browseable = yes - oplocks = False - level2 oplocks = False - aio read size = 1 - aio write size = 1 - case sensitive = yes - preserve case = yes - short preserve case = yes - strict sync = yes - - full_audit:success = mkdir rmdir read pread write pwrite rename unlink - full_audit:prefix = %u|%I|%m|%S - full_audit:failure = mkdir rmdir read pread write pwrite rename unlink - full_audit:syslog = false - vfs objects = full_audit proxyfs - - - - - -[vol-vg32-2] - comment = ProxyFS volume vol-vg32-2 - path = /opt/ss/var/lib/dummy_path - proxyfs:volume = vol-vg32-2 - valid users = "swift", "bob", "ed", "blake" - writable = yes - printable = no - browseable = yes - oplocks = False - level2 oplocks = False - aio read size = 1 - aio write size = 1 - case sensitive = yes - preserve case = yes - short preserve case = yes - strict sync = yes - - full_audit:success = mkdir rmdir read pread write pwrite rename unlink - full_audit:prefix = %u|%I|%m|%S - full_audit:failure = mkdir rmdir read pread write pwrite rename unlink - full_audit:syslog = false - vfs objects = full_audit proxyfs - - - - diff --git a/confgen/template.go b/confgen/template.go deleted file mode 100644 index 1d23ed45..00000000 --- a/confgen/template.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package confgen - -import ( - "fmt" - "os" - "text/template" -) - -// createSMBConf writes the per VG smb.conf file -func createSMBConf(initialDirPath string, localVolumeGroupMap volumeGroupMap) (err error) { - var ( - vipDirPath string - volumeGroup *VolumeGroup - ) - - // Locate the proxyfs config directory with the templates - var proxyfsConfDir string - confDirs := []string{proxyfsConfDir0, proxyfsConfDir1, proxyfsConfDir2} - for _, proxyfsConfDir = range confDirs { - - // if the directory has a "smb_globals.tmpl" - _, err = os.Stat(proxyfsConfDir + "/" + "smb_globals.tmpl") - if err == nil { - break - } - } - if err != nil { - fmt.Printf("Could not find '%s' in any of %v\n", "smb_globals.tmpl", confDirs) - return - } - - // Load the template for the global section of smb.conf - // from one of several possible locations - globalTplate, err := template.ParseFiles(proxyfsConfDir + "/" + "smb_globals.tmpl") - if nil != err { - - // TODO - log this appropriately - fmt.Printf("Parse of template file returned err: %v\n", err) - return - } - - // Load the template for the share section of smb.conf - sharesTplate, err := template.ParseFiles(proxyfsConfDir + "/" + "smb_shares.tmpl") - if nil != err { - // TODO - log this appropriately - fmt.Printf("Parse of template file returned err: %v\n", err) - return - } - - for _, volumeGroup = range localVolumeGroupMap { - vipDirPath = initialDirPath + "/" + vipsDirName + "/" + volumeGroup.VirtualIPAddr.String() - - err = os.Mkdir(vipDirPath, confDirPerm) - if nil != err { - return - } - - fileName := vipDirPath + "/smb-VG-" + volumeGroup.VolumeGroupName + ".conf" - f, openErr := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, smbConfPerm) - if nil != openErr { - err = openErr - return - } - - err = globalTplate.Execute(f, volumeGroup) - if nil != err { - return - } - - err = sharesTplate.Execute(f, volumeGroup) - if nil != err { - return - } - } - - return -} diff --git a/confgen/templates/smb_globals.tmpl b/confgen/templates/smb_globals.tmpl deleted file mode 100644 index 7ddd6236..00000000 --- a/confgen/templates/smb_globals.tmpl +++ /dev/null @@ -1,89 +0,0 @@ -{{/* Declare our global variables. Need Go 1.11+ */}} -{{$AdIDCommentStr := ""}} -{{$ADDisabledCommentStr := ""}} -{{$EnabledIfNoAD := ";"}} {{/* If no AD, print this line */}} -{{ if not .SMB.ADEnabled }} - {{$EnabledIfNoAD = ""}} - {{$ADDisabledCommentStr = ";"}} -{{ end }} - -{{ if not .SMB.ADIDMgmt}} - {{$AdIDCommentStr = ";"}} -{{ end }} - -[global] - -# ----------------------- Network-Related Options ------------------------- -{{ if .SMB.WorkGroup }} - workgroup = {{ .WorkGroup }} -{{ end }} - interfaces = {{ .VirtualIPAddr }} - pid directory = /opt/ss/var/run/samba/{{ .VolumeGroupName }} - lock directory = /opt/ss/var/cache/samba/{{ .VolumeGroupName }} - private dir = /opt/ss/var/cache/samba/{{ .VolumeGroupName }} - - server string = Samba Server Version %v - map to guest = {{ .SMB.MapToGuest }} - - netbios name = {{ .VirtualHostName }} - server min protocol = {{ .SMB.ServerMinProtocol }} - -# --------------------------- Temporary Options ------------------------- -# Until we resolve permissions -{{ $EnabledIfNoAD }}force user = root - - -# --------------------------- Logging Options ----------------------------- - - # log files split per-machine: - log file = /opt/ss/var/log/samba/log.%m - # maximum size of 50KB per log file, then rotate: - max log size = 50 -{{ if .SMB.AuditLogging }} - log level = full_audit:1 -{{ end }} - -# ----------------------- Standalone Server Options ------------------------ - security = {{ .SMB.Security }} - {{ $EnabledIfNoAD }}passdb backend = tdbsam:/opt/ss/lib64/samba/passdb.tdb - restrict anonymous = 2 - public = no - guest account = nobody - rpc_server:lsarpc = {{ .SMB.RPCServerLSARPC }} - - -# ----------------------- Domain Members Options ------------------------ -{{ $ADDisabledCommentStr }}realm = {{ .SMB.ADRealm }} -{{ $AdIDCommentStr }}winbind nss info = {{ .SMB.ADIDSchema }} -winbind use default domain = yes -winbind refresh tickets = yes -winbind enum users = yes -winbind enum groups = yes -winbind expand groups = 5 -winbind nested groups = yes - -{{ $ADDisabledCommentStr }}idmap config *:backend = tdb -{{ $ADDisabledCommentStr }}idmap config *:range = {{ .SMB.ADIDMapDefaultMin }} - {{ .SMB.ADIDMapDefaultMax }} - -{{ $ADDisabledCommentStr }}idmap config {{ .SMB.WorkGroup }}:backend = {{ .SMB.ADBackEnd }} -{{ $ADDisabledCommentStr }}idmap config {{ .SMB.WorkGroup }}:default = yes -{{ $ADDisabledCommentStr }}idmap config {{ .SMB.WorkGroup }}:range = {{ .SMB.ADIDMapWorkgroupMin }} - {{ .SMB.ADIDMapWorkgroupMax }} - -{{ $ADDisabledCommentStr }}template shell = /sbin/nologin -{{ $ADDisabledCommentStr }}domain master = no -{{ $ADDisabledCommentStr }}preferred master = no - -{{ $ADDisabledCommentStr }}kerberos method = secrets and keytab - - -#----------------------------- Name Resolution ------------------------------- - -{{ $ADDisabledCommentStr }}wins server = -{{ $ADDisabledCommentStr }}remote announce = {{ .SMB.BrowserAnnounce }}/{{ .SMB.WorkGroup }} - -# --------------------------- ProxyFS Options --------------------------- - - proxyfs:PrivateIPAddr = {{ .VirtualIPAddr }} - proxyfs:TCPPort = {{ .SMB.TCPPort }} - proxyfs:FastTCPPort = {{ .SMB.FastTCPPort }} - diff --git a/confgen/templates/smb_shares.tmpl b/confgen/templates/smb_shares.tmpl deleted file mode 100644 index 336a2ec0..00000000 --- a/confgen/templates/smb_shares.tmpl +++ /dev/null @@ -1,37 +0,0 @@ -#============================ Share Definitions ============================== - -{{range .VolumeMap}} - -[{{ .SMB.ShareName }}] - comment = ProxyFS volume {{ .SMB.ShareName }} - path = {{ .SMB.Path }} - proxyfs:volume = {{ .VolumeName }} - - valid users = {{ range $i, $e := .SMB.ValidUsers}}{{if $i}}, {{end}}"{{$e}}"{{- end}} - - writable = yes - printable = no - browseable = {{ if .SMB.Browseable }}yes{{- else }}no{{- end }} - oplocks = False - level2 oplocks = False - aio read size = 1 - aio write size = 1 - case sensitive = yes - preserve case = yes - short preserve case = yes - strict sync = {{ if .SMB.StrictSync }}yes{{- else }}no{{- end }} -{{ if .SMB.AuditLogging }} - full_audit:success = mkdir rmdir read pread write pwrite rename unlink - full_audit:prefix = %u|%I|%m|%S - full_audit:failure = mkdir rmdir read pread write pwrite rename unlink - full_audit:syslog = false - vfs objects = full_audit proxyfs -{{- else }} - vfs objects = proxyfs -{{- end }} -{{ if .SMB.EncryptionRequired }} - smb encrypt = required -{{ end }} - -{{end}} {{/* End of range loop */}} - diff --git a/dlm/Makefile b/dlm/Makefile deleted file mode 100644 index f54a4b86..00000000 --- a/dlm/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/dlm - -include ../GoMakefile diff --git a/dlm/api.go b/dlm/api.go deleted file mode 100644 index 843a0a9c..00000000 --- a/dlm/api.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Distributed Lock Manager (DLM) provides locking between threads on the same -// system and between threads on different systems. -// -// Example use of the lock: -/* -type myStruct struct { - Acct string - Volume string - InodeNum uint64 - myRwLock *RWLockStruct - } - -func my_function() { - var myval myStruct - myval.Acct = "Acct1" - myval.Volume = "Vol1" - myval.InodeNum = 11 - myLockId := fmt.Sprintf("%s:%s:%v\n", myval.Acct, myval.Volume, myval.InodeNum) - - myCookie := GenerateCallerID() - myval.myRwLock = &RWLockStruct{LockID: myLockId, Notify: nil, LockCallerID: myCookie} - myval.myRwLock.ReadLock() - myval.myRwLock.Unlock() - myval.myRwLock.WriteLock() - myval.myRwLock.Unlock() - err := myval.myRwLock.TryReadLock() - errno := blunder.Errno(err) - - switch errno { - case 0: - log.Printf("Got TryReadLock") - myval.myRwLock.Unlock() - case EAGAIN: // give up other locks. - default: // something wrong.. - } - - err = myval.myRwLock.TryWriteLock() - errno := blunder.Errno(err) - - switch errno { - case 0: - log.Printf("Got TryWriteLock") - myval.myRwLock.Unlock() - case EAGAIN: // give up other locks. - default: // something wrong.. - } - } -*/ -package dlm - -import ( - "fmt" - - "github.com/NVIDIA/proxyfs/trackedlock" -) - -type NotifyReason uint32 -type CallerID *string - -const ( - ReasonWriteRequest NotifyReason = iota + 1 // Another thread in the cluster wants "Write" access - ReasonReadRequest // Another thread wants "Read" access -) - -type LockHeldType uint32 - -const ( - ANYLOCK LockHeldType = iota + 1 - READLOCK - WRITELOCK -) - -type Notify interface { - NotifyNodeChange(reason NotifyReason) // DLM will call the last node which owns the lock before handing over - // the lock to another node. Useful for leases and data caching. -} - -type RWLockStruct struct { - LockID string - Notify Notify - LockCallerID CallerID -} - -// Lock for generating unique caller IDs -// For now, this is just an in-memory thing. -var callerIDLock trackedlock.Mutex -var nextCallerID uint64 = 1000 - -// GenerateCallerID() returns a cluster wide unique number useful in deadlock detection. -func GenerateCallerID() (callerID CallerID) { - - // TODO - we need to use a nonce value instead of this when we have clustering - callerIDLock.Lock() - - callerIDStr := fmt.Sprintf("%d", nextCallerID) - callerID = CallerID(&callerIDStr) - nextCallerID++ - - callerIDLock.Unlock() - - return callerID -} - -// IsLockHeld() returns -func IsLockHeld(lockID string, callerID CallerID, lockHeldType LockHeldType) (held bool) { - held = isLockHeld(lockID, callerID, lockHeldType) - return held -} - -// GetLockID() returns the lock ID from the lock struct -func (l *RWLockStruct) GetLockID() string { - return l.LockID -} - -// CallerID returns the caller ID from the lock struct -func (l *RWLockStruct) GetCallerID() CallerID { - return l.LockCallerID -} - -// Returns whether the lock is held for reading -func (l *RWLockStruct) IsReadHeld() bool { - held := isLockHeld(l.LockID, l.LockCallerID, READLOCK) - return held -} - -// Returns whether the lock is held for writing -func (l *RWLockStruct) IsWriteHeld() bool { - held := isLockHeld(l.LockID, l.LockCallerID, WRITELOCK) - return held -} - -// WriteLock() blocks until the lock for the inode can be held exclusively. -func (l *RWLockStruct) WriteLock() (err error) { - // TODO - what errors are possible here? - err = l.commonLock(exclusive, false) - return err -} - -// ReadLock() blocks until the lock for the inode can be held shared. -func (l *RWLockStruct) ReadLock() (err error) { - // TODO - what errors are possible here? - err = l.commonLock(shared, false) - return err -} - -// TryWriteLock() attempts to grab the lock if is is free. Otherwise, it returns EAGAIN. -func (l *RWLockStruct) TryWriteLock() (err error) { - err = l.commonLock(exclusive, true) - return err -} - -// TryReadLock() attempts to grab the lock if is is free or shared. Otherwise, it returns EAGAIN. -func (l *RWLockStruct) TryReadLock() (err error) { - err = l.commonLock(shared, true) - return err -} - -// Unlock() releases the lock and signals any waiters that the lock is free. -func (l *RWLockStruct) Unlock() (err error) { - // TODO what error is possible? - err = l.unlock() - return err -} diff --git a/dlm/config.go b/dlm/config.go deleted file mode 100644 index a8905630..00000000 --- a/dlm/config.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package dlm - -// Configuration variables for DLM - -import ( - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -type globalsStruct struct { - trackedlock.Mutex - - // Map used to store locks owned locally - // NOTE: This map is protected by the Mutex - localLockMap map[string]*localLockTrack - - // TODO - channels for STOP and from DLM lock master? - // is the channel lock one per lock or a global one from DLM? - // how could it be... probably just one receive thread the locks - // map, checks bit and releases lock if no one using, otherwise - // blocks until it is free... -} - -var globals globalsStruct - -func init() { - transitions.Register("dlm", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - // Create map used to store locks - globals.localLockMap = make(map[string]*localLockTrack) - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - return nil -} -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - return nil -} diff --git a/dlm/llm.go b/dlm/llm.go deleted file mode 100644 index 4a3090ad..00000000 --- a/dlm/llm.go +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package dlm - -import ( - "container/list" - "errors" - "fmt" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/trackedlock" -) - -// This struct is used by LLM to track a lock. -type localLockTrack struct { - trackedlock.Mutex - lockId string // lock identity (must be unique) - owners uint64 // Count of threads which own lock - waiters uint64 // Count of threads which want to own the lock (either shared or exclusive) - state lockState - exclOwner CallerID - listOfOwners []CallerID - waitReqQ *list.List // List of requests waiting for lock - rwMutexTrack trackedlock.RWMutexTrack // Track the lock to see how long its held -} - -var localLockTrackPool = sync.Pool{ - New: func() interface{} { - var track localLockTrack - - // every localLockTrack should have a waitReqQ - track.waitReqQ = list.New() - - return &track - }, -} - -type localLockRequest struct { - requestedState lockState - *sync.Cond - wakeUp bool - LockCallerID CallerID -} - -type lockState int - -const ( - nilType lockState = iota - shared - exclusive - stale -) - -// NOTE: This is a test-only interface used for unit tests. -// -// This function assumes that globals.Lock() is held. -// TODO - can this be used in more cases without creating entry it if does not exist? -func getTrack(lockId string) (track *localLockTrack, ok bool) { - track, ok = globals.localLockMap[lockId] - if !ok { - return track, ok - } - return track, ok -} - -// NOTE: This is a test-only interface used for unit tests. -func waitCountWaiters(lockId string, count uint64) { - for { - globals.Lock() - track, ok := getTrack(lockId) - - // If the tracking object has not been created yet, sleep and retry. - if !ok { - // Sleep 5 milliseconds and test again - globals.Unlock() - time.Sleep(5 * time.Millisecond) - break - } - - track.Mutex.Lock() - - globals.Unlock() - - waiters := track.waiters - track.Mutex.Unlock() - - if waiters == count { - return - } else { - // Sleep 5 milliseconds and test again - time.Sleep(5 * time.Millisecond) - } - } -} - -// NOTE: This is a test-only interface used for unit tests. -func waitCountOwners(lockId string, count uint64) { - for { - globals.Lock() - track, ok := getTrack(lockId) - - // If the tracking object has not been created yet, sleep and retry. - if !ok { - // Sleep 5 milliseconds and test again - globals.Unlock() - time.Sleep(5 * time.Millisecond) - break - } - - track.Mutex.Lock() - - globals.Unlock() - - owners := track.owners - track.Mutex.Unlock() - - if owners == count { - return - } else { - // Sleep 5 milliseconds and test again - time.Sleep(5 * time.Millisecond) - } - } -} - -// This function assumes the mutex is held on the tracker structure -func (t *localLockTrack) removeFromListOfOwners(callerID CallerID) { - - // Find Position and delete entry (a map might be more efficient) - for i, id := range t.listOfOwners { - if id == callerID { - lastIdx := len(t.listOfOwners) - 1 - t.listOfOwners[i] = t.listOfOwners[lastIdx] - t.listOfOwners = t.listOfOwners[:lastIdx] - return - } - } - - panic(fmt.Sprintf("Can't find CallerID: %v in list of lock owners!", callerID)) -} - -// This function assumes the mutex is held on the tracker structure -func callerInListOfOwners(listOfOwners []CallerID, callerID CallerID) (amOwner bool) { - // Find Position - for _, id := range listOfOwners { - if id == callerID { - return true - } - } - - return false -} - -func isLockHeld(lockID string, callerID CallerID, lockHeldType LockHeldType) (held bool) { - globals.Lock() - // NOTE: Not doing a defer globals.Unlock() here since grabbing another lock below. - - track, ok := globals.localLockMap[lockID] - if !ok { - - // Lock does not exist in map - globals.Unlock() - return false - } - - track.Mutex.Lock() - - globals.Unlock() - - defer track.Mutex.Unlock() - - switch lockHeldType { - case READLOCK: - return (track.state == shared) && (callerInListOfOwners(track.listOfOwners, callerID)) - case WRITELOCK: - return (track.state == exclusive) && (callerInListOfOwners(track.listOfOwners, callerID)) - case ANYLOCK: - return ((track.state == exclusive) || (track.state == shared)) && (callerInListOfOwners(track.listOfOwners, callerID)) - } - return false -} - -func grantAndSignal(track *localLockTrack, localQRequest *localLockRequest) { - track.state = localQRequest.requestedState - track.listOfOwners = append(track.listOfOwners, localQRequest.LockCallerID) - track.owners++ - - if track.state == exclusive { - if track.exclOwner != nil || track.owners != 1 { - panic(fmt.Sprintf("granted exclusive lock when (exclOwner != nil || track.owners != 1)! "+ - "track lockId %v owners %d waiters %d lockState %v exclOwner %v listOfOwners %v", - track.lockId, track.owners, track.waiters, track.state, - *track.exclOwner, track.listOfOwners)) - } - track.exclOwner = localQRequest.LockCallerID - } - - localQRequest.wakeUp = true - localQRequest.Cond.Broadcast() -} - -// Process the waitReqQ and see if any locks can be granted. -// -// This function assumes that the tracking mutex is held. -func processLocalQ(track *localLockTrack) { - - // If nothing on queue then return - if track.waitReqQ.Len() == 0 { - return - } - - // If the lock is already held exclusively then nothing to do. - if track.state == exclusive { - return - } - - // At this point, the lock is either stale or shared - // - // Loop through Q and see if a request can be granted. If it can then pop it off the Q. - for track.waitReqQ.Len() > 0 { - elem := track.waitReqQ.Remove(track.waitReqQ.Front()) - var localQRequest *localLockRequest - var ok bool - if localQRequest, ok = elem.(*localLockRequest); !ok { - panic("Remove of elem failed!!!") - } - - // If the lock is already free and then want it exclusive - if (localQRequest.requestedState == exclusive) && (track.state == stale) { - grantAndSignal(track, localQRequest) - return - } - - // If want exclusive and not free, we can't grant so push on front and break from loop. - if localQRequest.requestedState == exclusive { - track.waitReqQ.PushFront(localQRequest) - return - } - - // At this point we know the Q entry is shared. Grant it now. - grantAndSignal(track, localQRequest) - } -} - -func (l *RWLockStruct) commonLock(requestedState lockState, try bool) (err error) { - - globals.Lock() - track, ok := globals.localLockMap[l.LockID] - if !ok { - // TODO - handle blocking waiting for lock from DLM - - // Lock does not exist in map, get one - track = localLockTrackPool.Get().(*localLockTrack) - if track.waitReqQ.Len() != 0 { - panic(fmt.Sprintf("localLockTrack object %p from pool does not have empty waitReqQ", - track)) - } - if len(track.listOfOwners) != 0 { - panic(fmt.Sprintf("localLockTrack object %p from pool does not have empty ListOfOwners", - track)) - } - track.lockId = l.LockID - track.state = stale - - globals.localLockMap[l.LockID] = track - - } - - track.Mutex.Lock() - defer track.Mutex.Unlock() - - globals.Unlock() - - // If we are doing a TryWriteLock or TryReadLock, see if we could - // grab the lock before putting on queue. - if try { - if (requestedState == exclusive) && (track.state != stale) { - err = errors.New("Lock is busy - try again!") - return blunder.AddError(err, blunder.TryAgainError) - } else { - if track.state == exclusive { - err = errors.New("Lock is busy - try again!") - return blunder.AddError(err, blunder.TryAgainError) - } - } - } - localRequest := localLockRequest{requestedState: requestedState, LockCallerID: l.LockCallerID, wakeUp: false} - localRequest.Cond = sync.NewCond(&track.Mutex) - track.waitReqQ.PushBack(&localRequest) - - track.waiters++ - - // See if any locks can be granted - processLocalQ(track) - - // wakeUp will already be true if processLocalQ() signaled this thread to wakeup. - for localRequest.wakeUp == false { - localRequest.Cond.Wait() - } - - // sanity check request and lock state - if localRequest.wakeUp != true { - panic(fmt.Sprintf("commonLock(): thread awoke without being signalled; localRequest %v "+ - "track lockId %v owners %d waiters %d lockState %v exclOwner %v listOfOwners %v", - localRequest, track.lockId, track.owners, track.waiters, track.state, - *track.exclOwner, track.listOfOwners)) - } - if track.state == stale || track.owners == 0 || (track.owners > 1 && track.state != shared) { - panic(fmt.Sprintf("commonLock(): lock is in undefined state: localRequest %v "+ - "track lockId %v owners %d waiters %d lockState %v exclOwner %v listOfOwners %v", - localRequest, track.lockId, track.owners, track.waiters, track.state, - *track.exclOwner, track.listOfOwners)) - } - - // let trackedlock package track how long we hold the lock - if track.state == exclusive { - track.rwMutexTrack.LockTrack(track) - } else { - track.rwMutexTrack.RLockTrack(track) - } - - // At this point, we got the lock either by the call to processLocalQ() above - // or as a result of processLocalQ() being called from the unlock() path. - - // We decrement waiters here instead of in processLocalQ() so that other threads do not - // assume there are no waiters between the time the Cond is signaled and we wakeup this thread. - track.waiters-- - - return nil -} - -// unlock() releases the lock and signals any waiters that the lock is free. -func (l *RWLockStruct) unlock() (err error) { - - // TODO - assert not stale and if shared that count != 0 - globals.Lock() - track, ok := globals.localLockMap[l.LockID] - if !ok { - panic(fmt.Sprintf("Trying to Unlock() inode: %v and lock not found in localLockMap()!", l.LockID)) - } - - track.Mutex.Lock() - - // Remove lock from localLockMap if no other thread using. - // - // We have track structure for lock. While holding mutex on localLockMap, remove - // lock from map if we are the last holder of the lock. - // TODO - does this handle revoke case and any others? - var deleted = false - if (track.owners == 1) && (track.waiters == 0) { - deleted = true - delete(globals.localLockMap, l.LockID) - } - - globals.Unlock() - - // TODO - handle release of lock back to DLM and delete from localLockMap - // Set stale and signal any waiters - track.owners-- - track.removeFromListOfOwners(l.LockCallerID) - if track.state == exclusive { - if track.owners != 0 || track.exclOwner == nil { - panic(fmt.Sprintf("releasing exclusive lock when (exclOwner == nil || track.owners != 0)! "+ - "track lockId %v owners %d waiters %d lockState %v exclOwner %v listOfOwners %v", - track.lockId, track.owners, track.waiters, track.state, - *track.exclOwner, track.listOfOwners)) - } - track.exclOwner = nil - } - - if track.owners == 0 { - track.state = stale - } else { - if track.owners < 0 { - panic("track.owners < 0!!!") - } - } - // record the release of the lock - track.rwMutexTrack.DLMUnlockTrack(track) - - // See if any locks can be granted - processLocalQ(track) - - track.Mutex.Unlock() - - // can't return the - if deleted { - if track.waitReqQ.Len() != 0 || track.waiters != 0 || track.state != stale { - panic(fmt.Sprintf( - "localLockTrack object %p retrieved from pool does not have an empty waitReqQ", - track.waitReqQ)) - } - localLockTrackPool.Put(track) - } - - // TODO what error is possible? - return nil -} diff --git a/dlm/llm_test.go b/dlm/llm_test.go deleted file mode 100644 index 7682f170..00000000 --- a/dlm/llm_test.go +++ /dev/null @@ -1,691 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package dlm - -import ( - "flag" - "io/ioutil" - "os" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -// Test string for passing inode 1 -var s1 string = strconv.Itoa(1) - -// Mutex for protecting global variables -var mutex trackedlock.Mutex - -type testOpTyp int - -const ( - nilTestOp testOpTyp = iota - readLock - writeLock - tryReadLock - tryWriteLock - unlock - stopThread -) - -type testReq struct { - lockID string - typ testOpTyp // Operation type - writeLock, etc - t *testing.T -} - -// Per thread structure storing channel information -type threadInfo struct { - startedNode chan bool - requestForThread chan *testReq -} - -var globalSyncPt chan testReq // Channel used to synchronize test threads to simulate multiple threads - -var testConfMap conf.ConfMap - -// Largely stolen from fs/api_test.go -func testSetup() (err error) { - confStrings := []string{ - "TrackedLock.LockHoldTimeLimit=2s", - "TrackedLock.LockCheckPeriod=1s", - } - - testDir, err := ioutil.TempDir(os.TempDir(), "ProxyFS_test_ldlm_") - if nil != err { - return - } - - err = os.Chdir(testDir) - if nil != err { - return - } - - err = os.Mkdir("TestVolume", os.ModePerm) - - confMap, err := conf.MakeConfMapFromStrings(confStrings) - if err != nil { - return - } - - err = logger.Up(confMap) - if nil != err { - return - } - - // Setup channel used to synchronize multiple test thread operations - globalSyncPt = make(chan testReq) - - testConfMapStrings := []string{ - "Logging.LogFilePath=/dev/null", - "Cluster.WhoAmI=nobody", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - return - } - - err = transitions.Up(testConfMap) - if nil != err { - logger.ErrorWithError(err, "transitions.Up() failed") - return - } - - return -} - -// Largely stolen from fs/api_test.go -func testTeardown() (err error) { - err = transitions.Down(testConfMap) - if nil != err { - logger.ErrorWithError(err, "transitions.Down() failed") - return - } - - testDir, err := os.Getwd() - if nil != err { - return - } - - err = os.Chdir("..") - if nil != err { - return - } - - err = os.RemoveAll(testDir) - if nil != err { - return - } - - return -} - -// Largely stolen from fs/api_test.go -func TestMain(m *testing.M) { - flag.Parse() - - err := testSetup() - if nil != err { - logger.ErrorWithError(err) - } - - testResults := m.Run() - - err = testTeardown() - if nil != err { - logger.ErrorWithError(err, "testTeardown failed") - } - - os.Exit(testResults) -} - -// Test basic lock primitives -func TestLockAPI(t *testing.T) { - - // TODO - how cleanup lockMap here between test runs? - testSimpleLocks(t) - testTwoThreadsExclLocking(t) - testTwoThreadsSharedLocking(t) - testTwoThreadsAndExclToShared(t) - testTwoThreadsAndSharedToExcl(t) - test100ThreadsSharedLocking(t) - test100ThreadsExclLocking(t) -} - -// Test basic WriteLock, ReadLock and Unlock -func testSimpleLocks(t *testing.T) { - assert := assert.New(t) - - myCookie := GenerateCallerID() - myRwLock := &RWLockStruct{LockID: s1, Notify: nil, LockCallerID: myCookie} - - myRwLock.WriteLock() - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - assert.Equal(IsLockHeld(s1, myCookie, WRITELOCK), true) - assert.Equal(IsLockHeld(s1, myCookie, ANYLOCK), true) - assert.Equal(IsLockHeld(s1, myCookie, READLOCK), false) - - myRwLock.Unlock() - waitCountOwners(s1, 0) - waitCountWaiters(s1, 0) - - myRwLock.ReadLock() - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - assert.Equal(IsLockHeld(s1, myCookie, WRITELOCK), false) - assert.Equal(IsLockHeld(s1, myCookie, ANYLOCK), true) - assert.Equal(IsLockHeld(s1, myCookie, READLOCK), true) - - myRwLock.Unlock() - waitCountOwners(s1, 0) - waitCountWaiters(s1, 0) - - // Try locks - err := myRwLock.TryWriteLock() - assert.Nil(err, "TryWriteLock() should work if no lock owner.") - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - - myRwLock.Unlock() - waitCountOwners(s1, 0) - waitCountWaiters(s1, 0) - - err = myRwLock.TryReadLock() - assert.Nil(err, "TryReadLock() should work if no lock owner.") - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - - myRwLock.Unlock() - waitCountOwners(s1, 0) - waitCountWaiters(s1, 0) - assert.Equal(IsLockHeld(s1, myCookie, WRITELOCK), false) - assert.Equal(IsLockHeld(s1, myCookie, ANYLOCK), false) - assert.Equal(IsLockHeld(s1, myCookie, READLOCK), false) -} - -// -// Code related to multiple test threads. -// - -// Thread which currently owns the lock. -// -// This is a map indexed by a string of the inodeNumber -var currentLockOwner map[string]uint64 - -// Map of threads and channels used for communication -var threadMap map[uint64]*threadInfo - -// Setup thread stuctures based on number of threads test wants -func setupThreadMap(threadCount uint64) { - threadMap = make(map[uint64]*threadInfo) - currentLockOwner = make(map[string]uint64) - - for i := uint64(0); i < threadCount; i++ { - thread := &threadInfo{startedNode: make(chan bool), requestForThread: make(chan *testReq)} - threadMap[i] = thread - } -} - -func setupThreads(threadCount uint64) { - setupThreadMap(threadCount) - - // Start threads and wait for them - for i := range threadMap { - go threadNode(i) - _ = <-threadMap[i].startedNode - } -} - -func stopThreads(t *testing.T) { - for i := range threadMap { - sendRequestToThread(i, t, stopThread, s1) - } -} - -// Test thread. Just waits on channel and does operation requested. -func threadNode(threadID uint64) { - - // Tell control thread we are up and set channel to read. - threadMap[threadID].startedNode <- true - var request chan *testReq - request = threadMap[threadID].requestForThread - - var myLockMap map[string]*RWLockStruct - myLockMap = make(map[string]*RWLockStruct) - - // Get cookie which track this go routine - myCookie := GenerateCallerID() - - // Wait for a command - for { - lockRequest := <-request - assert := assert.New(lockRequest.t) - assert.NotNil(lockRequest.lockID) - - switch lockRequest.typ { - case stopThread: - return - - case writeLock: - // Init lock structure and add to map - // assert := assert.New(lockRequest.t) - myRwLock := &RWLockStruct{LockID: lockRequest.lockID, Notify: nil, LockCallerID: myCookie} - myLockMap[lockRequest.lockID] = myRwLock - - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, WRITELOCK), false) - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, ANYLOCK), false) - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, READLOCK), false) - - err := myRwLock.WriteLock() - assert.Nil(err, "No error from WriteLock().") - - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, WRITELOCK), true) - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, ANYLOCK), true) - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, READLOCK), false) - - mutex.Lock() - currentLockOwner[lockRequest.lockID] = threadID - mutex.Unlock() - - case readLock: - // Init lock structure and add to map - myRwLock := &RWLockStruct{LockID: lockRequest.lockID, Notify: nil, LockCallerID: myCookie} - myLockMap[lockRequest.lockID] = myRwLock - - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, WRITELOCK), false) - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, ANYLOCK), false) - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, READLOCK), false) - - err := myRwLock.ReadLock() - assert.Nil(err, "No error from ReadLock().") - - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, WRITELOCK), false) - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, ANYLOCK), true) - assert.Equal(IsLockHeld(lockRequest.lockID, myCookie, READLOCK), true) - - mutex.Lock() - currentLockOwner[lockRequest.lockID] = threadID - mutex.Unlock() - - case tryWriteLock: - // Init lock structure and add to map - myRwLock := &RWLockStruct{LockID: lockRequest.lockID, Notify: nil, LockCallerID: myCookie} - myLockMap[lockRequest.lockID] = myRwLock - - err := myRwLock.TryWriteLock() - if err != nil { - assert.True(blunder.Is(err, blunder.TryAgainError)) - - mutex.Lock() - currentLockOwner[lockRequest.lockID] = threadID - mutex.Unlock() - } - - case tryReadLock: - // Init lock structure and add to map - myRwLock := &RWLockStruct{LockID: lockRequest.lockID, Notify: nil, LockCallerID: myCookie} - myLockMap[lockRequest.lockID] = myRwLock - - err := myRwLock.TryReadLock() - if err != nil { - assert.True(blunder.Is(err, blunder.TryAgainError)) - - mutex.Lock() - currentLockOwner[lockRequest.lockID] = threadID - mutex.Unlock() - } - - case unlock: - // Lookup lock in map - myRwLock := myLockMap[lockRequest.lockID] - assert.NotNil(myRwLock) - assert.NotNil(myRwLock.LockID) - assert.Equal(IsLockHeld(myRwLock.LockID, myCookie, ANYLOCK), true) - - err := myRwLock.Unlock() - assert.Nil(err, "No error from Unlock()().") - - delete(myLockMap, lockRequest.lockID) - - // We do not clear currentLockOwner here since that would be a race condition with a thread - // which has just been granted the lock. It is possible that it could grab the lock and set - // currentLockOwner before we can grab the mutex. - } - } -} - -func sendRequestToThread(threadID uint64, t *testing.T, operation testOpTyp, lockID string) { - request := &testReq{typ: operation, lockID: lockID, t: t} - threadMap[threadID].requestForThread <- request - - // We do not wait until the operation completes before returning. -} - -// Test that two threads can grab a lock *exclusive* and the second thread -// only gets lock after first one has done Unlock(). -func testTwoThreadsExclLocking(t *testing.T) { - var numThreads uint64 = 2 - - // Initialize worker threads - setupThreads(numThreads) - - // Lock *exclusive* from thread 0 and wait until lock is owned. - sendRequestToThread(0, t, writeLock, s1) - waitCountOwners(s1, 1) - - // Send *exclusive* from thread 1, this will block until thread 0 does unlock. - // We just wait until we see a thread waiting for the lock. - sendRequestToThread(1, t, writeLock, s1) - waitCountWaiters(s1, 1) - - // Release lock from thread 0. This should grant lock to thread 1. - sendRequestToThread(0, t, unlock, s1) - - // Block until the lock is granted to thread 1. - waitCountWaiters(s1, 0) - waitCountOwners(s1, 1) - - sendRequestToThread(1, t, unlock, s1) - waitCountWaiters(s1, 0) - waitCountOwners(s1, 0) - - // Stop worker threads - stopThreads(t) -} - -// Test that two threads can grab a lock shared. -func testTwoThreadsSharedLocking(t *testing.T) { - var numThreads uint64 = 2 - - // Initialize worker threads - setupThreads(numThreads) - - // Lock *shared* from thread 0 and wait until lock is owned. - sendRequestToThread(0, t, readLock, s1) - waitCountOwners(s1, 1) - - // Send *shared* from thread 1. This should be granted right away. - sendRequestToThread(1, t, readLock, s1) - waitCountOwners(s1, 2) - waitCountWaiters(s1, 0) - - // Release lock from thread 0. Owners should just decrease by 1. - sendRequestToThread(0, t, unlock, s1) - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - - // Have thread 0 acquire the lock again - sendRequestToThread(0, t, readLock, s1) - waitCountOwners(s1, 2) - waitCountWaiters(s1, 0) - - // Thread 1 releases the lock - sendRequestToThread(1, t, unlock, s1) - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - - // Thread 1 acquires the lock again - sendRequestToThread(1, t, readLock, s1) - waitCountOwners(s1, 2) - waitCountWaiters(s1, 0) - - // both threads release their locks - sendRequestToThread(0, t, unlock, s1) - sendRequestToThread(1, t, unlock, s1) - waitCountOwners(s1, 0) - waitCountWaiters(s1, 0) - - // Stop worker threads - stopThreads(t) -} - -// Test that 100 threads can grab a lock shared. -func test100ThreadsSharedLocking(t *testing.T) { - var numThreads uint64 = 100 - - // Initialize worker threads - setupThreads(numThreads) - - var i uint64 - for i = 0; i < numThreads; i++ { - sendRequestToThread(i, t, readLock, s1) - waitCountOwners(s1, (i + 1)) - waitCountWaiters(s1, 0) - } - - currentOwners := numThreads - for i = 0; i < numThreads; i++ { - sendRequestToThread(i, t, unlock, s1) - currentOwners-- - waitCountOwners(s1, currentOwners) - waitCountWaiters(s1, 0) - } - - // Stop worker threads - stopThreads(t) -} - -// Force 100 threads to grab the lock exclusively. The operation -// should be serialized. -func test100ThreadsExclLocking(t *testing.T) { - var numThreads uint64 = 100 - - // Initialize worker threads - setupThreads(numThreads) - - var i uint64 - for i = 0; i < numThreads; i++ { - sendRequestToThread(i, t, writeLock, s1) - waitCountOwners(s1, 1) - waitCountWaiters(s1, i) - } - waitCountWaiters(s1, numThreads-1) - - // Any of the threads could get the lock once thread0 releases it. - // - // Therefore, we have to find the thread which currently owns the lock. - // - // We do this by having the thread that grabs the lock save its threadID - // in the global currentLockOwner map. - waiters := numThreads - 1 - var prevOwner int64 = -1 - for i = 0; i < numThreads; i++ { - - waitForLockAcquire: - mutex.Lock() - owner, ok := currentLockOwner[s1] - if !ok { - mutex.Unlock() - // Lock is not yet held, let us wait - time.Sleep(5 * time.Millisecond) - goto waitForLockAcquire - } - mutex.Unlock() - - if int64(owner) == prevOwner { - // We ran before the next caller is able to grab the lock. Let us wait and retry again: - time.Sleep(5 * time.Millisecond) - goto waitForLockAcquire - } - - prevOwner = int64(owner) - sendRequestToThread(owner, t, unlock, s1) - - // Wait until next thread picks up lock - if waiters > 0 { - waiters-- - waitCountWaiters(s1, waiters) - } - } - waitCountWaiters(s1, 0) - waitCountOwners(s1, 0) - - // Stop worker threads - stopThreads(t) -} - -// Test that if one thread grabs the lock exclusive and a second thread attempts -// to grab shared, the shared lock request is granted after the first thread unlocks. -func testTwoThreadsAndExclToShared(t *testing.T) { - var numThreads uint64 = 2 - - // Initialize worker threads - setupThreads(numThreads) - - // Lock *exclusive* from thread 0 and wait until lock is owned. - sendRequestToThread(0, t, writeLock, s1) - waitCountOwners(s1, 1) - - // Send *shared* from thread 1, this will block until thread 0 does unlock. - // We just wait until we see a thread waiting for the lock. - sendRequestToThread(1, t, readLock, s1) - waitCountWaiters(s1, 1) - - // Release lock from thread 0. This should grant lock to thread 1. - sendRequestToThread(0, t, unlock, s1) - - // Block until the lock is granted to thread 1. - waitCountWaiters(s1, 0) - waitCountOwners(s1, 1) - - sendRequestToThread(1, t, unlock, s1) - waitCountWaiters(s1, 0) - waitCountOwners(s1, 0) - - // Stop worker threads - stopThreads(t) -} - -// Test that if one thread grabs the lock shared and a second thread attempts -// to grab exclusive, the exclusive lock request is granted after the first thread unlocks. -func testTwoThreadsAndSharedToExcl(t *testing.T) { - var numThreads uint64 = 2 - - // Initialize worker threads - setupThreads(numThreads) - - // Lock *exclusive* from thread 0 and wait until lock is owned. - sendRequestToThread(0, t, readLock, s1) - waitCountOwners(s1, 1) - - // Send *shared* from thread 1, this will block until thread 0 does unlock. - // We just wait until we see a thread waiting for the lock. - sendRequestToThread(1, t, writeLock, s1) - waitCountWaiters(s1, 1) - - // Release lock from thread 0. This should grant lock to thread 1. - sendRequestToThread(0, t, unlock, s1) - - // Block until the lock is granted to thread 1. - waitCountWaiters(s1, 0) - waitCountOwners(s1, 1) - - sendRequestToThread(1, t, unlock, s1) - waitCountWaiters(s1, 0) - waitCountOwners(s1, 0) - - // Stop worker threads - stopThreads(t) -} - -// Test that if lock is held *exclusive* TryWriteLock() and TryReadLock() fail. -func testTryFailsIfHeldExclusive(t *testing.T) { - var numThreads uint64 = 2 - - // Initialize worker threads - setupThreads(numThreads) - - // Lock *exclusive* from thread 0 and wait until lock is owned. - sendRequestToThread(0, t, writeLock, s1) - waitCountOwners(s1, 1) - mutex.Lock() - lockOwner := currentLockOwner[s1] - mutex.Unlock() - assert := assert.New(t) - assert.Equal(lockOwner, 0, "Lock should be owned by thread 0.") - - // Try write lock which should fail and there should not be any waiters. - sendRequestToThread(1, t, tryWriteLock, s1) - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - mutex.Lock() - lockOwner = currentLockOwner[s1] - mutex.Unlock() - assert.Equal(lockOwner, 0, "Lock should be owned by thread 0.") - - // Try read lock which should fail and there should not be any waiters. - sendRequestToThread(1, t, tryReadLock, s1) - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - mutex.Lock() - lockOwner = currentLockOwner[s1] - mutex.Unlock() - assert.Equal(lockOwner, 0, "Lock should be owned by thread 0.") - - // Release lock from thread 0. Owners should now be 0. - sendRequestToThread(0, t, unlock, s1) - waitCountOwners(s1, 0) - waitCountWaiters(s1, 0) - - // Now try write lock from thread 1. This should work. - sendRequestToThread(1, t, tryWriteLock, s1) - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - mutex.Lock() - lockOwner = currentLockOwner[s1] - mutex.Unlock() - assert.Equal(lockOwner, 1, "Lock should be owned by thread 1.") - - // Unlock from thread 1 and try a read lock from thread 1. This should work. - sendRequestToThread(1, t, unlock, s1) - waitCountOwners(s1, 0) - waitCountWaiters(s1, 0) - - sendRequestToThread(1, t, tryReadLock, s1) - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - mutex.Lock() - lockOwner = currentLockOwner[s1] - mutex.Unlock() - assert.Equal(lockOwner, 1, "Lock should be owned by thread 1.") - - // A try of a write lock from thread 0 should fail if the lock is held shared by thread 1. - sendRequestToThread(0, t, tryWriteLock, s1) - waitCountOwners(s1, 1) - waitCountWaiters(s1, 0) - mutex.Lock() - lockOwner = currentLockOwner[s1] - mutex.Unlock() - assert.Equal(lockOwner, 1, "Lock should be owned by thread 1.") - - // Release the lock - sendRequestToThread(1, t, unlock, s1) - waitCountWaiters(s1, 0) - waitCountOwners(s1, 0) - - // Stop worker threads - stopThreads(t) -} diff --git a/emswift/Makefile b/emswift/Makefile deleted file mode 100644 index 1deda501..00000000 --- a/emswift/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/emswift - -include ../GoMakefile diff --git a/emswift/dummy_test.go b/emswift/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/emswift/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/emswift/emswift.conf b/emswift/emswift.conf deleted file mode 100644 index d36206bf..00000000 --- a/emswift/emswift.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -[EMSWIFT] -AuthIPAddr: 127.0.0.1 # If AuthIPAddr left blank or missing, -AuthTCPPort: 8080 # AuthTCPPort is ignored and no AuthEmulator is launched - -JRPCIPAddr: 127.0.0.1 # If AuthIPAddr left blank or missing, -JRPCTCPPort: 12345 # JRPC{IPAddr|TCPPort} are ignored - -NoAuthIPAddr: 127.0.0.1 -NoAuthTCPPort: 8090 - -MaxAccountNameLength: 256 -MaxContainerNameLength: 256 -MaxObjectNameLength: 1024 -AccountListingLimit: 10000 -ContainerListingLimit: 10000 diff --git a/emswift/emswiftpkg/Makefile b/emswift/emswiftpkg/Makefile deleted file mode 100644 index ee8ca60d..00000000 --- a/emswift/emswiftpkg/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/emswift/emswiftpkg - -include ../../GoMakefile diff --git a/emswift/emswiftpkg/api.go b/emswift/emswiftpkg/api.go deleted file mode 100644 index 308cbd9d..00000000 --- a/emswift/emswiftpkg/api.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package emswiftpkg - -import ( - "github.com/NVIDIA/proxyfs/conf" -) - -// Start is called to start serving the NoAuth Swift Proxy Port and, -// optionally, the Auth Swift Proxy Port -// -func Start(confMap conf.ConfMap) (err error) { - err = start(confMap) - return -} - -// Stop is called to stop serving -// -func Stop() (err error) { - err = stop() - return -} diff --git a/emswift/emswiftpkg/impl.go b/emswift/emswiftpkg/impl.go deleted file mode 100644 index 34296a62..00000000 --- a/emswift/emswiftpkg/impl.go +++ /dev/null @@ -1,1586 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package emswiftpkg - -import ( - "fmt" - "io/ioutil" - "math/rand" - "net" - "net/http" - "strconv" - "strings" - "sync" - "syscall" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/utils" -) - -type swiftAccountStruct struct { - name string - headers http.Header - swiftContainerTree sortedmap.LLRBTree // key is swiftContainerStruct.name; value is *swiftContainerStruct -} - -type swiftContainerStruct struct { - name string - swiftAccount *swiftAccountStruct // back-reference to swiftAccountStruct - headers http.Header - swiftObjectTree sortedmap.LLRBTree // key is swiftObjectStruct.name; value is *swiftObjectStruct -} - -type swiftObjectStruct struct { - name string - swiftContainer *swiftContainerStruct // back-reference to swiftContainerStruct - headers http.Header - contents []byte -} - -type configStruct struct { - AuthIPAddr string // Only required if Auth Swift Proxy enabled - AuthTCPPort uint16 // Only required if Auth Swift Proxy enabled - - JRPCIPAddr string // Only required if Auth Swift Proxy enabled - JRPCTCPPort uint16 // Only required if Auth Swift Proxy enabled - - NoAuthIPAddr string - NoAuthTCPPort uint16 - - MaxAccountNameLength uint64 - MaxContainerNameLength uint64 - MaxObjectNameLength uint64 - AccountListingLimit uint64 - ContainerListingLimit uint64 -} - -type authEmulatorStruct struct { - sync.Mutex - httpServer *http.Server - resolvedJRPCTCPAddr *net.TCPAddr - wg sync.WaitGroup -} - -type noAuthEmulatorStruct struct { - sync.Mutex - httpServer *http.Server - wg sync.WaitGroup -} - -type globalsStruct struct { - config configStruct - authEmulator *authEmulatorStruct - noAuthEmulator *noAuthEmulatorStruct - swiftAccountMap map[string]*swiftAccountStruct // key is swiftAccountStruct.name; value is *swiftAccountStruct -} - -var globals globalsStruct - -const ( - startGETInfoMaxRetries = 10 - startGETInfoRetryDelay = 100 * time.Millisecond -) - -const ( - fixedAuthToken = "AUTH_tk0123456789abcde0123456789abcdef0" - fixedUserToAccountPrefix = "AUTH_" // Prefixed to User truncated before colon (":") if necessary -) - -type jrpcRequestStruct struct { - JSONrpc string `json:"jsonrpc"` - Method string `json:"method"` - ID uint64 `json:"id"` - Params [1]interface{} `json:"params"` -} - -type jrpcRequestEmptyParamStruct struct{} - -type jrpcResponseIDAndErrorStruct struct { - ID uint64 `json:"id"` - Error string `json:"error"` -} - -type jrpcResponseNoErrorStruct struct { - ID uint64 `json:"id"` - Result interface{} `json:"result"` -} - -type httpRequestHandler struct{} - -type rangeStruct struct { - startOffset uint64 - stopOffset uint64 -} - -type stringSet map[string]bool - -var headerNameIgnoreSet = stringSet{"Accept": true, "Accept-Encoding": true, "User-Agent": true, "Content-Length": true} - -func start(confMap conf.ConfMap) (err error) { - err = initializeGlobals(confMap) - if nil != err { - return - } - - err = startNoAuth() - if nil != err { - return - } - - err = startAuthIfRequested() - if nil != err { - return - } - - return -} - -func stop() (err error) { - err = stopAuthIfRequested() - if nil != err { - return - } - - err = stopNoAuth() - if nil != err { - return - } - - uninitializeGlobals() - - err = nil - return -} - -func initializeGlobals(confMap conf.ConfMap) (err error) { - globals.config.AuthIPAddr, err = confMap.FetchOptionValueString("EMSWIFT", "AuthIPAddr") - if nil == err { - globals.config.AuthTCPPort, err = confMap.FetchOptionValueUint16("EMSWIFT", "AuthTCPPort") - if nil != err { - return - } - - globals.config.JRPCIPAddr, err = confMap.FetchOptionValueString("EMSWIFT", "JRPCIPAddr") - if nil != err { - return - } - globals.config.JRPCTCPPort, err = confMap.FetchOptionValueUint16("EMSWIFT", "JRPCTCPPort") - if nil != err { - return - } - } else { - err = nil - globals.config.AuthIPAddr = "" - globals.config.AuthTCPPort = 0 - } - - globals.config.NoAuthIPAddr, err = confMap.FetchOptionValueString("EMSWIFT", "NoAuthIPAddr") - if nil != err { - return - } - globals.config.NoAuthTCPPort, err = confMap.FetchOptionValueUint16("EMSWIFT", "NoAuthTCPPort") - if nil != err { - return - } - - globals.config.MaxAccountNameLength, err = confMap.FetchOptionValueUint64("EMSWIFT", "MaxAccountNameLength") - if nil != err { - return - } - globals.config.MaxContainerNameLength, err = confMap.FetchOptionValueUint64("EMSWIFT", "MaxContainerNameLength") - if nil != err { - return - } - globals.config.MaxObjectNameLength, err = confMap.FetchOptionValueUint64("EMSWIFT", "MaxObjectNameLength") - if nil != err { - return - } - globals.config.AccountListingLimit, err = confMap.FetchOptionValueUint64("EMSWIFT", "AccountListingLimit") - if nil != err { - return - } - globals.config.ContainerListingLimit, err = confMap.FetchOptionValueUint64("EMSWIFT", "ContainerListingLimit") - if nil != err { - return - } - - globals.authEmulator = nil - globals.noAuthEmulator = nil - - globals.swiftAccountMap = make(map[string]*swiftAccountStruct) - - return -} - -func uninitializeGlobals() { - globals.config.AuthIPAddr = "" - globals.config.AuthTCPPort = 0 - - globals.config.NoAuthIPAddr = "" - globals.config.NoAuthTCPPort = 0 - - globals.config.MaxAccountNameLength = 0 - globals.config.MaxContainerNameLength = 0 - globals.config.MaxObjectNameLength = 0 - globals.config.AccountListingLimit = 0 - globals.config.ContainerListingLimit = 0 - - globals.authEmulator = nil - globals.noAuthEmulator = nil - - globals.swiftAccountMap = make(map[string]*swiftAccountStruct) -} - -func startAuthIfRequested() (err error) { - var ( - authEmulator *authEmulatorStruct - startGETInfoNumRetries int - ) - - if "" == globals.config.AuthIPAddr { - globals.authEmulator = nil - err = nil - return - } - - authEmulator = &authEmulatorStruct{ - httpServer: &http.Server{ - Addr: net.JoinHostPort(globals.config.AuthIPAddr, fmt.Sprintf("%d", globals.config.AuthTCPPort)), - }, - } - authEmulator.httpServer.Handler = authEmulator - - authEmulator.resolvedJRPCTCPAddr, err = net.ResolveTCPAddr("tcp", net.JoinHostPort(globals.config.JRPCIPAddr, fmt.Sprintf("%d", globals.config.JRPCTCPPort))) - if nil != err { - return - } - - authEmulator.wg.Add(1) - - globals.authEmulator = authEmulator - - go func() { - _ = globals.authEmulator.httpServer.ListenAndServe() - globals.authEmulator.wg.Done() - }() - - startGETInfoNumRetries = 0 - - for { - _, err = http.Get("http://" + globals.authEmulator.httpServer.Addr + "/info") - if nil == err { - break - } - startGETInfoNumRetries++ - if startGETInfoNumRetries > startGETInfoMaxRetries { - _ = stopAuthIfRequested() - err = fmt.Errorf("startAuthIfRequested() failed to establish that authEmulator is up") - return - } - time.Sleep(startGETInfoRetryDelay) - } - - err = nil - return -} - -func stopAuthIfRequested() (err error) { - if nil == globals.authEmulator { - err = nil - return - } - - err = globals.authEmulator.httpServer.Close() - if nil != err { - return - } - - globals.authEmulator.wg.Wait() - - globals.authEmulator = nil - - return -} - -func startNoAuth() (err error) { - var ( - noAuthEmulator *noAuthEmulatorStruct - startGETInfoNumRetries int - ) - - noAuthEmulator = &noAuthEmulatorStruct{ - httpServer: &http.Server{ - Addr: net.JoinHostPort(globals.config.NoAuthIPAddr, fmt.Sprintf("%d", globals.config.NoAuthTCPPort)), - }, - } - noAuthEmulator.httpServer.Handler = noAuthEmulator - - noAuthEmulator.wg.Add(1) - - globals.noAuthEmulator = noAuthEmulator - - go func() { - _ = globals.noAuthEmulator.httpServer.ListenAndServe() - globals.noAuthEmulator.wg.Done() - }() - - startGETInfoNumRetries = 0 - - for { - _, err = http.Get("http://" + globals.noAuthEmulator.httpServer.Addr + "/info") - if nil == err { - break - } - startGETInfoNumRetries++ - if startGETInfoNumRetries > startGETInfoMaxRetries { - _ = stopNoAuth() - err = fmt.Errorf("startNoAuth() failed to establish that noAuthEmulator is up") - return - } - time.Sleep(startGETInfoRetryDelay) - } - - err = nil - return -} - -func stopNoAuth() (err error) { - err = globals.noAuthEmulator.httpServer.Close() - if nil != err { - return - } - - globals.noAuthEmulator.wg.Wait() - - globals.noAuthEmulator = nil - - return -} - -func (dummy *authEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - var ( - noAuthPath string - xAuthKey string - xAuthUser string - xAuthUserSplit2OnColon []string - xStorageURL string - ) - - // Handle the GET of/on info & AuthURL cases - - if http.MethodGet == request.Method { - switch request.URL.Path { - case "/info": - _ = request.Body.Close() - doNoAuthGET(responseWriter, request) - return - case "/auth/v1.0": - _ = request.Body.Close() - xAuthUser = request.Header.Get("X-Auth-User") - xAuthKey = request.Header.Get("X-Auth-Key") - if ("" == xAuthUser) || ("" == xAuthKey) { - responseWriter.WriteHeader(http.StatusUnauthorized) - return - } - xAuthUserSplit2OnColon = strings.SplitN(xAuthUser, ":", 2) - xStorageURL = "http://" + globals.authEmulator.httpServer.Addr + "/v1/" + fixedUserToAccountPrefix + xAuthUserSplit2OnColon[0] - responseWriter.Header().Add("X-Auth-Token", fixedAuthToken) - responseWriter.Header().Add("X-Storage-Url", xStorageURL) - return - default: - // Fall through to normal processing - } - } - - // Require X-Auth-Token to match fixedAuthToken - - if fixedAuthToken != request.Header.Get("X-Auth-Token") { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusUnauthorized) - return - } - - // Require "version" portion of request.URL.Path to be "proxyfs" - - if !strings.HasPrefix(request.URL.Path, "/proxyfs/") { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - // Branch off to individual request method handlers - - switch request.Method { - case http.MethodGet: - noAuthPath = strings.Replace(request.URL.Path, "proxyfs", "v1", 1) - request.URL.Path = noAuthPath - doNoAuthGET(responseWriter, request) - case http.MethodPut: - noAuthPath = strings.Replace(request.URL.Path, "proxyfs", "v1", 1) - request.URL.Path = noAuthPath - doNoAuthPUT(responseWriter, request) - case "PROXYFS": - doAuthPROXYFS(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func doAuthPROXYFS(responseWriter http.ResponseWriter, request *http.Request) { - var ( - bytesWritten int - bytesWrittenInTotal int - err error - jrpcRequest []byte - jrpcResponse []byte - jrpcResponseNestedLeftBraces uint32 - jrpcResponseSingleByte []byte - jrpcTCPConn *net.TCPConn - ) - - globals.authEmulator.Lock() - defer globals.authEmulator.Unlock() - - jrpcRequest, err = ioutil.ReadAll(request.Body) - if nil != err { - panic(err) - } - _ = request.Body.Close() - - jrpcTCPConn, err = net.DialTCP("tcp", nil, globals.authEmulator.resolvedJRPCTCPAddr) - if nil != err { - panic(err) - } - - bytesWrittenInTotal = 0 - - for bytesWrittenInTotal < len(jrpcRequest) { - bytesWritten, err = jrpcTCPConn.Write(jrpcRequest[bytesWrittenInTotal:]) - if nil != err { - panic(err) - } - bytesWrittenInTotal += bytesWritten - } - - jrpcResponse = make([]byte, 0) - jrpcResponseSingleByte = make([]byte, 1) - - _, err = jrpcTCPConn.Read(jrpcResponseSingleByte) - if nil != err { - panic(err) - } - jrpcResponse = append(jrpcResponse, jrpcResponseSingleByte[0]) - - if '{' != jrpcResponseSingleByte[0] { - err = fmt.Errorf("Opening character of jrpcResponse must be '{'") - panic(err) - } - - jrpcResponseNestedLeftBraces = 1 - - for 0 < jrpcResponseNestedLeftBraces { - _, err = jrpcTCPConn.Read(jrpcResponseSingleByte) - if nil != err { - panic(err) - } - jrpcResponse = append(jrpcResponse, jrpcResponseSingleByte[0]) - - switch jrpcResponseSingleByte[0] { - case '{': - jrpcResponseNestedLeftBraces++ - case '}': - jrpcResponseNestedLeftBraces-- - default: - // Nothing special to do here - } - } - - err = jrpcTCPConn.Close() - if nil != err { - panic(err) - } - - responseWriter.Header().Add("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(jrpcResponse) -} - -func (dummy *noAuthEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - switch request.Method { - case http.MethodDelete: - doNoAuthDELETE(responseWriter, request) - case http.MethodGet: - doNoAuthGET(responseWriter, request) - case http.MethodHead: - doNoAuthHEAD(responseWriter, request) - case http.MethodPost: - doNoAuthPOST(responseWriter, request) - case http.MethodPut: - doNoAuthPUT(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func (dummy *globalsStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsString = fmt.Sprintf("%v", key) - err = nil - return -} - -func (dummy *globalsStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - valueAsString = fmt.Sprintf("%v", value) - err = nil - return -} - -func parsePath(request *http.Request) (infoOnly bool, swiftAccountName string, swiftContainerName string, swiftObjectName string) { - var ( - pathSplit []string - ) - - infoOnly = false - swiftAccountName = "" - swiftContainerName = "" - swiftObjectName = "" - - if "/info" == request.URL.Path { - infoOnly = true - return - } - - if strings.HasPrefix(request.URL.Path, "/v1/") { - pathSplit = strings.SplitN(request.URL.Path[4:], "/", 3) - swiftAccountName = pathSplit[0] - if 1 == len(pathSplit) { - swiftContainerName = "" - swiftObjectName = "" - } else { - swiftContainerName = pathSplit[1] - if 2 == len(pathSplit) { - swiftObjectName = "" - } else { - swiftObjectName = pathSplit[2] - } - } - } - - return -} - -func parseRangeHeader(request *http.Request, objectLen int) (ranges []rangeStruct, err error) { - var ( - off int - rangeHeaderValue string - rangeHeaderValueSuffix string - rangeString string - rangeStringSlice []string - rangesStrings []string - rangesStringsIndex int - startOffset int64 - stopOffset int64 - ) - - rangeHeaderValue = request.Header.Get("Range") - if "" == rangeHeaderValue { - ranges = make([]rangeStruct, 0) - err = nil - return - } - - if !strings.HasPrefix(rangeHeaderValue, "bytes=") { - err = fmt.Errorf("rangeHeaderValue (%v) does not start with expected \"bytes=\"", rangeHeaderValue) - return - } - - rangeHeaderValueSuffix = rangeHeaderValue[len("bytes="):] - - rangesStrings = strings.SplitN(rangeHeaderValueSuffix, ",", 2) - - ranges = make([]rangeStruct, len(rangesStrings)) - - for rangesStringsIndex, rangeString = range rangesStrings { - rangeStringSlice = strings.SplitN(rangeString, "-", 2) - if 2 != len(rangeStringSlice) { - err = fmt.Errorf("rangeHeaderValue (%v) malformed", rangeHeaderValue) - return - } - if "" == rangeStringSlice[0] { - startOffset = int64(-1) - } else { - off, err = strconv.Atoi(rangeStringSlice[0]) - if nil != err { - err = fmt.Errorf("rangeHeaderValue (%v) malformed (strconv.Atoi() failure: %v)", rangeHeaderValue, err) - return - } - startOffset = int64(off) - } - - if "" == rangeStringSlice[1] { - stopOffset = int64(-1) - } else { - off, err = strconv.Atoi(rangeStringSlice[1]) - if nil != err { - err = fmt.Errorf("rangeHeaderValue (%v) malformed (strconv.Atoi() failure: %v)", rangeHeaderValue, err) - return - } - stopOffset = int64(off) - } - - if ((0 > startOffset) && (0 > stopOffset)) || (startOffset > stopOffset) { - err = fmt.Errorf("rangeHeaderValue (%v) malformed", rangeHeaderValue) - return - } - - if startOffset < 0 { - startOffset = int64(objectLen) - stopOffset - if startOffset < 0 { - err = fmt.Errorf("rangeHeaderValue (%v) malformed...computed startOffset negative", rangeHeaderValue) - return - } - stopOffset = int64(objectLen - 1) - } else if stopOffset < 0 { - stopOffset = int64(objectLen - 1) - } else { - if stopOffset > int64(objectLen-1) { - stopOffset = int64(objectLen - 1) - } - } - - ranges[rangesStringsIndex].startOffset = uint64(startOffset) - ranges[rangesStringsIndex].stopOffset = uint64(stopOffset) - } - - err = nil - return -} - -func locateSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, errno syscall.Errno) { - var ( - ok bool - ) - - swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] - if !ok { - errno = unix.ENOENT - return - } - errno = 0 - return -} - -func createSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, errno syscall.Errno) { - var ( - ok bool - ) - - _, ok = globals.swiftAccountMap[swiftAccountName] - if ok { - errno = unix.EEXIST - return - } - swiftAccount = &swiftAccountStruct{ - name: swiftAccountName, - headers: make(http.Header), - swiftContainerTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), - } - globals.swiftAccountMap[swiftAccountName] = swiftAccount - errno = 0 - return -} - -func createOrLocateSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, wasCreated bool) { - var ( - ok bool - ) - - swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] - if ok { - wasCreated = false - } else { - swiftAccount = &swiftAccountStruct{ - name: swiftAccountName, - headers: make(http.Header), - swiftContainerTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), - } - globals.swiftAccountMap[swiftAccountName] = swiftAccount - wasCreated = true - } - return -} - -func deleteSwiftAccount(swiftAccountName string, force bool) (errno syscall.Errno) { - var ( - err error - ok bool - swiftAccount *swiftAccountStruct - swiftswiftAccountContainerCount int - ) - - swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] - if ok { - if force { - // ok if account contains data... we'll forget it - } else { - swiftswiftAccountContainerCount, err = swiftAccount.swiftContainerTree.Len() - if nil != err { - panic(err) - } - if 0 != swiftswiftAccountContainerCount { - errno = unix.ENOTEMPTY - return - } - } - delete(globals.swiftAccountMap, swiftAccountName) - } else { - errno = unix.ENOENT - return - } - errno = 0 - return -} - -func locateSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, errno syscall.Errno) { - var ( - err error - ok bool - swiftContainerAsValue sortedmap.Value - ) - - swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) - if nil != err { - panic(err) - } - if ok { - swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) - } else { - errno = unix.ENOENT - return - } - errno = 0 - return -} - -func createSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, errno syscall.Errno) { - var ( - err error - ok bool - ) - - _, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) - if nil != err { - panic(err) - } - if ok { - errno = unix.EEXIST - return - } else { - swiftContainer = &swiftContainerStruct{ - name: swiftContainerName, - swiftAccount: swiftAccount, - headers: make(http.Header), - swiftObjectTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), - } - _, err = swiftAccount.swiftContainerTree.Put(swiftContainerName, swiftContainer) - if nil != err { - panic(err) - } - } - errno = 0 - return -} - -func createOrLocateSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, wasCreated bool) { - var ( - err error - ok bool - swiftContainerAsValue sortedmap.Value - ) - - swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) - if nil != err { - panic(err) - } - if ok { - swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) - wasCreated = false - } else { - swiftContainer = &swiftContainerStruct{ - name: swiftContainerName, - swiftAccount: swiftAccount, - headers: make(http.Header), - swiftObjectTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), - } - _, err = swiftAccount.swiftContainerTree.Put(swiftContainerName, swiftContainer) - if nil != err { - panic(err) - } - wasCreated = true - } - return -} - -func deleteSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (errno syscall.Errno) { - var ( - err error - ok bool - swiftContainer *swiftContainerStruct - swiftContainerAsValue sortedmap.Value - swiftContainerObjectCount int - ) - - swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) - if nil != err { - panic(err) - } - if ok { - swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) - swiftContainerObjectCount, err = swiftContainer.swiftObjectTree.Len() - if nil != err { - panic(err) - } - if 0 != swiftContainerObjectCount { - errno = unix.ENOTEMPTY - return - } - _, err = swiftAccount.swiftContainerTree.DeleteByKey(swiftContainerName) - if nil != err { - panic(err) - } - } else { - errno = unix.ENOENT - return - } - errno = 0 - return -} - -func locateSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, errno syscall.Errno) { - var ( - err error - ok bool - swiftObjectAsValue sortedmap.Value - ) - - swiftObjectAsValue, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) - if nil != err { - panic(err) - } - if ok { - swiftObject = swiftObjectAsValue.(*swiftObjectStruct) - } else { - errno = unix.ENOENT - return - } - errno = 0 - return -} - -func createSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, errno syscall.Errno) { - var ( - err error - ok bool - ) - - _, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) - if nil != err { - panic(err) - } - if ok { - errno = unix.EEXIST - return - } else { - swiftObject = &swiftObjectStruct{name: swiftObjectName, swiftContainer: swiftContainer, contents: []byte{}} - _, err = swiftContainer.swiftObjectTree.Put(swiftObjectName, swiftObject) - if nil != err { - panic(err) - } - } - errno = 0 - return -} - -func createOrLocateSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, wasCreated bool) { - var ( - err error - ok bool - swiftObjectAsValue sortedmap.Value - ) - - swiftObjectAsValue, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) - if nil != err { - panic(err) - } - if ok { - swiftObject = swiftObjectAsValue.(*swiftObjectStruct) - wasCreated = false - } else { - swiftObject = &swiftObjectStruct{name: swiftObjectName, swiftContainer: swiftContainer, contents: []byte{}} - _, err = swiftContainer.swiftObjectTree.Put(swiftObjectName, swiftObject) - if nil != err { - panic(err) - } - wasCreated = true - } - return -} - -func deleteSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (errno syscall.Errno) { - var ( - err error - ok bool - ) - - _, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) - if nil != err { - panic(err) - } - if ok { - _, err = swiftContainer.swiftObjectTree.DeleteByKey(swiftObjectName) - if nil != err { - panic(err) - } - } else { - errno = unix.ENOENT - return - } - errno = 0 - return -} - -func doNoAuthDELETE(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - errno syscall.Errno - infoOnly bool - swiftAccount *swiftAccountStruct - swiftAccountName string - swiftContainer *swiftContainerStruct - swiftContainerName string - swiftObjectName string - ) - - globals.noAuthEmulator.Lock() - defer globals.noAuthEmulator.Unlock() - - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) - if infoOnly || ("" == swiftAccountName) { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - if "" == swiftContainerName { - // DELETE SwiftAccount - errno = deleteSwiftAccount(swiftAccountName, false) - switch errno { - case 0: - responseWriter.WriteHeader(http.StatusNoContent) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - case unix.ENOTEMPTY: - responseWriter.WriteHeader(http.StatusConflict) - default: - err = fmt.Errorf("deleteSwiftAccount(\"%v\", false) returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } else { - // DELETE SwiftContainer or SwiftObject - swiftAccount, errno = locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftObjectName { - // DELETE SwiftContainer - errno = deleteSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - responseWriter.WriteHeader(http.StatusNoContent) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - case unix.ENOTEMPTY: - responseWriter.WriteHeader(http.StatusConflict) - default: - err = fmt.Errorf("deleteSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } else { - // DELETE SwiftObject - swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - errno = deleteSwiftObject(swiftContainer, swiftObjectName) - switch errno { - case 0: - responseWriter.WriteHeader(http.StatusNoContent) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("deleteSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) - panic(err) - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } - } -} - -func doNoAuthGET(responseWriter http.ResponseWriter, request *http.Request) { - var ( - boundaryString string - containerIndex int - containerIndexLimit int - err error - errno syscall.Errno - found bool - headerName string - headerValue string - headerValueSlice []string - infoOnly bool - marker string - markerSlice []string - objectIndex int - objectIndexLimit int - ok bool - numContainers int - numObjects int - ranges []rangeStruct - rS rangeStruct - swiftAccount *swiftAccountStruct - swiftAccountName string - swiftContainer *swiftContainerStruct - swiftContainerName string - swiftContainerNameAsKey sortedmap.Key - swiftObject *swiftObjectStruct - swiftObjectName string - swiftObjectNameAsKey sortedmap.Key - ) - - globals.noAuthEmulator.Lock() - defer globals.noAuthEmulator.Unlock() - - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) - if infoOnly { - _, _ = responseWriter.Write(utils.StringToByteSlice("{")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"swift\": {")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_account_name_length\": " + strconv.Itoa(int(globals.config.MaxAccountNameLength)) + ",")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_container_name_length\": " + strconv.Itoa(int(globals.config.MaxContainerNameLength)) + ",")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_object_name_length\": " + strconv.Itoa(int(globals.config.MaxObjectNameLength)) + ",")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"account_listing_limit\": " + strconv.Itoa(int(globals.config.AccountListingLimit)) + ",")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"container_listing_limit\": " + strconv.Itoa(int(globals.config.ContainerListingLimit)))) - _, _ = responseWriter.Write(utils.StringToByteSlice("}")) - _, _ = responseWriter.Write(utils.StringToByteSlice("}")) - } else { - if "" == swiftAccountName { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - swiftAccount, errno = locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftContainerName { - // GET SwiftAccount - for headerName, headerValueSlice = range swiftAccount.headers { - for _, headerValue = range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - numContainers, err = swiftAccount.swiftContainerTree.Len() - if nil != err { - panic(err) - } - if 0 == numContainers { - responseWriter.WriteHeader(http.StatusNoContent) - } else { - marker = "" - markerSlice, ok = request.URL.Query()["marker"] - if ok && (0 < len(markerSlice)) { - marker = markerSlice[0] - } - containerIndex, found, err = swiftAccount.swiftContainerTree.BisectRight(marker) - if nil != err { - panic(err) - } - if found { - containerIndex++ - } - if containerIndex < numContainers { - containerIndexLimit = numContainers - if (containerIndexLimit - containerIndex) > int(globals.config.AccountListingLimit) { - containerIndexLimit = containerIndex + int(globals.config.AccountListingLimit) - } - for containerIndex < containerIndexLimit { - swiftContainerNameAsKey, _, _, err = swiftAccount.swiftContainerTree.GetByIndex(containerIndex) - if nil != err { - panic(err) - } - swiftContainerName = swiftContainerNameAsKey.(string) - _, _ = responseWriter.Write(utils.StringToByteSlice(swiftContainerName)) - _, _ = responseWriter.Write([]byte{'\n'}) - containerIndex++ - } - } else { - responseWriter.WriteHeader(http.StatusNoContent) - } - } - } else { - // GET SwiftContainer or SwiftObject - swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - if "" == swiftObjectName { - // GET SwiftContainer - for headerName, headerValueSlice = range swiftContainer.headers { - for _, headerValue = range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - numObjects, err = swiftContainer.swiftObjectTree.Len() - if nil != err { - panic(err) - } - if 0 == numObjects { - responseWriter.WriteHeader(http.StatusNoContent) - } else { - marker = "" - markerSlice, ok = request.URL.Query()["marker"] - if ok && (0 < len(markerSlice)) { - marker = markerSlice[0] - } - objectIndex, found, err = swiftContainer.swiftObjectTree.BisectRight(marker) - if nil != err { - panic(err) - } - if found { - objectIndex++ - } - if objectIndex < numObjects { - objectIndexLimit = numObjects - if (objectIndexLimit - objectIndex) > int(globals.config.ContainerListingLimit) { - objectIndexLimit = objectIndex + int(globals.config.ContainerListingLimit) - } - for objectIndex < objectIndexLimit { - swiftObjectNameAsKey, _, _, err = swiftContainer.swiftObjectTree.GetByIndex(objectIndex) - if nil != err { - panic(err) - } - swiftObjectName = swiftObjectNameAsKey.(string) - _, _ = responseWriter.Write(utils.StringToByteSlice(swiftObjectName)) - _, _ = responseWriter.Write([]byte{'\n'}) - objectIndex++ - } - } else { - responseWriter.WriteHeader(http.StatusNoContent) - } - } - } else { - // GET SwiftObject - swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) - switch errno { - case 0: - for headerName, headerValueSlice = range swiftObject.headers { - for _, headerValue = range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - ranges, err = parseRangeHeader(request, len(swiftObject.contents)) - if nil == err { - switch len(ranges) { - case 0: - responseWriter.Header().Add("Content-Type", "application/octet-stream") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(swiftObject.contents) - case 1: - responseWriter.Header().Add("Content-Type", "application/octet-stream") - responseWriter.Header().Add("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ranges[0].startOffset, ranges[0].stopOffset, len(swiftObject.contents))) - responseWriter.WriteHeader(http.StatusPartialContent) - _, _ = responseWriter.Write(swiftObject.contents[ranges[0].startOffset:(ranges[0].stopOffset + 1)]) - default: - boundaryString = fmt.Sprintf("%016x%016x", rand.Uint64(), rand.Uint64()) - responseWriter.Header().Add("Content-Type", fmt.Sprintf("multipart/byteranges; boundary=%v", boundaryString)) - responseWriter.WriteHeader(http.StatusPartialContent) - for _, rS = range ranges { - _, _ = responseWriter.Write([]byte("--" + boundaryString + "\r\n")) - _, _ = responseWriter.Write([]byte("Content-Type: application/octet-stream\r\n")) - _, _ = responseWriter.Write([]byte(fmt.Sprintf("Content-Range: bytes %d-%d/%d\r\n", rS.startOffset, rS.stopOffset, len(swiftObject.contents)))) - _, _ = responseWriter.Write([]byte("\r\n")) - _, _ = responseWriter.Write(swiftObject.contents[rS.startOffset:(rS.stopOffset + 1)]) - _, _ = responseWriter.Write([]byte("\r\n")) - } - _, _ = responseWriter.Write([]byte("--" + boundaryString + "--")) - } - } else { - responseWriter.WriteHeader(http.StatusBadRequest) - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } - } -} - -func doNoAuthHEAD(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - errno syscall.Errno - headerName string - headerValue string - headerValueSlice []string - infoOnly bool - swiftAccount *swiftAccountStruct - swiftAccountName string - swiftContainer *swiftContainerStruct - swiftContainerName string - swiftObject *swiftObjectStruct - swiftObjectName string - ) - - globals.noAuthEmulator.Lock() - defer globals.noAuthEmulator.Unlock() - - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) - if infoOnly || ("" == swiftAccountName) { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - swiftAccount, errno = locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftContainerName { - // HEAD SwiftAccount - for headerName, headerValueSlice = range swiftAccount.headers { - for _, headerValue = range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - responseWriter.WriteHeader(http.StatusNoContent) - } else { - // HEAD SwiftContainer or SwiftObject - swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - if "" == swiftObjectName { - // HEAD SwiftContainer - for headerName, headerValueSlice = range swiftContainer.headers { - for _, headerValue = range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - responseWriter.WriteHeader(http.StatusNoContent) - } else { - // HEAD SwiftObject - swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) - switch errno { - case 0: - for headerName, headerValueSlice = range swiftObject.headers { - for _, headerValue = range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - responseWriter.Header().Set("Content-Length", strconv.Itoa(len(swiftObject.contents))) - responseWriter.WriteHeader(http.StatusOK) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } -} - -func doNoAuthPOST(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - errno syscall.Errno - headerName string - headerValue string - headerValueSlice []string - headerValueSliceLen int - ignoreHeader bool - infoOnly bool - swiftAccount *swiftAccountStruct - swiftAccountName string - swiftContainer *swiftContainerStruct - swiftContainerName string - swiftObject *swiftObjectStruct - swiftObjectName string - ) - - globals.noAuthEmulator.Lock() - defer globals.noAuthEmulator.Unlock() - - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) - if infoOnly || ("" == swiftAccountName) { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - swiftAccount, errno = locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftContainerName { - // POST SwiftAccount - for headerName, headerValueSlice = range request.Header { - _, ignoreHeader = headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen = len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftAccount.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue = range headerValueSlice { - if 0 < len(headerValue) { - swiftAccount.headers[headerName] = append(swiftAccount.headers[headerName], headerValue) - } - } - if 0 == len(swiftAccount.headers[headerName]) { - delete(swiftAccount.headers, headerName) - } - } - } - } - responseWriter.WriteHeader(http.StatusNoContent) - } else { - // POST SwiftContainer or SwiftObject - swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - if "" == swiftObjectName { - // POST SwiftContainer - for headerName, headerValueSlice = range request.Header { - _, ignoreHeader = headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen = len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftContainer.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue = range headerValueSlice { - if 0 < len(headerValue) { - swiftContainer.headers[headerName] = append(swiftContainer.headers[headerName], headerValue) - } - } - if 0 == len(swiftContainer.headers[headerName]) { - delete(swiftContainer.headers, headerName) - } - } - } - } - responseWriter.WriteHeader(http.StatusNoContent) - } else { - // POST SwiftObject - swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) - switch errno { - case 0: - for headerName, headerValueSlice = range request.Header { - _, ignoreHeader = headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen = len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftObject.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue = range headerValueSlice { - if 0 < len(headerValue) { - swiftObject.headers[headerName] = append(swiftObject.headers[headerName], headerValue) - } - } - if 0 == len(swiftObject.headers[headerName]) { - delete(swiftObject.headers, headerName) - } - } - } - } - responseWriter.WriteHeader(http.StatusNoContent) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } -} - -func doNoAuthPUT(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - errno syscall.Errno - headerName string - headerValue string - headerValueSlice []string - headerValueSliceLen int - ignoreHeader bool - infoOnly bool - swiftAccount *swiftAccountStruct - swiftAccountName string - swiftContainer *swiftContainerStruct - swiftContainerName string - swiftObject *swiftObjectStruct - swiftObjectName string - wasCreated bool - ) - - globals.noAuthEmulator.Lock() - defer globals.noAuthEmulator.Unlock() - - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) - if infoOnly || ("" == swiftAccountName) { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - if "" == swiftContainerName { - // PUT SwiftAccount - swiftAccount, wasCreated = createOrLocateSwiftAccount(swiftAccountName) - if wasCreated { - swiftAccount.headers = make(http.Header) - } - for headerName, headerValueSlice = range request.Header { - _, ignoreHeader = headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen = len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftAccount.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue = range headerValueSlice { - if 0 < len(headerValue) { - swiftAccount.headers[headerName] = append(swiftAccount.headers[headerName], headerValue) - } - } - if 0 == len(swiftAccount.headers[headerName]) { - delete(swiftAccount.headers, headerName) - } - } - } - } - if wasCreated { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusAccepted) - } - } else { - // PUT SwiftContainer or SwiftObject - swiftAccount, errno = locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftObjectName { - // PUT SwiftContainer - swiftContainer, wasCreated = createOrLocateSwiftContainer(swiftAccount, swiftContainerName) - if wasCreated { - swiftContainer.headers = make(http.Header) - } - for headerName, headerValueSlice = range request.Header { - _, ignoreHeader = headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen = len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftContainer.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue = range headerValueSlice { - if 0 < len(headerValue) { - swiftContainer.headers[headerName] = append(swiftContainer.headers[headerName], headerValue) - } - } - if 0 == len(swiftContainer.headers[headerName]) { - delete(swiftContainer.headers, headerName) - } - } - } - } - if wasCreated { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusAccepted) - } - } else { - // PUT SwiftObject - swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - swiftObject, wasCreated = createOrLocateSwiftObject(swiftContainer, swiftObjectName) - if wasCreated { - swiftObject.headers = make(http.Header) - } - for headerName, headerValueSlice = range request.Header { - _, ignoreHeader = headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen = len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftObject.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue = range headerValueSlice { - if 0 < len(headerValue) { - swiftObject.headers[headerName] = append(swiftObject.headers[headerName], headerValue) - } - } - if 0 == len(swiftObject.headers[headerName]) { - delete(swiftObject.headers, headerName) - } - } - } - } - swiftObject.contents, _ = ioutil.ReadAll(request.Body) - if wasCreated { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusCreated) - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusForbidden) - default: - err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusForbidden) - default: - err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } - } -} diff --git a/emswift/emswiftpkg/pkg_test.go b/emswift/emswiftpkg/pkg_test.go deleted file mode 100644 index 31a3c896..00000000 --- a/emswift/emswiftpkg/pkg_test.go +++ /dev/null @@ -1,1529 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package emswiftpkg - -import ( - "bytes" - "io" - "io/ioutil" - "net" - "net/http" - "strings" - "sync" - "testing" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/utils" -) - -func TestAuthEmulation(t *testing.T) { - var ( - confMap conf.ConfMap - confStrings = []string{ - "EMSWIFT.AuthIPAddr=127.0.0.1", - "EMSWIFT.AuthTCPPort=9997", - "EMSWIFT.JRPCIPAddr=127.0.0.1", - "EMSWIFT.JRPCTCPPort=9998", - "EMSWIFT.NoAuthIPAddr=127.0.0.1", - "EMSWIFT.NoAuthTCPPort=9999", - "EMSWIFT.MaxAccountNameLength=256", - "EMSWIFT.MaxContainerNameLength=256", - "EMSWIFT.MaxObjectNameLength=1024", - "EMSWIFT.AccountListingLimit=10000", - "EMSWIFT.ContainerListingLimit=10000", - } - err error - expectedInfo string - expectedStorageURL string - httpClient *http.Client - httpRequest *http.Request - httpResponse *http.Response - readBuf []byte - testJRPCServerHandlerTCPListener *net.TCPListener - testJRPCServerHandlerWG *sync.WaitGroup - urlForAuth string - urlForInfo string - urlPrefix string - ) - - confMap, err = conf.MakeConfMapFromStrings(confStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned unexpected error: %v", err) - } - - err = Start(confMap) - if nil != err { - t.Fatalf("Start(confMap) returned unexpected error: %v", err) - } - - testJRPCServerHandlerTCPListener, err = net.ListenTCP("tcp", globals.authEmulator.resolvedJRPCTCPAddr) - if nil != err { - t.Fatalf("net.ListenTCP() returned unexpected error: %v", err) - } - - testJRPCServerHandlerWG = new(sync.WaitGroup) - testJRPCServerHandlerWG.Add(1) - go testJRPCServerHandler(testJRPCServerHandlerTCPListener, testJRPCServerHandlerWG) - - // Format URLs - - urlForInfo = "http://" + globals.authEmulator.httpServer.Addr + "/info" - urlForAuth = "http://" + globals.authEmulator.httpServer.Addr + "/auth/v1.0" - urlPrefix = "http://" + globals.authEmulator.httpServer.Addr + "/proxyfs/" - - expectedStorageURL = "http://" + globals.authEmulator.httpServer.Addr + "/v1/" + "AUTH_test" - - // Setup http.Client that we will use for all HTTP requests - - httpClient = &http.Client{} - - // Send a GET for "/info" expecting [EMSWIFT] data in compact JSON form - - httpRequest, err = http.NewRequest("GET", urlForInfo, nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - expectedInfo = "{\"swift\": {\"max_account_name_length\": 256,\"max_container_name_length\": 256,\"max_object_name_length\": 1024,\"account_listing_limit\": 10000,\"container_listing_limit\": 10000}}" - if int64(len(expectedInfo)) != httpResponse.ContentLength { - t.Fatalf("GET of /info httpResponse.ContentLength unexpected") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if expectedInfo != utils.ByteSliceToString(readBuf) { - t.Fatalf("GET of /info httpResponse.Body contents unexpected") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for "/auth/v1.0" expecting X-Auth-Token & X-Storage-Url - - httpRequest, err = http.NewRequest("GET", urlForAuth, nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("X-Auth-User", "test:tester") - httpRequest.Header.Add("X-Auth-Key", "testing") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - if httpResponse.Header.Get("X-Auth-Token") != fixedAuthToken { - t.Fatalf("Auth response should have header X-Auth-Token: %s", fixedAuthToken) - } - if httpResponse.Header.Get("X-Storage-Url") != expectedStorageURL { - t.Fatalf("Auth response should have header X-Storage-Url: %s", expectedStorageURL) - } - - // Send a PUT for account "TestAccount" - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("X-Auth-Token", fixedAuthToken) - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" expecting Content-Length: 0 - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("X-Auth-Token", fixedAuthToken) - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PROXYFS JSON-RPC "request" that simply gets echo'd - - httpRequest, err = http.NewRequest("PROXYFS", urlPrefix+"TestAccount", bytes.NewReader([]byte{'{', 'p', 'i', 'n', 'g', '}'})) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("X-Auth-Token", fixedAuthToken) - httpRequest.Header.Add("Content-Type", "application/json") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Content-Type") != "application/json" { - t.Fatalf("TestAccount should have header Content-Type: application/json") - } - if int64(len("{ping}")) != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain only \"{ping}\"") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "{ping}" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestContainer should contain only \"{ping}\"") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - err = testJRPCServerHandlerTCPListener.Close() - if nil != err { - t.Fatalf("testJRPCServerHandlerTCPListener.Close() returned unexpected error: %v", err) - } - - testJRPCServerHandlerWG.Wait() - - err = Stop() - if nil != err { - t.Fatalf("Stop() returned unexpected error: %v", err) - } -} - -func testJRPCServerHandler(testJRPCServerHandlerTCPListener *net.TCPListener, testJRPCServerHandlerWG *sync.WaitGroup) { - var ( - bytesWritten int - bytesWrittenInTotal int - err error - jrpcRequest []byte - jrpcRequestNestedLeftBraces uint32 - jrpcRequestSingleByte []byte - jrpcResponse []byte - jrpcTCPConn *net.TCPConn - ) - -DoAcceptTCP: - jrpcTCPConn, err = testJRPCServerHandlerTCPListener.AcceptTCP() - if nil != err { - testJRPCServerHandlerWG.Done() - return - } - - jrpcRequest = make([]byte, 0) - jrpcRequestSingleByte = make([]byte, 1) - - _, err = jrpcTCPConn.Read(jrpcRequestSingleByte) - if nil != err { - _ = jrpcTCPConn.Close() - goto DoAcceptTCP - } - jrpcRequest = append(jrpcRequest, jrpcRequestSingleByte[0]) - - if '{' != jrpcRequestSingleByte[0] { - _ = jrpcTCPConn.Close() - goto DoAcceptTCP - } - - jrpcRequestNestedLeftBraces = 1 - - for 0 < jrpcRequestNestedLeftBraces { - _, err = jrpcTCPConn.Read(jrpcRequestSingleByte) - if nil != err { - _ = jrpcTCPConn.Close() - goto DoAcceptTCP - } - jrpcRequest = append(jrpcRequest, jrpcRequestSingleByte[0]) - - switch jrpcRequestSingleByte[0] { - case '{': - jrpcRequestNestedLeftBraces++ - case '}': - jrpcRequestNestedLeftBraces-- - default: - // Nothing special to do here - } - } - - jrpcResponse = jrpcRequest - - bytesWrittenInTotal = 0 - - for bytesWrittenInTotal < len(jrpcResponse) { - bytesWritten, err = jrpcTCPConn.Write(jrpcResponse[bytesWrittenInTotal:]) - if nil != err { - _ = jrpcTCPConn.Close() - goto DoAcceptTCP - } - bytesWrittenInTotal += bytesWritten - } - - _ = jrpcTCPConn.Close() - goto DoAcceptTCP -} - -func TestNoAuthEmulation(t *testing.T) { - var ( - confMap conf.ConfMap - confStrings = []string{ - "EMSWIFT.NoAuthIPAddr=127.0.0.1", - "EMSWIFT.NoAuthTCPPort=9999", - "EMSWIFT.MaxAccountNameLength=256", - "EMSWIFT.MaxContainerNameLength=256", - "EMSWIFT.MaxObjectNameLength=1024", - "EMSWIFT.AccountListingLimit=10000", - "EMSWIFT.ContainerListingLimit=10000", - } - contentType string - contentTypeMultiPartBoundary string - err error - errChan chan error - expectedBuf []byte - expectedInfo string - httpClient *http.Client - httpRequest *http.Request - httpResponse *http.Response - mouseHeaderPresent bool - pipeReader *io.PipeReader - pipeWriter *io.PipeWriter - readBuf []byte - urlForInfo string - urlPrefix string - ) - - confMap, err = conf.MakeConfMapFromStrings(confStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned unexpected error: %v", err) - } - - err = Start(confMap) - if nil != err { - t.Fatalf("Start(confMap) returned unexpected error: %v", err) - } - - // Format URLs - - urlForInfo = "http://" + globals.noAuthEmulator.httpServer.Addr + "/info" - urlPrefix = "http://" + globals.noAuthEmulator.httpServer.Addr + "/v1/" - - // Setup http.Client that we will use for all HTTP requests - - httpClient = &http.Client{} - - // Send a GET for "/info" expecting [EMSWIFT] data in compact JSON form - - httpRequest, err = http.NewRequest("GET", urlForInfo, nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - expectedInfo = "{\"swift\": {\"max_account_name_length\": 256,\"max_container_name_length\": 256,\"max_object_name_length\": 1024,\"account_listing_limit\": 10000,\"container_listing_limit\": 10000}}" - if int64(len(expectedInfo)) != httpResponse.ContentLength { - t.Fatalf("GET of /info httpResponse.ContentLength unexpected") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if expectedInfo != utils.ByteSliceToString(readBuf) { - t.Fatalf("GET of /info httpResponse.Body contents unexpected") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for account "TestAccount" and header Cat: Dog - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Cat", "Dog") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for account "TestAccount" and header Mouse: Bird - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestAccount should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for account "TestAccount" deleting header Mouse - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestAccount should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for account "TestAccount" and header Mouse: Bird - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusAccepted != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestAccount should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for account "TestAccount" deleting header Mouse - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusAccepted != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestAccount should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for container "TestContainer" and header Cat: Dog - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Cat", "Dog") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" expecting "TestContainer\n" and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if int64(len("TestContainer\n")) != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "TestContainer\n" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" with marker "AAA" expecting "TestContainer\n" and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount?marker=AAA", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if int64(len("TestContainer\n")) != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "TestContainer\n" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" with marker "ZZZ" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount?marker=ZZZ", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for container "TestContainer" and header Mouse: Bird - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestContainer should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for container "TestContainer" deleting header Mouse - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestContainer should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for container "TestContainer" and header Mouse: Bird - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusAccepted != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestContainer should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for container "TestContainer" deleting header Mouse - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusAccepted != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestContainer should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a non-chunked PUT for object "Foo" to contain []byte{0x00, 0x01, 0x02} - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/Foo", bytes.NewReader([]byte{0x00, 0x01, 0x02})) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a chunked PUT for object "Bar"" with 1st chunk being []byte{0xAA, 0xBB} & 2nd chunk being []byte{0xCC, 0xDD, 0xEE} - - pipeReader, pipeWriter = io.Pipe() - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/Bar", pipeReader) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.ContentLength = -1 - httpRequest.Header.Del("Content-Length") - errChan = make(chan error, 1) - go func() { - var ( - nonShadowingErr error - nonShadowingHTTPResponse *http.Response - ) - nonShadowingHTTPResponse, nonShadowingErr = httpClient.Do(httpRequest) - if nil == nonShadowingErr { - httpResponse = nonShadowingHTTPResponse - } - errChan <- nonShadowingErr - }() - _, err = pipeWriter.Write([]byte{0xAA, 0xBB}) - if nil != err { - t.Fatalf("pipeWriter.Write() returned unexpected error: %v", err) - } - _, err = pipeWriter.Write([]byte{0xCC, 0xDD, 0xEE}) - if nil != err { - t.Fatalf("pipeWriter.Write() returned unexpected error: %v", err) - } - err = pipeWriter.Close() - if nil != err { - t.Fatalf("pipeWriter.Close() returned unexpected error: %v", err) - } - err = <-errChan - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" expecting "Bar\nFoo\n" and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if int64(len("Bar\nFoo\n")) != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "Bar\nFoo\n" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" with marker "AAA" expecting "Bar\nFoo\n" and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer?marker=AAA", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if int64(len("Bar\nFoo\n")) != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "Bar\nFoo\n" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" with marker "ZZZ" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer?marker=ZZZ", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "Foo" expecting Content-Length: 3 - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/Foo", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if 3 != httpResponse.ContentLength { - t.Fatalf("httpResponse.ContentLength contained unexpected value: %v", httpResponse.ContentLength) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a full object GET for object "Foo" expecting []byte{0x00, 0x01, 0x02} - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Foo", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if int64(len([]byte{0x00, 0x01, 0x02})) != httpResponse.ContentLength { - t.Fatalf("Foo should contain precisely []byte{0x00, 0x01, 0x02}") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if 0 != bytes.Compare([]byte{0x00, 0x01, 0x02}, readBuf) { - t.Fatalf("Foo should contain precisely []byte{0x00, 0x01, 0x02}") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a range GET of bytes at offset 1 for length 3 for object "Bar" expecting []byte{0xBB, 0xCC, 0xDD} - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Range", "bytes=1-3") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusPartialContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if int64(len([]byte{0xBB, 0xCC, 0xDD})) != httpResponse.ContentLength { - t.Fatalf("Bar's bytes 1-3 should contain precisely []byte{0xBB, 0xCC, 0xDD}") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, readBuf) { - t.Fatalf("Bar's bytes 1-3 should contain precisely []byte{0xBB, 0xCC, 0xDD}") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a range GET of bytes at offset 0 for length 2 - // and offset 3 for length of 1 for object "Bar" - // expecting two MIME parts: []byte{0xAA, 0xBB} and []byte{0xDD} - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Range", "bytes=0-1,3-3") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusPartialContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - contentType = httpResponse.Header.Get("Content-Type") - contentTypeMultiPartBoundary = strings.TrimPrefix(contentType, "multipart/byteranges; boundary=") - if (len(contentType) == len(contentTypeMultiPartBoundary)) || (0 == len(contentTypeMultiPartBoundary)) { - t.Fatalf("httpReponse.Header[\"Content-Type\"] contained unexpected value: \"%v\"", contentType) - } - expectedBuf = make([]byte, 0, httpResponse.ContentLength) - expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"\r\n")...) - expectedBuf = append(expectedBuf, []byte("Content-Type: application/octet-stream\r\n")...) - expectedBuf = append(expectedBuf, []byte("Content-Range: bytes 0-1/5\r\n")...) - expectedBuf = append(expectedBuf, []byte("\r\n")...) - expectedBuf = append(expectedBuf, []byte{0xAA, 0xBB}...) - expectedBuf = append(expectedBuf, []byte("\r\n")...) - expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"\r\n")...) - expectedBuf = append(expectedBuf, []byte("Content-Type: application/octet-stream\r\n")...) - expectedBuf = append(expectedBuf, []byte("Content-Range: bytes 3-3/5\r\n")...) - expectedBuf = append(expectedBuf, []byte("\r\n")...) - expectedBuf = append(expectedBuf, []byte{0xDD}...) - expectedBuf = append(expectedBuf, []byte("\r\n")...) - expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"--")...) - if int64(len(expectedBuf)) != httpResponse.ContentLength { - t.Fatalf("Unexpected multi-part GET response Content-Length") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if 0 != bytes.Compare(expectedBuf, readBuf) { - t.Fatalf("Unexpected payload of multi-part GET response") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a tail GET of the last 2 bytes for object "Bar" expecting []byte{0xDD, 0xEE} - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Range", "bytes=-2") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusPartialContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if int64(len([]byte{0xDD, 0xEE})) != httpResponse.ContentLength { - t.Fatalf("Bar's last 2 bytes should contain precisely []byte{0xDD, 0xEE}") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if 0 != bytes.Compare([]byte{0xDD, 0xEE}, readBuf) { - t.Fatalf("Bar's last 2 bytes should contain precisely []byte{0xDD, 0xEE}") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for object "ZigZag" and header Cat: Dog - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Cat", "Dog") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "ZigZag" expecting header Cat: Dog - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "ZigZag" expecting header Cat: Dog - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for object "ZigZag" and header Mouse: Bird - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "ZigZag" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestAccount should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for object "ZigZag" deleting header Mouse - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "ZigZag" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestAccount should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for object "Foo" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/Foo", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for object "Bar" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/Bar", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for object "ZigZag" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for container "TestContainer" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for account "TestAccount" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - err = Stop() - if nil != err { - t.Fatalf("Stop() returned unexpected error: %v", err) - } -} diff --git a/emswift/main.go b/emswift/main.go deleted file mode 100644 index 4e907384..00000000 --- a/emswift/main.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "os" - "os/signal" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/emswift/emswiftpkg" -) - -func main() { - var ( - confMap conf.ConfMap - err error - signalChan chan os.Signal - ) - - if len(os.Args) < 2 { - fmt.Fprintf(os.Stderr, "no .conf file specified\n") - os.Exit(1) - } - - confMap, err = conf.MakeConfMapFromFile(os.Args[1]) - if nil != err { - fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err) - os.Exit(1) - } - - err = confMap.UpdateFromStrings(os.Args[2:]) - if nil != err { - fmt.Fprintf(os.Stderr, "failed to apply config overrides: %v\n", err) - os.Exit(1) - } - - // Start Swift Emulation - - err = emswiftpkg.Start(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "emswiftpkg.Start(confMap) failed: %v\n", err) - os.Exit(1) - } - - // Arm signal handler used to indicate termination & wait on it - // - // Note: signal'd chan must be buffered to avoid race with window between - // arming handler and blocking on the chan read - - signalChan = make(chan os.Signal, 1) - - signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) - - _ = <-signalChan - - // Stop Swift Emulation - - err = emswiftpkg.Stop() - if nil != err { - fmt.Fprintf(os.Stderr, "emswiftpkg.Stop() failed: %v\n", err) - os.Exit(1) - } -} diff --git a/etcdclient/Makefile b/etcdclient/Makefile deleted file mode 100644 index 302c61c4..00000000 --- a/etcdclient/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/etcdclient - -include ../GoMakefile diff --git a/etcdclient/api.go b/etcdclient/api.go deleted file mode 100644 index 21b599a7..00000000 --- a/etcdclient/api.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package etcdclient - -import ( - "log" - "os" - "time" - - etcd "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" -) - -const ( - trustedCAFile = "/ca.pem" -) - -// New initializes etcd config structures and returns an etcd client -func New(tlsInfo *transport.TLSInfo, endPoints []string, autoSyncInterval time.Duration, dialTimeout time.Duration) (etcdClient *etcd.Client, err error) { - tlsConfig, etcdErr := tlsInfo.ClientConfig() - if etcdErr != nil { - log.Fatal(etcdErr) - } - - etcdClient, err = etcd.New(etcd.Config{ - Endpoints: endPoints, - AutoSyncInterval: autoSyncInterval, - DialTimeout: dialTimeout, - TLS: tlsConfig, - }) - return -} - -// GetCertFilePath returns the name of the cert file for the local node -func GetCertFilePath(certDir string) string { - h, _ := os.Hostname() - return certDir + "/node-" + h + ".pem" -} - -// GetKeyFilePath returns the name of the key file for the local node -func GetKeyFilePath(certDir string) string { - h, _ := os.Hostname() - return certDir + "/node-" + h + "-key.pem" -} - -// GetCA returns the name of the certificate authority for the local node -func GetCA(certDir string) string { - var ( - caFile string - ) - - trustedCAFilePath := certDir + trustedCAFile - _, statErr := os.Stat(trustedCAFilePath) - if os.IsExist(statErr) { - caFile = trustedCAFilePath - } - - return caFile -} diff --git a/etcdclient/dummy_test.go b/etcdclient/dummy_test.go deleted file mode 100644 index baf4898d..00000000 --- a/etcdclient/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package etcdclient - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/evtlog/Makefile b/evtlog/Makefile deleted file mode 100644 index 54a10b7a..00000000 --- a/evtlog/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/evtlog - -include ../GoMakefile diff --git a/evtlog/api.go b/evtlog/api.go deleted file mode 100644 index 9670fe12..00000000 --- a/evtlog/api.go +++ /dev/null @@ -1,380 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package evtlog - -// FormatType is an efficiently passed indicator of the event type being logged. -type FormatType uint32 // Used as index to event slice (so keep this sequence in sync with event slice) - -const ( - FormatTestPatternFixed FormatType = iota - FormatTestPatternS - FormatTestPatternS03D - FormatTestPatternS08X - FormatTestPatternS016X - FormatTestPatternS016X016X - FormatTestPatternS016X016X016X - FormatTestPatternS016Xslice - FormatTestPatternS016XS - FormatTestPatternSS - FormatTestPatternSS03D - FormatTestPatternSS016X1X - FormatTestPatternSS016X1X1X - FormatTestPatternSSS - FormatTestPatternSSS03D - FormatTestPatternSSS016X03D - FormatTestPatternSSS016X016X03D - FormatHalterArm - FormatHalterDisarm - FormatAccountDelete - FormatAccountGet - FormatAccountHead - FormatAccountPost - FormatAccountPut - FormatContainerDelete - FormatContainerGet - FormatContainerHead - FormatContainerPost - FormatContainerPut - FormatObjectDelete - FormatObjectGet - FormatObjectHead - FormatObjectLoad - FormatObjectRead - FormatObjectTail - FormatObjectPutChunkedStart - FormatObjectPutChunkedEnd - FormatHeadhunterCheckpointStart - FormatHeadhunterCheckpointEndSuccess - FormatHeadhunterCheckpointEndFailure - FormatHeadhunterRecordTransactionNonceRangeReserve - FormatHeadhunterRecordTransactionPutInodeRec - FormatHeadhunterRecordTransactionPutInodeRecs - FormatHeadhunterRecordTransactionDeleteInodeRec - FormatHeadhunterRecordTransactionPutLogSegmentRec - FormatHeadhunterRecordTransactionDeleteLogSegmentRec - FormatHeadhunterRecordTransactionPutBPlusTreeObject - FormatHeadhunterRecordTransactionDeleteBPlusTreeObject - FormatHeadhunterMissingInodeRec - FormatHeadhunterMissingLogSegmentRec - FormatHeadhunterMissingBPlusTreeObject - FormatHeadhunterBPlusTreeNodeFault - FormatDirFileBPlusTreeNodeFault - FormatFlushInodesEntry - FormatFlushInodesDirOrFilePayloadObjectNumberUpdated - FormatFlushInodesErrorOnInode - FormatFlushInodesErrorOnHeadhunterPut - FormatFlushInodesExit - FormatLeaseRequest - FormatLeaseReply - FormatLeaseInterrupt - // - formatTypeCount // Used to quickly check upper limit of FormatType values -) - -type patternType uint32 - -const ( - patternFixed patternType = iota // + "..." - patternS // + "...%s..." - patternS03D // + "...%s...%03d..." - patternS08X // + "...%s...%08X..." - patternS016X // + "...%s...%016X..." - patternS016X016X // + "...%s...%016X...%016X..." - patternS016X016X016X // + "...%s...%016X...%016X...%016X..." - patternS016Xslice // + "...%s...[...%016X...]..." where '[' & ']' delineate slice - patternS016XS // + "...%s...%016X...%s..." - patternSS // + "...%s...%s..." - patternSS03D // + "...%s...%s...%03d..." - patternSS016X1X // + "...%s...%s...%016X...%1X..." - patternSS016X1X1X // + "...%s...%s...%016X...%1X...%1X..." - patternSSS // + "...%s...%s...%s..." - patternSSS03D // + "...%s...%s...%s...%03d..." - patternSSS016X03D // + "...%s...%s...%s...%016X...%03d..." - patternSSS016X016X03D // + "...%s...%s...%s...%016X...%016X...%03d..." -) - -const ( - patternTimestampFormat = "2006-01-02 15:04:05.0000" -) - -type eventType struct { - patternType - formatString string -} - -var ( - event = []eventType{ // Indexed by FormatType (so keep that const iota seqence in sync here) - eventType{ // FormatTestPatternFixed - patternType: patternFixed, - formatString: "%s Test for patternFixed", - }, - eventType{ // FormatTestPatternS - patternType: patternS, - formatString: "%s Test for patternS arg0:%s", - }, - eventType{ // FormatTestPatternS03D - patternType: patternS03D, - formatString: "%s Test for patternS03D arg0:%s arg1:%03d", - }, - eventType{ // FormatTestPatternS08X - patternType: patternS08X, - formatString: "%s Test for patternS08X arg0:%s arg1:%08X", - }, - eventType{ // FormatTestPatternS016X - patternType: patternS016X, - formatString: "%s Test for patternS016X arg0:%s arg1:%016X", - }, - eventType{ // FormatTestPatternS016X016X - patternType: patternS016X016X, - formatString: "%s Test for patternS016X016X arg0:%s arg1:%016X arg2:%016X", - }, - eventType{ // FormatTestPatternS016X016X016X - patternType: patternS016X016X016X, - formatString: "%s Test for patternS016X016X016X arg0:%s arg1:%016X arg2:%016X arg3:%016X", - }, - eventType{ // FormatTestPatternS016Xslice - patternType: patternS016Xslice, - formatString: "%s Test for patternS016Xslice arg0:%s arg1:[0x%016X]", - }, - eventType{ // FormatTestPatternS016XS - patternType: patternS016XS, - formatString: "%s Test for patternS016XS arg0:%s arg1:%016X arg2:%s", - }, - eventType{ // FormatTestPatternSS - patternType: patternSS, - formatString: "%s Test for patternSS arg0:%s arg1:%s", - }, - eventType{ // FormatTestPatternSS03D - patternType: patternSS03D, - formatString: "%s Test for patternSS03D arg0:%s arg1:%s arg2:%03d", - }, - eventType{ // FormatTestPatternSS016X1X - patternType: patternSS016X1X, - formatString: "%s Test for patternSS016X1X arg0:%s arg1:%s arg2:%016X arg3:%1X", - }, - eventType{ // FormatTestPatternSS016X1X1X - patternType: patternSS016X1X1X, - formatString: "%s Test for patternSS016X1X1X arg0:%s arg1:%s arg2:%016X arg3:%1X arg4:%1X", - }, - eventType{ // FormatTestPatternSSS - patternType: patternSSS, - formatString: "%s Test for patternSSS arg0:%s arg1:%s arg2:%s", - }, - eventType{ // FormatTestPatternSSS03D - patternType: patternSSS03D, - formatString: "%s Test for patternSSS03D arg0:%s arg1:%s arg2:%s arg3:%03d", - }, - eventType{ // FormatTestPatternSSS016X03D - patternType: patternSSS016X03D, - formatString: "%s Test for patternSSS016X03D arg0:%s arg1:%s arg2:%s arg3:%016X arg4:%03d", - }, - eventType{ // FormatTestPatternSSS016X016X03D - patternType: patternSSS016X016X03D, - formatString: "%s Test for patternSSS016X016X03D arg0:%s arg1:%s arg2:%s arg3:%016X arg4:%016X arg5:%03d", - }, - eventType{ // FormatHalterArm - patternType: patternS08X, - formatString: "%s halter.Arm(%s, 0x%08X) called", - }, - eventType{ // FormatHalterDisarm - patternType: patternS, - formatString: "%s halter.Disarm(%s) called", - }, - eventType{ // FormatAccountDelete - patternType: patternS03D, - formatString: "%s Account DELETE %s had status %03d", - }, - eventType{ // FormatAccountGet - patternType: patternS03D, - formatString: "%s Account GET %s had status %03d", - }, - eventType{ // FormatAccountHead - patternType: patternS03D, - formatString: "%s Account HEAD %s had status %03d", - }, - eventType{ // FormatAccountPost - patternType: patternS03D, - formatString: "%s Account POST %s had status %03d", - }, - eventType{ // FormatAccountPut - patternType: patternS03D, - formatString: "%s Account PUT %s had status %03d", - }, - eventType{ // FormatContainerDelete - patternType: patternSS03D, - formatString: "%s Container DELETE %s/%s had status %03d", - }, - eventType{ // FormatContainerGet - patternType: patternSS03D, - formatString: "%s Container GET %s/%s had status %03d", - }, - eventType{ // FormatContainerHead - patternType: patternSS03D, - formatString: "%s Container HEAD %s/%s had status %03d", - }, - eventType{ // FormatContainerPost - patternType: patternSS03D, - formatString: "%s Container POST %s/%s had status %03d", - }, - eventType{ // FormatContainerPut - patternType: patternSS03D, - formatString: "%s Container PUT %s/%s had status %03d", - }, - eventType{ // FormatObjectDelete - patternType: patternSSS03D, - formatString: "%s Object DELETE %s/%s/%s had status %03d", - }, - eventType{ // FormatObjectGet - patternType: patternSSS016X016X03D, - formatString: "%s Object GET %s/%s/%s (offset 0x%016X length 0x%016X) had status %03d", - }, - eventType{ // FormatObjectHead - patternType: patternSSS03D, - formatString: "%s Object HEAD %s/%s/%s had status %03d", - }, - eventType{ // FormatObjectLoad - patternType: patternSSS03D, - formatString: "%s Object LOAD %s/%s/%s had status %03d", - }, - eventType{ // FormatObjectRead - patternType: patternSSS016X016X03D, - formatString: "%s Object READ %s/%s/%s (offset 0x%016X length 0x%016X) had status %03d", - }, - eventType{ // FormatObjectTail - patternType: patternSSS016X03D, - formatString: "%s Object TAIL %s/%s/%s (length 0x%016X) had status %03d", - }, - eventType{ // FormatObjectPutChunkedStart - patternType: patternSSS, - formatString: "%s Object (chunked) PUT %s/%s/%s initiated", - }, - eventType{ // FormatObjectPutChunkedEnd - patternType: patternSSS016X03D, - formatString: "%s Object (chunked) PUT %s/%s/%s (length 0x%016X) had status %03d", - }, - eventType{ // FormatHeadhunterCheckpointStart - patternType: patternS, - formatString: "%s headhunter.checkpointDaemon calling putCheckpoint() for Volume '%s'", - }, - eventType{ // FormatHeadhunterCheckpointEndSuccess - patternType: patternS, - formatString: "%s headhunter.checkpointDaemon completed putCheckpoint() for Volume '%s' successfully", - }, - eventType{ // FormatHeadhunterCheckpointEndFailure - patternType: patternSS, - formatString: "%s headhunter.checkpointDaemon completed putCheckpoint() for Volume '%s' with error: %s", - }, - eventType{ // FormatHeadhunterRecordTransactionNonceRangeReserve - patternType: patternS016X016X, - formatString: "%s Headhunter recording reservation of Volume '%s' Nonces 0x%016X thru 0x%016X (inclusive)", - }, - eventType{ // FormatHeadhunterRecordTransactionPutInodeRec - patternType: patternS016X, - formatString: "%s Headhunter recording PutInodeRec for Volume '%s' Inode# 0x%016X", - }, - eventType{ // FormatHeadhunterRecordTransactionPutInodeRecs - patternType: patternS016Xslice, - formatString: "%s Headhunter recording PutInodeRecs for Volume '%s' Inode#'s [0x%016X]", - }, - eventType{ // FormatHeadhunterRecordTransactionDeleteInodeRec - patternType: patternS016X, - formatString: "%s Headhunter recording DeleteInodeRec for Volume '%s' Inode# 0x%016X", - }, - eventType{ // FormatHeadhunterRecordTransactionPutLogSegmentRec - patternType: patternS016XS, - formatString: "%s Headhunter recording PutLogSegmentRec for Volume '%s' LogSegment# 0x%016X => Container: '%s'", - }, - eventType{ // FormatHeadhunterRecordTransactionDeleteLogSegmentRec - patternType: patternS016X, - formatString: "%s Headhunter recording DeleteLogSegmentRec for Volume '%s' LogSegment# 0x%016X", - }, - eventType{ // FormatHeadhunterRecordTransactionPutBPlusTreeObject - patternType: patternS016X, - formatString: "%s Headhunter recording PutBPlusTreeObject for Volume '%s' Virtual Object# 0x%016X", - }, - eventType{ // FormatHeadhunterRecordTransactionDeleteBPlusTreeObject - patternType: patternS016X, - formatString: "%s Headhunter recording DeleteBPlusTreeObject for Volume '%s' Virtual Object# 0x%016X", - }, - eventType{ // FormatHeadhunterMissingInodeRec - patternType: patternS016X, - formatString: "%s Headhunter recording DeleteBPlusTreeObject for Volume '%s' Inode# 0x%016X", - }, - eventType{ // FormatHeadhunterMissingLogSegmentRec - patternType: patternS016X, - formatString: "%s Headhunter recording DeleteBPlusTreeObject for Volume '%s' LogSegment# 0x%016X", - }, - eventType{ // FormatHeadhunterMissingBPlusTreeObject - patternType: patternS016X, - formatString: "%s Headhunter recording DeleteBPlusTreeObject for Volume '%s' Virtual Object# 0x%016X", - }, - eventType{ // FormatHeadhunterBPlusTreeNodeFault - patternType: patternS016X016X016X, - formatString: "%s Headhunter B+Tree NodeFault for Volume '%s' from Object# 0x%016X Offset 0x%016X Length 0x%016X", - }, - eventType{ // FormatDirFileBPlusTreeNodeFault - patternType: patternS016X016X, - formatString: "%s Inode B+Tree NodeFault for Volume '%s' for Inode# 0x%016X from Virtual Object# 0x%016X", - }, - eventType{ // FormatFlushInodesEntry - patternType: patternS016Xslice, - formatString: "%s inode.flushInodes() entered for Volume '%s' Inode#'s [0x%016X]", - }, - eventType{ // FormatFlushInodesDirOrFilePayloadObjectNumberUpdated - patternType: patternS016X016X, - formatString: "%s inode.flushInodes() updated payload for Volume '%s' Inode# 0x%016X with ObjectNumber 0x%016X", - }, - eventType{ // FormatFlushInodesErrorOnInode - patternType: patternS016XS, - formatString: "%s inode.flushInodes() exiting for Volume '%s' Inode# 0x%016X with error: %s", - }, - eventType{ // FormatFlushInodesErrorOnHeadhunterPut - patternType: patternSS, - formatString: "%s inode.flushInodes() exiting for Volume '%s' with headhunter.PutInodeRecs() error: %s", - }, - eventType{ // FormatFlushInodesExit - patternType: patternS016Xslice, - formatString: "%s inode.flushInodes() exited for Volume '%s' Inode#'s [0x%016X]", - }, - eventType{ // FormatLeaseRequest - patternType: patternSS016X1X, - formatString: "%s jrpcfs.RpcLease() entered for Volume '%s' MountID '%s' Inode# 0x%016X RequestType %1X", - }, - eventType{ // FormatLeaseReply - patternType: patternSS016X1X1X, - formatString: "%s jrpcfs.RpcLease() exited for Volume '%s' MountID '%s' Inode# 0x%016X RequestType %1X ReplyType %1X", - }, - eventType{ // FormatLeaseInterrupt - patternType: patternSS016X1X, - formatString: "%s jrpcfs.RpcLease() entered for Volume '%s' MountID '%s' Inode# 0x%016X InterruptType %1X", - }, - } -) - -// Record is used to log an event to the shared memory object. -func Record(formatType FormatType, args ...interface{}) { - if globals.eventLogEnabled { - record(formatType, args...) - } -} - -// Retrieve is used to fetch an event from the shared memory object. -// If the event log is enabled, the call will block until an event is available. -// If the event log is not enabled, the call will return an empty string (""). -func Retrieve() (formattedRecord string, numDroppedRecords uint64) { - if globals.eventLogEnabled { - formattedRecord, numDroppedRecords = retrieve() - } else { - formattedRecord = "" - numDroppedRecords = 0 - } - return -} - -// Mark for deletion causes the the shared memory object to be deleted upon the last Down() call referencing it. -func MarkForDeletion() { - if globals.eventLogEnabled { - markForDeletion() - } -} diff --git a/evtlog/api_internal.go b/evtlog/api_internal.go deleted file mode 100644 index 9371417e..00000000 --- a/evtlog/api_internal.go +++ /dev/null @@ -1,1219 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package evtlog - -import ( - "crypto/rand" - "encoding/binary" - "fmt" - "log" - "math/big" - "strings" - "time" - "unsafe" -) - -// #include -// #include -// #include -// #include -// -// uint8_t read_uint8(uintptr_t shmaddr, uint64_t offset) { -// uint8_t *src_ptr; -// -// src_ptr = (uint8_t *)shmaddr + offset; -// -// return *src_ptr; -// } -// -// uint32_t read_uint32(uintptr_t shmaddr, uint64_t offset) { -// uint32_t *src_ptr; -// -// src_ptr = (uint32_t *)((uint8_t *)shmaddr + offset); -// -// return *src_ptr; -// } -// -// uint64_t read_uint64(uintptr_t shmaddr, uint64_t offset) { -// uint64_t *src_ptr; -// -// src_ptr = (uint64_t *)((uint8_t *)shmaddr + offset); -// -// return *src_ptr; -// } -// -// void read_buf(uintptr_t shmaddr, uint64_t offset, uint64_t length, uintptr_t buf) { -// uint8_t *src_ptr; -// -// src_ptr = (uint8_t *)shmaddr + offset; -// -// (void)memcpy((void *)buf, (const void *)src_ptr, (size_t)length); -// } -// -// void write_uint8(uintptr_t shmaddr, uint64_t offset, uint8_t value) { -// uint8_t *dst_ptr; -// -// dst_ptr = (uint8_t *)shmaddr + offset; -// -// *dst_ptr = value; -// } -// -// void write_uint32(uintptr_t shmaddr, uint64_t offset, uint32_t value) { -// uint32_t *dst_ptr; -// -// dst_ptr = (uint32_t *)((uint8_t *)shmaddr + offset); -// -// *dst_ptr = value; -// } -// -// void write_uint64(uintptr_t shmaddr, uint64_t offset, uint64_t value) { -// uint64_t *dst_ptr; -// -// dst_ptr = (uint64_t *)((uint8_t *)shmaddr + offset); -// -// *dst_ptr = value; -// } -// -// void write_buf(uintptr_t shmaddr, uint64_t offset, uint64_t length, uintptr_t buf) { -// uint8_t *dst_ptr; -// -// dst_ptr = (uint8_t *)shmaddr + offset; -// -// (void)memcpy((void *)dst_ptr, (const void *)buf, (size_t)length); -// } -import "C" - -// Shared memory object has the following format: -// -// type sharedMemoryStruct struct { -// producerActive uint8 // 0 if Producer is not currently requesting or holding the lock; otherwise 1 -// consumerActive uint8 // 0 if Consumer is not currently requesting or holding the lock; otherwise 1 -// producerNext uint8 // 1 if Producer is "next"; 0 if Consumer is "next" -// nextProducerOffset uint64 // where next call to record() will place Record -// nextConsumerOffset uint64 // where next call to retieve() will read a Record from -// numBufferWraps uint64 // number of times the Producer had to wrap back to offsetToFirstRecordEver -// numDroppedRecords uint64 // number of times the Consumer fell too far behind Producer -// } -// -// Synchronization uses Peterson's algorithm (simple one... for just two "threads") - -const ( - offsetToProducerActive = C.uint64_t(0x00) - offsetToConsumerActive = C.uint64_t(0x01) - offsetToProducerNext = C.uint64_t(0x02) - offsetToNextProducerOffset = C.uint64_t(0x08) - offsetToNextConsumerOffset = C.uint64_t(0x10) - offsetToNumBufferWraps = C.uint64_t(0x18) - offsetToNumDroppedRecords = C.uint64_t(0x20) - offsetToFirstRecordEver = C.uint64_t(0x28) // contents (possibly not on Record boundary) will wrap to here also -) - -// Each Record has the following format: -// -// type recordStruct struct { -// recordLength uint32 // includes recordLength field... and what follows -// unixNano uint64 // time.Now().UnixNano() of event -// formatType FormatType // indicates the format of the event -// // For each format specifier in event[formatType].patternType" -// // %s: -// stringLength uint32 // number of UTF-8 bytes in stringBytes array -// stringBytes [stringLength]byte // bytes of the UTF-8 string (not terminated) -// // %03d: -// u32 uint32 // 32-bit unsigned value (though <= 999) -// // %08X: -// u32 uint32 // 32-bit unsigned value -// // %016X: -// u64 uint64 // 64-bit unsigned value -// } - -func backoff() { - var ( - backoffDuration time.Duration - err error - mustBeLessThanBigIntPtr *big.Int - u64BigIntPtr *big.Int - ) - - mustBeLessThanBigIntPtr = big.NewInt(int64(globals.eventLogLockMaxBackoff - globals.eventLogLockMinBackoff + 1)) - u64BigIntPtr, err = rand.Int(rand.Reader, mustBeLessThanBigIntPtr) - if nil != err { - log.Fatal(err) - } - backoffDuration = globals.eventLogLockMinBackoff + time.Duration(u64BigIntPtr.Uint64()) - - time.Sleep(backoffDuration) -} - -func ensureSharedMemoryIsInitialized() { - var ( - nextProducerOffset C.uint64_t - nextConsumerOffset C.uint64_t - ) - - nextProducerOffset = C.read_uint64(globals.shmAddr, offsetToNextProducerOffset) - if C.uint64_t(0) == nextProducerOffset { - C.write_uint64(globals.shmAddr, offsetToNextProducerOffset, offsetToFirstRecordEver) - } - - nextConsumerOffset = C.read_uint64(globals.shmAddr, offsetToNextConsumerOffset) - if C.uint64_t(0) == nextConsumerOffset { - C.write_uint64(globals.shmAddr, offsetToNextConsumerOffset, offsetToFirstRecordEver) - } - - globals.shmKnownToBeInitialized = true -} - -func getProducerLock() { - var ( - consumerActive C.uint8_t - producerNext C.uint8_t - ) - - // Lock out other Producers (all must be within this process) - - globals.Lock() - - // Perform 2-thread acquire() from Peterson's algorithm - - C.write_uint8(globals.shmAddr, offsetToProducerActive, 1) - C.write_uint8(globals.shmAddr, offsetToProducerNext, 1) - - consumerActive = C.read_uint8(globals.shmAddr, offsetToConsumerActive) - producerNext = C.read_uint8(globals.shmAddr, offsetToProducerNext) - - for (C.uint8_t(1) == consumerActive) && (C.uint8_t(1) == producerNext) { - backoff() - - consumerActive = C.read_uint8(globals.shmAddr, offsetToConsumerActive) - producerNext = C.read_uint8(globals.shmAddr, offsetToProducerNext) - } - - if !globals.shmKnownToBeInitialized { - ensureSharedMemoryIsInitialized() - } -} - -func freeProducerLock() { - // Indicate this Producer is no longer active - - C.write_uint8(globals.shmAddr, offsetToProducerActive, 0) - - // Unlock to allow other Producers (all within this process) to proceed - - globals.Unlock() // Unlock to allow other Producers (all within this process) to proceed -} - -func getConsumerLock() { - var ( - producerActive C.uint8_t - producerNext C.uint8_t - ) - - // Perform 2-thread acquire() from Peterson's algorithm - - C.write_uint8(globals.shmAddr, offsetToConsumerActive, 1) - C.write_uint8(globals.shmAddr, offsetToProducerNext, 0) - - producerActive = C.read_uint8(globals.shmAddr, offsetToProducerActive) - producerNext = C.read_uint8(globals.shmAddr, offsetToProducerNext) - - for (C.uint8_t(1) == producerActive) && (C.uint8_t(0) == producerNext) { - backoff() - - producerActive = C.read_uint8(globals.shmAddr, offsetToProducerActive) - producerNext = C.read_uint8(globals.shmAddr, offsetToProducerNext) - } - - if !globals.shmKnownToBeInitialized { - ensureSharedMemoryIsInitialized() - } -} - -func freeConsumerLock() { - // Indicate this Consumer is no longer active - - C.write_uint8(globals.shmAddr, offsetToConsumerActive, 0) -} - -func record(formatType FormatType, args ...interface{}) { - var ( - arg0Len uint32 - arg1Len uint32 - arg1Index uint32 - arg2Len uint32 - discardLength C.uint32_t - err error - nextProducerOffset C.uint64_t - nextConsumerOffset C.uint64_t - numBufferWraps C.uint64_t - numDroppedRecords C.uint64_t - pattern patternType - record []byte - recordLength uint32 - recordPosition uint32 - room uint64 - wrapAmount uint64 - ) - - // Format record - - if formatType >= formatTypeCount { - err = fmt.Errorf("Unrecognized formatType: %v", formatType) - panic(err) - } - - pattern = event[formatType].patternType - - switch pattern { - case patternFixed: - if len(args) != 0 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - recordLength = 4 + // recordLength - 8 + // unixNano - 4 // formatType - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - case patternS: - if len(args) != 1 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len // args[0] %s - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - case patternS03D: - if len(args) != 2 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 // args[1] %03d - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[1].(uint32)) - case patternS08X: - if len(args) != 2 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 // args[1] %08X - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[1].(uint32)) - case patternS016X: - if len(args) != 2 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 8 // args[1] %016X - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[1].(uint64)) - case patternS016X016X: - if len(args) != 3 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 8 + // args[1] %016X - 8 // args[2] %016X - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[1].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[2].(uint64)) - case patternS016X016X016X: - if len(args) != 4 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 8 + // args[1] %016X - 8 + // args[2] %016X - 8 // args[3] %016X - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[1].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[2].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[3].(uint64)) - case patternS016Xslice: - if len(args) != 2 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].([]uint64))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + (arg1Len * 8) // args[1] [%016X] - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - for arg1Index = uint32(0); arg1Index < arg1Len; arg1Index++ { - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[1].([]uint64)[arg1Index]) - recordPosition += 8 - } - case patternS016XS: - if len(args) != 3 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg2Len = uint32(len(args[2].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 8 + // args[1] %016X - 4 + arg2Len // args[2] %s - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[1].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg2Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg2Len], args[2].(string)) - case patternSS: - if len(args) != 2 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + arg1Len // args[1] %s - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg1Len], args[1].(string)) - case patternSS03D: - if len(args) != 3 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + arg1Len + // args[1] %s - 4 // args[2] %03d - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg1Len], args[1].(string)) - recordPosition += arg1Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[2].(uint32)) - case patternSS016X1X: - if len(args) != 4 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + arg1Len + // args[1] %s - 8 + // args[2] %016X - 4 // args[3] %1X - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg1Len], args[1].(string)) - recordPosition += arg1Len - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[2].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[3].(uint32)) - case patternSS016X1X1X: - if len(args) != 5 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + arg1Len + // args[1] %s - 8 + // args[2] %016X - 4 + // args[3] %1X - 4 // args[4] %1X - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg1Len], args[1].(string)) - recordPosition += arg1Len - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[2].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[3].(uint32)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[4].(uint32)) - case patternSSS: - if len(args) != 3 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].(string))) - arg2Len = uint32(len(args[2].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + arg1Len + // args[1] %s - 4 + arg2Len // args[2] %s - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg1Len], args[1].(string)) - recordPosition += arg1Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg2Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg2Len], args[2].(string)) - case patternSSS03D: - if len(args) != 4 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].(string))) - arg2Len = uint32(len(args[2].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + arg1Len + // args[1] %s - 4 + arg2Len + // args[2] %s - 4 // args[3] %03d - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg1Len], args[1].(string)) - recordPosition += arg1Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg2Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg2Len], args[2].(string)) - recordPosition += arg2Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[3].(uint32)) - case patternSSS016X03D: - if len(args) != 5 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].(string))) - arg2Len = uint32(len(args[2].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + arg1Len + // args[1] %s - 4 + arg2Len + // args[2] %s - 8 + // args[3] %016X - 4 // args[4] %03d - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg1Len], args[1].(string)) - recordPosition += arg1Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg2Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg2Len], args[2].(string)) - recordPosition += arg2Len - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[3].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[4].(uint32)) - case patternSSS016X016X03D: - if len(args) != 6 { - err = fmt.Errorf("Unexpected number of arguments (%v) for formatType %v", len(args), formatType) - panic(err) - } - arg0Len = uint32(len(args[0].(string))) - arg1Len = uint32(len(args[1].(string))) - arg2Len = uint32(len(args[2].(string))) - recordLength = 4 + // recordLength - 8 + // unixNano - 4 + // formatType - 4 + arg0Len + // args[0] %s - 4 + arg1Len + // args[1] %s - 4 + arg2Len + // args[2] %s - 8 + // args[3] %016X - 8 + // args[4] %016X - 4 // args[5] %03d - recordLength = (recordLength + 3) & ^uint32(3) // round up so that a uint32 is never split during wrapping - record = make([]byte, recordLength) - recordPosition = 0 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], recordLength) - recordPosition += 4 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], uint64(time.Now().UnixNano())) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], uint32(formatType)) - recordPosition += 4 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg0Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg0Len], args[0].(string)) - recordPosition += arg0Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg1Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg1Len], args[1].(string)) - recordPosition += arg1Len - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], arg2Len) - recordPosition += 4 - copy(record[recordPosition:recordPosition+arg2Len], args[2].(string)) - recordPosition += arg2Len - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[3].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint64(record[recordPosition:recordPosition+8], args[4].(uint64)) - recordPosition += 8 - binary.LittleEndian.PutUint32(record[recordPosition:recordPosition+4], args[5].(uint32)) - default: - err = fmt.Errorf("Unrecognized patternType (%v) for formatType %v", pattern, formatType) - panic(err) - } - - // Insert record in shared memory object at nextProducerOffset - - getProducerLock() - - nextProducerOffset = C.read_uint64(globals.shmAddr, offsetToNextProducerOffset) - nextConsumerOffset = C.read_uint64(globals.shmAddr, offsetToNextConsumerOffset) - - // Ensure there is room for this record - - if nextProducerOffset == nextConsumerOffset { // Empty, not full - room = globals.eventLogBufferLength - uint64(offsetToFirstRecordEver) - 4 - } else if nextProducerOffset < nextConsumerOffset { - room = uint64(nextConsumerOffset-nextProducerOffset) - 4 - } else { // nextProducerOffset > nextConsumerOffset - room = uint64(nextConsumerOffset) + (globals.eventLogBufferLength - uint64(offsetToFirstRecordEver)) - uint64(nextProducerOffset) - 4 - } - - for room < uint64(recordLength) { - discardLength = C.read_uint32(globals.shmAddr, nextConsumerOffset) - nextConsumerOffset += C.uint64_t(discardLength) - if nextConsumerOffset >= C.uint64_t(globals.eventLogBufferLength) { - nextConsumerOffset -= C.uint64_t(globals.eventLogBufferLength) - offsetToFirstRecordEver - } - C.write_uint64(globals.shmAddr, offsetToNextConsumerOffset, nextConsumerOffset) - numDroppedRecords = C.read_uint64(globals.shmAddr, offsetToNumDroppedRecords) - numDroppedRecords++ - C.write_uint64(globals.shmAddr, offsetToNumDroppedRecords, numDroppedRecords) - room += uint64(discardLength) - } - - // Copy record to shared memory object honoring any required wrap - - if (uint64(nextProducerOffset) + uint64(recordLength)) <= globals.eventLogBufferLength { - // No wrap needed to "fit" record - C.write_buf(globals.shmAddr, nextProducerOffset, C.uint64_t(uint64(recordLength)), C.uintptr_t(uintptr(unsafe.Pointer(&record[0])))) - nextProducerOffset += C.uint64_t(uint64(recordLength)) - if nextProducerOffset == C.uint64_t(globals.eventLogBufferLength) { - // But the next record will start at the beginning - nextProducerOffset = offsetToFirstRecordEver - numBufferWraps = C.read_uint64(globals.shmAddr, offsetToNumBufferWraps) - numBufferWraps++ - C.write_uint64(globals.shmAddr, offsetToNumBufferWraps, numBufferWraps) - } - } else { - // Some portion of record must wrap to the beginning - wrapAmount = uint64(nextProducerOffset) + uint64(recordLength) - globals.eventLogBufferLength - C.write_buf(globals.shmAddr, nextProducerOffset, C.uint64_t(uint64(recordLength)-wrapAmount), C.uintptr_t(uintptr(unsafe.Pointer(&record[0])))) - nextProducerOffset = offsetToFirstRecordEver - C.write_buf(globals.shmAddr, nextProducerOffset, C.uint64_t(wrapAmount), C.uintptr_t(uintptr(unsafe.Pointer(&record[uint64(recordLength)-wrapAmount])))) - nextProducerOffset += C.uint64_t(wrapAmount) - numBufferWraps = C.read_uint64(globals.shmAddr, offsetToNumBufferWraps) - numBufferWraps++ - C.write_uint64(globals.shmAddr, offsetToNumBufferWraps, numBufferWraps) - } - - C.write_uint64(globals.shmAddr, offsetToNextProducerOffset, nextProducerOffset) - - freeProducerLock() -} - -func retrieve() (formattedRecord string, numDroppedRecords uint64) { - var ( - arg0String string - arg0StringLen uint32 - arg1String string - arg1StringLen uint32 - arg1U32 uint32 - arg1U64 uint64 - arg1U64Slice []uint64 - arg1U64SliceIndex uint32 - arg1U64SliceLen uint32 - arg2String string - arg2StringLen uint32 - arg2U32 uint32 - arg2U64 uint64 - arg3U32 uint32 - arg3U64 uint64 - arg4U32 uint32 - arg4U64 uint64 - arg5U32 uint32 - err error - formatStringLeftBracketIndex int - formatStringRightBracketIndex int - formatType FormatType - nextProducerOffset C.uint64_t - nextConsumerOffset C.uint64_t - pattern patternType - record []byte - recordLength C.uint32_t - recordPosition uint32 - replicatedFormatStringSubset string - timestamp time.Time - unixNano uint64 - unixNanoSecPart int64 - unixNanoNSecPart int64 - wrapAmount uint64 - ) - - // Fetch record (if any) - - getConsumerLock() - - numDroppedRecords = uint64(C.read_uint64(globals.shmAddr, offsetToNumDroppedRecords)) - if 0 != numDroppedRecords { - C.write_uint64(globals.shmAddr, offsetToNumDroppedRecords, C.uint64_t(0)) - } - - nextProducerOffset = C.read_uint64(globals.shmAddr, offsetToNextProducerOffset) - nextConsumerOffset = C.read_uint64(globals.shmAddr, offsetToNextConsumerOffset) - - if nextProducerOffset == nextConsumerOffset { - // No record available - so just return "" - freeConsumerLock() - formattedRecord = "" - return - } - - recordLength = C.read_uint32(globals.shmAddr, nextConsumerOffset) - - record = make([]byte, recordLength) - - if uint64(nextConsumerOffset+C.uint64_t(recordLength)) <= globals.eventLogBufferLength { - C.read_buf(globals.shmAddr, nextConsumerOffset, C.uint64_t(recordLength), C.uintptr_t(uintptr(unsafe.Pointer(&record[0])))) - nextConsumerOffset += C.uint64_t(recordLength) - if nextConsumerOffset == C.uint64_t(globals.eventLogBufferLength) { - nextConsumerOffset = offsetToFirstRecordEver - } - } else { - wrapAmount = uint64(nextConsumerOffset) + uint64(C.uint64_t(recordLength)) - globals.eventLogBufferLength - C.read_buf(globals.shmAddr, nextConsumerOffset, C.uint64_t(recordLength)-C.uint64_t(wrapAmount), C.uintptr_t(uintptr(unsafe.Pointer(&record[0])))) - nextConsumerOffset = offsetToFirstRecordEver - C.read_buf(globals.shmAddr, nextConsumerOffset, C.uint64_t(wrapAmount), C.uintptr_t(uintptr(unsafe.Pointer(&record[uint64(C.uint64_t(recordLength))-wrapAmount])))) - nextConsumerOffset += C.uint64_t(wrapAmount) - } - - C.write_uint64(globals.shmAddr, offsetToNextConsumerOffset, nextConsumerOffset) - - freeConsumerLock() - - // Decode/format record - - recordPosition = 4 // No need to re-decode recordLength - - unixNano = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - unixNanoSecPart = int64(unixNano / 1000000000) - unixNanoNSecPart = int64(unixNano % 1000000000) - timestamp = time.Unix(unixNanoSecPart, unixNanoNSecPart) - - formatType = FormatType(binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4])) - recordPosition += 4 - - if formatType >= formatTypeCount { - err = fmt.Errorf("Unrecognized formatType: %v", formatType) - panic(err) - } - - pattern = event[formatType].patternType - - switch pattern { - case patternFixed: - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat)) - case patternS: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String) - case patternS03D: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1U32) - case patternS08X: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1U32) - case patternS016X: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1U64) - case patternS016X016X: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - - arg2U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1U64, arg2U64) - case patternS016X016X016X: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - - arg2U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - - arg3U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1U64, arg2U64, arg3U64) - case patternS016Xslice: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1U64SliceLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1U64Slice = make([]uint64, arg1U64SliceLen) - for arg1U64SliceIndex = uint32(0); arg1U64SliceIndex < arg1U64SliceLen; arg1U64SliceIndex++ { - arg1U64Slice[arg1U64SliceIndex] = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - } - - formatStringLeftBracketIndex = strings.Index(event[formatType].formatString, "[") - formatStringRightBracketIndex = strings.Index(event[formatType].formatString, "]") - - formattedRecord = fmt.Sprintf(event[formatType].formatString[:formatStringLeftBracketIndex+1], timestamp.Format(patternTimestampFormat), arg0String) - - if 0 < arg1U64SliceLen { - replicatedFormatStringSubset = event[formatType].formatString[formatStringLeftBracketIndex+1 : formatStringRightBracketIndex] - formattedRecord = formattedRecord + fmt.Sprintf(replicatedFormatStringSubset, arg1U64Slice[0]) - for arg1U64SliceIndex = uint32(1); arg1U64SliceIndex < arg1U64SliceLen; arg1U64SliceIndex++ { - formattedRecord = formattedRecord + " " + fmt.Sprintf(replicatedFormatStringSubset, arg1U64Slice[arg1U64SliceIndex]) - } - } - - formattedRecord = formattedRecord + fmt.Sprintf(event[formatType].formatString[formatStringRightBracketIndex:]) - case patternS016XS: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - - arg2StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg2String = string(record[recordPosition : recordPosition+arg2StringLen]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1U64, arg2String) - case patternSS: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1String = string(record[recordPosition : recordPosition+arg1StringLen]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1String) - case patternSS03D: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1String = string(record[recordPosition : recordPosition+arg1StringLen]) - recordPosition += arg1StringLen - - arg2U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1String, arg2U32) - case patternSS016X1X: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1String = string(record[recordPosition : recordPosition+arg1StringLen]) - recordPosition += arg1StringLen - - arg2U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - arg3U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1String, arg2U64, arg3U32) - case patternSS016X1X1X: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1String = string(record[recordPosition : recordPosition+arg1StringLen]) - recordPosition += arg1StringLen - - arg2U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - arg3U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg4U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1String, arg2U64, arg3U32, arg4U32) - case patternSSS: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1String = string(record[recordPosition : recordPosition+arg1StringLen]) - recordPosition += arg1StringLen - - arg2StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg2String = string(record[recordPosition : recordPosition+arg2StringLen]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1String, arg2String) - case patternSSS03D: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1String = string(record[recordPosition : recordPosition+arg1StringLen]) - recordPosition += arg1StringLen - - arg2StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg2String = string(record[recordPosition : recordPosition+arg2StringLen]) - recordPosition += arg2StringLen - - arg3U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1String, arg2String, arg3U32) - case patternSSS016X03D: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1String = string(record[recordPosition : recordPosition+arg1StringLen]) - recordPosition += arg1StringLen - - arg2StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg2String = string(record[recordPosition : recordPosition+arg2StringLen]) - recordPosition += arg2StringLen - - arg3U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - - arg4U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1String, arg2String, arg3U64, arg4U32) - case patternSSS016X016X03D: - arg0StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg0String = string(record[recordPosition : recordPosition+arg0StringLen]) - recordPosition += arg0StringLen - - arg1StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg1String = string(record[recordPosition : recordPosition+arg1StringLen]) - recordPosition += arg1StringLen - - arg2StringLen = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - recordPosition += 4 - arg2String = string(record[recordPosition : recordPosition+arg2StringLen]) - recordPosition += arg2StringLen - - arg3U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - - arg4U64 = binary.LittleEndian.Uint64(record[recordPosition : recordPosition+8]) - recordPosition += 8 - - arg5U32 = binary.LittleEndian.Uint32(record[recordPosition : recordPosition+4]) - - formattedRecord = fmt.Sprintf(event[formatType].formatString, timestamp.Format(patternTimestampFormat), arg0String, arg1String, arg2String, arg3U64, arg4U64, arg5U32) - default: - err = fmt.Errorf("Unrecognized patternType (%v) for formatType %v", pattern, formatType) - panic(err) - } - - return -} - -func markForDeletion() { - var ( - err error - errno error - rmidResult C.int - ) - - rmidResult, errno = C.shmctl(globals.shmID, C.IPC_RMID, nil) - - if C.int(-1) == rmidResult { - globals.eventLogEnabled = false - err = fmt.Errorf("C.shmctl(globals.shmID, C.IPC_RMID, nil) failed with errno: %v", errno) - panic(err) - } -} diff --git a/evtlog/api_test.go b/evtlog/api_test.go deleted file mode 100644 index 95cb1511..00000000 --- a/evtlog/api_test.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package evtlog - -import ( - "testing" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/transitions" -) - -func TestAPI(t *testing.T) { - var ( - testConfMap conf.ConfMap - testConfMapStrings []string - err error - // - retrievedRecordForFormatTestPatternFixed string - retrievedRecordForFormatTestPatternS string - retrievedRecordForFormatTestPatternS03D string - retrievedRecordForFormatTestPatternS08X string - retrievedRecordForFormatTestPatternS016X string - retrievedRecordForFormatTestPatternS016X016X string - retrievedRecordForFormatTestPatternS016X016X016X string - retrievedRecordForFormatTestPatternS016XsliceLen0 string - retrievedRecordForFormatTestPatternS016XsliceLen1 string - retrievedRecordForFormatTestPatternS016XsliceLen3 string - retrievedRecordForFormatTestPatternS016XS string - retrievedRecordForFormatTestPatternSS string - retrievedRecordForFormatTestPatternSS03D string - retrievedRecordForFormatTestPatternSS016X1X string - retrievedRecordForFormatTestPatternSS016X1X1X string - retrievedRecordForFormatTestPatternSSS string - retrievedRecordForFormatTestPatternSSS03D string - retrievedRecordForFormatTestPatternSSS016X03D string - retrievedRecordForFormatTestPatternSSS016X016X03D string - ) - - testConfMapStrings = []string{ - "Logging.LogFilePath=/dev/null", - "Logging.TraceLevelLogging=none", - "Logging.DebugLevelLogging=none", - "Logging.LogToConsole=false", - "Cluster.WhoAmI=nobody", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "EventLog.Enabled=true", - "EventLog.BufferKey=9876", // Don't conflict with a running instance - "EventLog.BufferLength=65536", // 64KiB - "EventLog.MinBackoff=1us", - "EventLog.MaxBackoff=2us", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatal(err) - } - - err = transitions.Up(testConfMap) - if nil != err { - t.Fatal(err) - } - - MarkForDeletion() - - Record(FormatTestPatternFixed) - Record(FormatTestPatternS, "arg0") - Record(FormatTestPatternS03D, "arg0", uint32(1)) - Record(FormatTestPatternS08X, "arg0", uint32(1)) - Record(FormatTestPatternS016X, "arg0", uint64(1)) - Record(FormatTestPatternS016X016X, "arg0", uint64(1), uint64(2)) - Record(FormatTestPatternS016X016X016X, "arg0", uint64(1), uint64(2), uint64(3)) - Record(FormatTestPatternS016Xslice, "arg0", []uint64{}) - Record(FormatTestPatternS016Xslice, "arg0", []uint64{uint64(0x101)}) - Record(FormatTestPatternS016Xslice, "arg0", []uint64{uint64(0x101), uint64(0x102), uint64(0x103)}) - Record(FormatTestPatternS016XS, "arg0", uint64(1), "arg..2") - Record(FormatTestPatternSS, "arg0", "arg.1") - Record(FormatTestPatternSS03D, "arg0", "arg.1", uint32(2)) - Record(FormatTestPatternSS016X1X, "arg0", "arg.1", uint64(2), uint32(3)) - Record(FormatTestPatternSS016X1X1X, "arg0", "arg.1", uint64(2), uint32(3), uint32(4)) - Record(FormatTestPatternSSS, "arg0", "arg.1", "arg..2") - Record(FormatTestPatternSSS03D, "arg0", "arg.1", "arg..2", uint32(3)) - Record(FormatTestPatternSSS016X03D, "arg0", "arg.1", "arg..2", uint64(3), uint32(4)) - Record(FormatTestPatternSSS016X016X03D, "arg0", "arg.1", "arg..2", uint64(3), uint64(4), uint32(5)) - - retrievedRecordForFormatTestPatternFixed, _ = Retrieve() - retrievedRecordForFormatTestPatternS, _ = Retrieve() - retrievedRecordForFormatTestPatternS03D, _ = Retrieve() - retrievedRecordForFormatTestPatternS08X, _ = Retrieve() - retrievedRecordForFormatTestPatternS016X, _ = Retrieve() - retrievedRecordForFormatTestPatternS016X016X, _ = Retrieve() - retrievedRecordForFormatTestPatternS016X016X016X, _ = Retrieve() - retrievedRecordForFormatTestPatternS016XsliceLen0, _ = Retrieve() - retrievedRecordForFormatTestPatternS016XsliceLen1, _ = Retrieve() - retrievedRecordForFormatTestPatternS016XsliceLen3, _ = Retrieve() - retrievedRecordForFormatTestPatternS016XS, _ = Retrieve() - retrievedRecordForFormatTestPatternSS, _ = Retrieve() - retrievedRecordForFormatTestPatternSS03D, _ = Retrieve() - retrievedRecordForFormatTestPatternSS016X1X, _ = Retrieve() - retrievedRecordForFormatTestPatternSS016X1X1X, _ = Retrieve() - retrievedRecordForFormatTestPatternSSS, _ = Retrieve() - retrievedRecordForFormatTestPatternSSS03D, _ = Retrieve() - retrievedRecordForFormatTestPatternSSS016X03D, _ = Retrieve() - retrievedRecordForFormatTestPatternSSS016X016X03D, _ = Retrieve() - - if "Test for patternFixed" != retrievedRecordForFormatTestPatternFixed[25:] { - t.Fatalf("Retrieval of FormatTestPatternFixed failed") - } - if "Test for patternS arg0:arg0" != retrievedRecordForFormatTestPatternS[25:] { - t.Fatalf("Retrieval of FormatTestPatternS failed") - } - if "Test for patternS03D arg0:arg0 arg1:001" != retrievedRecordForFormatTestPatternS03D[25:] { - t.Fatalf("Retrieval of FormatTestPatternS03D failed") - } - if "Test for patternS08X arg0:arg0 arg1:00000001" != retrievedRecordForFormatTestPatternS08X[25:] { - t.Fatalf("Retrieval of FormatTestPatternS08X failed") - } - if "Test for patternS016X arg0:arg0 arg1:0000000000000001" != retrievedRecordForFormatTestPatternS016X[25:] { - t.Fatalf("Retrieval of FormatTestPatternS016X failed") - } - if "Test for patternS016X016X arg0:arg0 arg1:0000000000000001 arg2:0000000000000002" != retrievedRecordForFormatTestPatternS016X016X[25:] { - t.Fatalf("Retrieval of FormatTestPatternS016X016X failed") - } - if "Test for patternS016X016X016X arg0:arg0 arg1:0000000000000001 arg2:0000000000000002 arg3:0000000000000003" != retrievedRecordForFormatTestPatternS016X016X016X[25:] { - t.Fatalf("Retrieval of FormatTestPatternS016X016X failed") - } - if "Test for patternS016Xslice arg0:arg0 arg1:[]" != retrievedRecordForFormatTestPatternS016XsliceLen0[25:] { - t.Fatalf("Retrieval of FormatTestPatternS016Xslice with len([]uint64) == 0 failed") - } - if "Test for patternS016Xslice arg0:arg0 arg1:[0x0000000000000101]" != retrievedRecordForFormatTestPatternS016XsliceLen1[25:] { - t.Fatalf("Retrieval of FormatTestPatternS016Xslice with len([]uint64) == 1 failed") - } - if "Test for patternS016Xslice arg0:arg0 arg1:[0x0000000000000101 0x0000000000000102 0x0000000000000103]" != retrievedRecordForFormatTestPatternS016XsliceLen3[25:] { - t.Fatalf("Retrieval of FormatTestPatternS016Xslice with len([]uint64) == 3 failed") - } - if "Test for patternS016XS arg0:arg0 arg1:0000000000000001 arg2:arg..2" != retrievedRecordForFormatTestPatternS016XS[25:] { - t.Fatalf("Retrieval of FormatTestPatternS016XS failed") - } - if "Test for patternSS arg0:arg0 arg1:arg.1" != retrievedRecordForFormatTestPatternSS[25:] { - t.Fatalf("Retrieval of FormatTestPatternSS failed") - } - if "Test for patternSS03D arg0:arg0 arg1:arg.1 arg2:002" != retrievedRecordForFormatTestPatternSS03D[25:] { - t.Fatalf("Retrieval of FormatTestPatternSS03D failed") - } - if "Test for patternSS016X1X arg0:arg0 arg1:arg.1 arg2:0000000000000002 arg3:3" != retrievedRecordForFormatTestPatternSS016X1X[25:] { - t.Fatalf("Retrieval of patternSS016X1X failed") - } - if "Test for patternSS016X1X1X arg0:arg0 arg1:arg.1 arg2:0000000000000002 arg3:3 arg4:4" != retrievedRecordForFormatTestPatternSS016X1X1X[25:] { - t.Fatalf("Retrieval of patternSS016X1X1X failed") - } - if "Test for patternSSS arg0:arg0 arg1:arg.1 arg2:arg..2" != retrievedRecordForFormatTestPatternSSS[25:] { - t.Fatalf("Retrieval of FormatTestPatternSSS failed") - } - if "Test for patternSSS03D arg0:arg0 arg1:arg.1 arg2:arg..2 arg3:003" != retrievedRecordForFormatTestPatternSSS03D[25:] { - t.Fatalf("Retrieval of FormatTestPatternSSS03D failed") - } - if "Test for patternSSS016X03D arg0:arg0 arg1:arg.1 arg2:arg..2 arg3:0000000000000003 arg4:004" != retrievedRecordForFormatTestPatternSSS016X03D[25:] { - t.Fatalf("Retrieval of FormatTestPatternSSS016X03D failed") - } - if "Test for patternSSS016X016X03D arg0:arg0 arg1:arg.1 arg2:arg..2 arg3:0000000000000003 arg4:0000000000000004 arg5:005" != retrievedRecordForFormatTestPatternSSS016X016X03D[25:] { - t.Fatalf("Retrieval of FormatTestPatternSSS016X03D failed") - } - - // TODO: Eventually, it would be nice to test the overrun & wrap cases... - - err = transitions.Down(testConfMap) - if nil != err { - t.Fatal(err) - } -} diff --git a/evtlog/benchmark_test.go b/evtlog/benchmark_test.go deleted file mode 100644 index cd447605..00000000 --- a/evtlog/benchmark_test.go +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package evtlog - -import ( - "testing" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/transitions" -) - -var ( - benchmarkConfMap conf.ConfMap -) - -func benchmarkSetup(b *testing.B, enable bool) { - var ( - benchmarkConfMapStrings []string - err error - ) - - if enable { - benchmarkConfMapStrings = []string{ - "Logging.LogFilePath=/dev/null", - "Logging.TraceLevelLogging=none", - "Logging.DebugLevelLogging=none", - "Logging.LogToConsole=false", - "Cluster.WhoAmI=nobody", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "EventLog.Enabled=true", - "EventLog.BufferKey=1234", - "EventLog.BufferLength=65536", //64KiB - "EventLog.MinBackoff=1us", - "EventLog.MaxBackoff=2us", - } - } else { - benchmarkConfMapStrings = []string{ - "Logging.LogFilePath=/dev/null", - "Logging.TraceLevelLogging=none", - "Logging.DebugLevelLogging=none", - "Logging.LogToConsole=false", - "Cluster.WhoAmI=nobody", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "EventLog.Enabled=false", - } - } - - benchmarkConfMap, err = conf.MakeConfMapFromStrings(benchmarkConfMapStrings) - if nil != err { - b.Fatal(err) - } - - err = transitions.Up(benchmarkConfMap) - if nil != err { - b.Fatal(err) - } - - MarkForDeletion() -} - -func benchmarkTeardown(b *testing.B) { - var ( - err error - ) - - err = transitions.Down(benchmarkConfMap) - if nil != err { - b.Fatal(err) - } -} - -func Benchmark1KRecordTestPatternFixedWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternFixed) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS, "arg0") - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS03DWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS03D, "arg0", uint32(1)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS08XWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS03D, "arg0", uint32(1)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS016XWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS016X, "arg0", uint64(1)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS016X016XWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS016X016X, "arg0", uint64(1), uint64(2)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS016XsliceWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS016Xslice, "arg0", []uint64{uint64(0x101), uint64(0x102), uint64(0x103)}) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS016XSWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS016XS, "arg0", uint64(1), "arg..2") - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSSWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSS, "arg0", "arg.1") - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSS03DWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSS03D, "arg0", "arg.1", uint32(2)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSSSWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSSS, "arg0", "arg.1", "arg..2") - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSSS03DWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSSS03D, "arg0", "arg.1", "arg..2", uint32(3)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSSS016X03DWhileDisabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSSS016X03D, "arg0", "arg.1", "arg..2", uint64(3), uint32(4)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternFixedWhileEnabled(b *testing.B) { - benchmarkSetup(b, false) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternFixed) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS, "arg0") - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS03DWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS03D, "arg0", uint32(1)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS08XWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS03D, "arg0", uint32(1)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS016XWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS016X, "arg0", uint64(1)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS016X016XWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS016X016X, "arg0", uint64(1), uint64(2)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS016XsliceWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS016Xslice, "arg0", []uint64{uint64(0x101), uint64(0x102), uint64(0x103)}) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternS016XSWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternS016XS, "arg0", uint64(1), "arg..2") - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSSWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSS, "arg0", "arg.1") - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSS03DWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSS03D, "arg0", "arg.1", uint32(2)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSSSWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSSS, "arg0", "arg.1", "arg..2") - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSSS03DWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSSS03D, "arg0", "arg.1", "arg..2", uint32(3)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} - -func Benchmark1KRecordTestPatternSSS016X03DWhileEnabled(b *testing.B) { - benchmarkSetup(b, true) - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 1000; j++ { - Record(FormatTestPatternSSS016X03D, "arg0", "arg.1", "arg..2", uint64(3), uint32(4)) - } - } - b.StopTimer() - benchmarkTeardown(b) -} diff --git a/evtlog/config.go b/evtlog/config.go deleted file mode 100644 index 786a743d..00000000 --- a/evtlog/config.go +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package evtlog - -import ( - "fmt" - "syscall" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -// #include -// #include -// #include -// #include -// #include -// -// uintptr_t shmat_returning_uintptr(int shmid, uintptr_t shmaddr, int shmflg) { -// void *shmat_return; -// shmat_return = shmat(shmid, (void *)shmaddr, shmflg); -// return (uintptr_t)shmat_return; -// } -// -// int shmdt_returning_errno(uintptr_t shmaddr) { -// int shmdt_return; -// shmdt_return = shmdt((void *)shmaddr); -// if (0 == shmdt_return) { -// return 0; -// } else { -// return errno; -// } -// } -import "C" - -type globalsStruct struct { - trackedlock.Mutex // While there can only ever be a single Consumer, multiple Producers are possible (within the same process) - eventLogEnabled bool - eventLogBufferKey uint64 - eventLogBufferLength uint64 - eventLogLockMinBackoff time.Duration - eventLogLockMaxBackoff time.Duration - shmKey C.key_t - shmSize C.size_t - shmID C.int - shmAddr C.uintptr_t - shmKnownToBeInitialized bool // Iindicates that this instance "knows" initialization has completed -} - -var globals globalsStruct - -type eventLogConfigSettings struct { - eventLogEnabled bool - eventLogBufferKey uint64 - eventLogBufferLength uint64 - eventLogLockMinBackoff time.Duration - eventLogLockMaxBackoff time.Duration -} - -func init() { - transitions.Register("evtlog", &globals) -} - -// Extract the settings from the confMap and perform minimal sanity checking -// -func parseConfMap(confMap conf.ConfMap) (settings eventLogConfigSettings, err error) { - settings.eventLogEnabled, err = confMap.FetchOptionValueBool("EventLog", "Enabled") - if (nil != err) || !settings.eventLogEnabled { - // ignore parsing errors and treat this as logging disabled - settings.eventLogEnabled = false - err = nil - return - } - - settings.eventLogBufferKey, err = confMap.FetchOptionValueUint64("EventLog", "BufferKey") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueUint32(\"EventLog\", \"BufferKey\") failed: %v", err) - return - } - - settings.eventLogBufferLength, err = confMap.FetchOptionValueUint64("EventLog", "BufferLength") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueUint64(\"EventLog\", \"BufferLength\") failed: %v", err) - return - } - if 0 != (globals.eventLogBufferLength % 4) { - err = fmt.Errorf("confMap.FetchOptionValueUint64(\"EventLog\", \"BufferLength\") not divisible by 4") - return - } - - settings.eventLogLockMinBackoff, err = confMap.FetchOptionValueDuration("EventLog", "MinBackoff") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueDuration(\"EventLog\", \"MinBackoff\") failed: %v", err) - return - } - - settings.eventLogLockMaxBackoff, err = confMap.FetchOptionValueDuration("EventLog", "MaxBackoff") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueDuration(\"EventLog\", \"MaxBackoff\") failed: %v", err) - return - } - - return -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - var settings eventLogConfigSettings - - settings, err = parseConfMap(confMap) - if nil != err { - return - } - - logger.Infof("evtlog.Up(): event logging is %v", settings.eventLogEnabled) - - globals.eventLogEnabled = settings.eventLogEnabled - if !globals.eventLogEnabled { - return - } - - globals.eventLogBufferKey = settings.eventLogBufferKey - globals.eventLogBufferLength = settings.eventLogBufferLength - globals.eventLogLockMinBackoff = settings.eventLogLockMinBackoff - globals.eventLogLockMaxBackoff = settings.eventLogLockMaxBackoff - - err = enableLogging() - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - return nil -} - -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - var settings eventLogConfigSettings - - settings, err = parseConfMap(confMap) - if nil != err { - return - } - - logger.Infof("evtlog.Signaled(): event logging is now %v (was %v)", settings.eventLogEnabled, globals.eventLogEnabled) - - if !settings.eventLogEnabled { - if !globals.eventLogEnabled { - // was disabled and still is; no work to do - return - } - - // was enabled but is now disabled - err = disableLogging() - return - } - - // event logging will be enabled - // - // if it was enabled previously certain settings cannot be changed - if globals.eventLogEnabled { - if settings.eventLogBufferKey != globals.eventLogBufferKey { - err = fmt.Errorf("confMap[EventLog][BufferKey] not modifable without a restart") - return - } - if settings.eventLogBufferLength != globals.eventLogBufferLength { - err = fmt.Errorf("confMap[EventLog][BufferLength] not modifable without a restart") - return - } - } - - globals.eventLogEnabled = settings.eventLogEnabled - globals.eventLogBufferKey = settings.eventLogBufferKey - globals.eventLogBufferLength = settings.eventLogBufferLength - globals.eventLogLockMinBackoff = settings.eventLogLockMinBackoff - globals.eventLogLockMaxBackoff = settings.eventLogLockMaxBackoff - - err = enableLogging() - return -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - if globals.eventLogEnabled { - err = disableLogging() - } - - return -} - -func enableLogging() (err error) { - var ( - rmidResult C.int - errno error - sharedMemoryObjectPermissions C.int - ) - - globals.shmKey = C.key_t(globals.eventLogBufferKey) - globals.shmSize = C.size_t(globals.eventLogBufferLength) - - sharedMemoryObjectPermissions = 0 | - syscall.S_IRUSR | - syscall.S_IWUSR | - syscall.S_IRGRP | - syscall.S_IWGRP | - syscall.S_IROTH | - syscall.S_IWOTH - - globals.shmID, errno = C.shmget(globals.shmKey, globals.shmSize, C.IPC_CREAT|sharedMemoryObjectPermissions) - - if C.int(-1) == globals.shmID { - globals.eventLogEnabled = false - err = fmt.Errorf("C.shmget(globals.shmKey, globals.shmSize, C.IPC_CREAT) failed with errno: %v", errno) - return - } - - globals.shmAddr = C.shmat_returning_uintptr(globals.shmID, C.uintptr_t(0), C.int(0)) - - if ^C.uintptr_t(0) == globals.shmAddr { - globals.eventLogEnabled = false - rmidResult = C.shmctl(globals.shmID, C.IPC_RMID, nil) - if C.int(-1) == rmidResult { - err = fmt.Errorf("C.shmat_returning_uintptr(globals.shmID, C.uintptr_t(0), C.int(0)) then C.shmctl(globals.shmID, C.IPC_RMID, nil) failed") - return - } - err = fmt.Errorf("C.shmat_returning_uintptr(globals.shmID, C.uintptr_t(0), C.int(0)) failed") - return - } - - globals.eventLogEnabled = true - - err = nil - return -} - -func disableLogging() (err error) { - var ( - shmdtErrnoReturn syscall.Errno - shmdtIntReturn C.int - ) - - globals.eventLogEnabled = false - - shmdtIntReturn = C.shmdt_returning_errno(globals.shmAddr) - if C.int(0) != shmdtIntReturn { - shmdtErrnoReturn = syscall.Errno(shmdtIntReturn) - err = fmt.Errorf("C.shmdt() returned non-zero failure... errno: %v", shmdtErrnoReturn.Error()) - return - } - - err = nil - return -} diff --git a/evtlog/pfsevtlogd/Makefile b/evtlog/pfsevtlogd/Makefile deleted file mode 100644 index f9804187..00000000 --- a/evtlog/pfsevtlogd/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/evtlog/pfsevtlogd - -include ../../GoMakefile diff --git a/evtlog/pfsevtlogd/dummy_test.go b/evtlog/pfsevtlogd/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/evtlog/pfsevtlogd/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/evtlog/pfsevtlogd/main.go b/evtlog/pfsevtlogd/main.go deleted file mode 100644 index a585da13..00000000 --- a/evtlog/pfsevtlogd/main.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "log" - "os" - "os/signal" - "syscall" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/transitions" -) - -func usage() { - fmt.Println("pfsevtlogd -?") - fmt.Println(" Prints this help text") - fmt.Println("pfsevtlogd [-D] ConfFile [ConfFileOverrides]*") - fmt.Println(" -D requests that just the Shared Memory Object be deleted") - fmt.Println(" ConfFile specifies the .conf file as also passed to proxyfsd et. al.") - fmt.Println(" ConfFileOverrides is an optional list of modifications to ConfFile to apply") -} - -func main() { - var ( - args []string - confMap conf.ConfMap - err error - formattedRecord string - justDeleteSharedMemoryObject bool - numDroppedRecords uint64 - outputFile *os.File - outputPath string - pollDelay time.Duration - signalChan chan os.Signal - ) - - if (2 == len(os.Args)) && ("-?" == os.Args[1]) { - usage() - os.Exit(0) - } - - if (1 < len(os.Args)) && ("-D" == os.Args[1]) { - justDeleteSharedMemoryObject = true - args = os.Args[2:] - } else { - justDeleteSharedMemoryObject = false - args = os.Args[1:] - } - - if len(args) < 1 { - log.Fatalf("no .conf file specified") - } - - confMap, err = conf.MakeConfMapFromFile(args[0]) - if nil != err { - log.Fatalf("failed to load config: %v", err) - } - - err = confMap.UpdateFromStrings(args[1:]) - if nil != err { - log.Fatalf("failed to apply config overrides: %v", err) - } - - // Upgrade confMap if necessary - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - log.Fatalf("Failed to upgrade config: %v", err) - } - - err = transitions.Up(confMap) - if nil != err { - log.Fatalf("transitions.Up() failed: %v", err) - } - - if justDeleteSharedMemoryObject { - evtlog.MarkForDeletion() - err = transitions.Down(confMap) - if nil != err { - log.Fatalf("transitions.Down() failed: %v", err) - } - os.Exit(0) - } - - pollDelay, err = confMap.FetchOptionValueDuration("EventLog", "DaemonPollDelay") - if nil != err { - log.Fatalf("confMap.FetchOptionValueDuration(\"EventLog\", \"DaemonPollDelay\") failed: %v", err) - } - - outputPath, err = confMap.FetchOptionValueString("EventLog", "DaemonOutputPath") - if nil == err { - outputFile, err = os.OpenFile(outputPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, syscall.S_IRUSR|syscall.S_IWUSR) - } else { - outputFile = os.Stdout - } - - signalChan = make(chan os.Signal, 1) - - signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM) - - for { - formattedRecord, numDroppedRecords = evtlog.Retrieve() - - if 0 != numDroppedRecords { - fmt.Fprintf(outputFile, "*** numDroppedRecords == 0x%016X\n", numDroppedRecords) - } - - if "" == formattedRecord { - select { - case _ = <-signalChan: - err = outputFile.Close() - if nil != err { - log.Fatalf("outputFile.Close() failed: %v", err) - } - - err = transitions.Down(confMap) - if nil != err { - log.Fatalf("transitions.Down() failed: %v", err) - } - - os.Exit(0) - case _ = <-time.After(pollDelay): - // Just loop back to attempt evtlog.Retrieve() - } - } else { - fmt.Fprintf(outputFile, "%s\n", formattedRecord) - } - } -} diff --git a/fs/Makefile b/fs/Makefile deleted file mode 100644 index cb362156..00000000 --- a/fs/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/fs - -include ../GoMakefile diff --git a/fs/api.go b/fs/api.go deleted file mode 100644 index 350bf07c..00000000 --- a/fs/api.go +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package fs, sitting on top of the inode manager, defines the filesystem exposed by ProxyFS. -package fs - -// #include -import "C" - -import ( - "time" - - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/utils" -) - -// ReadRangeIn is the ReadPlan range requested -// -// Either Offset or Len can be omitted, but not both. Those correspond -// to HTTP byteranges "bytes=N-" (no Len; asks for byte N to the end -// of the file) and "bytes=-N" (no Offset; asks for the last N bytes -// of the file). -type ReadRangeIn struct { - Offset *uint64 - Len *uint64 -} - -// Returned by MiddlewareGetAccount -type AccountEntry struct { - Basename string - ModificationTime uint64 // nanoseconds since epoch - AttrChangeTime uint64 // nanoseconds since epoch -} - -// Returned by MiddlewareGetContainer -// -type ContainerEntry struct { - Basename string - FileSize uint64 - ModificationTime uint64 // nanoseconds since epoch - AttrChangeTime uint64 // nanoseconds since epoch - IsDir bool - NumWrites uint64 - InodeNumber uint64 - Metadata []byte -} - -type HeadResponse struct { - Metadata []byte - FileSize uint64 - ModificationTime uint64 // nanoseconds since epoch - AttrChangeTime uint64 // nanoseconds since epoch - IsDir bool - InodeNumber inode.InodeNumber - NumWrites uint64 -} - -// The following constants are used to ensure that the length of file fullpath and basenames are POSIX-compliant -const ( - FilePathMax = C.PATH_MAX - FileNameMax = C.NAME_MAX -) - -// The maximum number of symlinks we will follow -const MaxSymlinks = 8 // same as Linux; see include/linux/namei.h in Linux's Git repository - -// Constant defining the name of the alternate data stream used by Swift Middleware -const MiddlewareStream = "middleware" - -// Base-2 constants -const ( - Kibi = 1024 - Mebi = Kibi * 1024 - Gibi = Mebi * 1024 - Tebi = Gibi * 1024 -) - -// StatVfs defaults -const ( - DefaultReportedBlockSize uint64 = 64 * Kibi - DefaultReportedFragmentSize uint64 = 64 * Kibi - DefaultReportedNumBlocks uint64 = 100 * Tebi / DefaultReportedBlockSize - DefaultReportedNumInodes uint64 = 100 * Gibi -) - -// SetXAttr constants (Go should wrap these from /usr/include/attr/xattr.h>) -const ( - SetXAttrCreateOrReplace = 0 - SetXAttrCreate = 1 - SetXAttrReplace = 2 -) - -type FlockStruct struct { - Type int32 - Whence int32 - Start uint64 - Len uint64 - Pid uint64 -} - -type StatKey uint64 - -const ( - StatCTime StatKey = iota + 1 // time of last inode attribute change (ctime in posix stat) - StatCRTime // time of inode creation (crtime in posix stat) - StatMTime // time of last data modification (mtime in posix stat) - StatATime // time of last data access (atime in posix stat) - StatSize // inode data size in bytes - StatNLink // Number of hard links to the inode - StatFType // file type of inode - StatINum // inode number - StatMode // file mode - StatUserID // file userid - StatGroupID // file groupid - StatNumWrites // number of writes to inode -) - -// TODO: StatMode, StatUserID, and StatGroupID are really -// uint32, not uint64. How to expose a stat map with -// values of different types? -type Stat map[StatKey]uint64 // key is one of StatKey consts - -// Whole-filesystem stats for StatVfs calls -// -type StatVFSKey uint64 - -const ( - StatVFSBlockSize StatVFSKey = iota + 1 // statvfs.f_bsize - Filesystem block size - StatVFSFragmentSize // statvfs.f_frsize - Filesystem fragment size, smallest addressable data size in the filesystem - StatVFSTotalBlocks // statvfs.f_blocks - Filesystem size in StatVFSFragmentSize units - StatVFSFreeBlocks // statvfs.f_bfree - number of free blocks - StatVFSAvailBlocks // statvfs.f_bavail - number of free blocks for unprivileged users - StatVFSTotalInodes // statvfs.f_files - number of inodes in the filesystem - StatVFSFreeInodes // statvfs.f_ffree - number of free inodes in the filesystem - StatVFSAvailInodes // statvfs.f_favail - number of free inodes for unprivileged users - StatVFSFilesystemID // statvfs.f_fsid - Our filesystem ID - StatVFSMountFlags // statvfs.f_flag - mount flags - StatVFSMaxFilenameLen // statvfs.f_namemax - maximum filename length -) - -type StatVFS map[StatVFSKey]uint64 // key is one of StatVFSKey consts - -type JobHandle interface { - Active() (active bool) - Wait() - Cancel() - Error() (err []string) - Info() (info []string) -} - -// Volume handle interface - -func FetchVolumeHandleByAccountName(accountName string) (volumeHandle VolumeHandle, err error) { - volumeHandle, err = fetchVolumeHandleByAccountName(accountName) - return -} - -func FetchVolumeHandleByVolumeName(volumeName string) (volumeHandle VolumeHandle, err error) { - volumeHandle, err = fetchVolumeHandleByVolumeName(volumeName) - return -} - -type VolumeHandle interface { - Access(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, accessMode inode.InodeMode) (accessReturn bool) - CallInodeToProvisionObject() (pPath string, err error) - Create(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, dirInodeNumber inode.InodeNumber, basename string, filePerm inode.InodeMode) (fileInodeNumber inode.InodeNumber, err error) - DefragmentFile(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, fileInodeNumber inode.InodeNumber) (err error) - Destroy(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (err error) - FetchExtentMapChunk(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, fileInodeNumber inode.InodeNumber, fileOffset uint64, maxEntriesFromFileOffset int64, maxEntriesBeforeFileOffset int64) (extentMapChunk *inode.ExtentMapChunkStruct, err error) - Flush(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (err error) - Flock(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, lockCmd int32, inFlockStruct *FlockStruct) (outFlockStruct *FlockStruct, err error) - Getstat(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (stat Stat, err error) - GetType(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (inodeType inode.InodeType, err error) - GetXAttr(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, streamName string) (value []byte, err error) - IsDir(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (inodeIsDir bool, err error) - IsFile(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (inodeIsFile bool, err error) - IsSymlink(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (inodeIsSymlink bool, err error) - Link(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, dirInodeNumber inode.InodeNumber, basename string, targetInodeNumber inode.InodeNumber) (err error) - ListXAttr(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (streamNames []string, err error) - Lookup(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, dirInodeNumber inode.InodeNumber, basename string) (inodeNumber inode.InodeNumber, err error) - LookupPath(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, fullpath string) (inodeNumber inode.InodeNumber, err error) - MiddlewareCoalesce(destPath string, metaData []byte, elementPaths []string) (ino uint64, numWrites uint64, attrChangeTime uint64, modificationTime uint64, err error) - MiddlewareDelete(parentDir string, baseName string) (err error) - MiddlewareGetAccount(maxEntries uint64, marker string, endmarker string) (accountEnts []AccountEntry, mtime uint64, ctime uint64, err error) - MiddlewareGetContainer(vContainerName string, maxEntries uint64, marker string, endmarker string, prefix string, delimiter string) (containerEnts []ContainerEntry, err error) - MiddlewareGetObject(containerObjectPath string, readRangeIn []ReadRangeIn, readRangeOut *[]inode.ReadPlanStep) (response HeadResponse, err error) - MiddlewareHeadResponse(entityPath string) (response HeadResponse, err error) - MiddlewareMkdir(vContainerName string, vObjectPath string, metadata []byte) (mtime uint64, ctime uint64, inodeNumber inode.InodeNumber, numWrites uint64, err error) - MiddlewarePost(parentDir string, baseName string, newMetaData []byte, oldMetaData []byte) (err error) - MiddlewarePutComplete(vContainerName string, vObjectPath string, pObjectPaths []string, pObjectLengths []uint64, pObjectMetadata []byte) (mtime uint64, ctime uint64, fileInodeNumber inode.InodeNumber, numWrites uint64, err error) - MiddlewarePutContainer(containerName string, oldMetadata []byte, newMetadata []byte) (err error) - Mkdir(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, basename string, filePerm inode.InodeMode) (newDirInodeNumber inode.InodeNumber, err error) - Move(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, srcDirInodeNumber inode.InodeNumber, srcBasename string, dstDirInodeNumber inode.InodeNumber, dstBasename string) (toDestroyInodeNumber inode.InodeNumber, err error) - RemoveXAttr(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, streamName string) (err error) - Rename(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, srcDirInodeNumber inode.InodeNumber, srcBasename string, dstDirInodeNumber inode.InodeNumber, dstBasename string) (err error) - Read(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, offset uint64, length uint64, profiler *utils.Profiler) (buf []byte, err error) - Readdir(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, maxEntries uint64, prevReturned ...interface{}) (entries []inode.DirEntry, numEntries uint64, areMoreEntries bool, err error) - ReaddirPlus(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, maxEntries uint64, prevReturned ...interface{}) (dirEntries []inode.DirEntry, statEntries []Stat, numEntries uint64, areMoreEntries bool, err error) - Readsymlink(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (target string, err error) - Resize(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, newSize uint64) (err error) - Rmdir(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, basename string) (err error) - Setstat(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, stat Stat) (err error) - SetXAttr(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, streamName string, value []byte, flags int) (err error) - StatVfs() (statVFS StatVFS, err error) - Symlink(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, basename string, target string) (symlinkInodeNumber inode.InodeNumber, err error) - Unlink(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, basename string) (err error) - VolumeName() (volumeName string) - Write(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, offset uint64, buf []byte, profiler *utils.Profiler) (size uint64, err error) - Wrote(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, containerName string, objectName string, fileOffset []uint64, objectOffset []uint64, length []uint64, wroteTime uint64) (err error) -} - -// ValidateVolume performs an "FSCK" on the specified volumeName. -func ValidateVolume(volumeName string) (validateVolumeHandle JobHandle) { - var ( - vVS *validateVolumeStruct - ) - startTime := time.Now() - defer func() { - globals.ValidateVolumeUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - vVS = &validateVolumeStruct{} - - vVS.jobType = "FSCK" - vVS.volumeName = volumeName - vVS.active = true - vVS.stopFlag = false - vVS.err = make([]string, 0) - vVS.info = make([]string, 0) - - vVS.globalWaitGroup.Add(1) - go vVS.validateVolume() - - validateVolumeHandle = vVS - - return -} - -// ScrubVolume performs a "SCRUB" on the specified volumeName. -func ScrubVolume(volumeName string) (scrubVolumeHandle JobHandle) { - var ( - sVS *scrubVolumeStruct - ) - startTime := time.Now() - defer func() { - globals.ScrubVolumeUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - sVS = &scrubVolumeStruct{} - - sVS.jobType = "SCRUB" - sVS.volumeName = volumeName - sVS.active = true - sVS.stopFlag = false - sVS.err = make([]string, 0) - sVS.info = make([]string, 0) - - sVS.globalWaitGroup.Add(1) - go sVS.scrubVolume() - - scrubVolumeHandle = sVS - - return -} - -// Utility functions - -func ValidateBaseName(baseName string) (err error) { - startTime := time.Now() - defer func() { - globals.ValidateBaseNameUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.ValidateBaseNameErrors.Add(1) - } - }() - - err = validateBaseName(baseName) - return -} - -func ValidateFullPath(fullPath string) (err error) { - startTime := time.Now() - defer func() { - globals.ValidateFullPathUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.ValidateFullPathErrors.Add(1) - } - }() - - err = validateFullPath(fullPath) - return -} - -func AccountNameToVolumeName(accountName string) (volumeName string, ok bool) { - startTime := time.Now() - defer func() { - globals.AccountNameToVolumeNameUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - volumeName, ok = inode.AccountNameToVolumeName(accountName) - return -} - -func VolumeNameToActivePeerPrivateIPAddr(volumeName string) (activePeerPrivateIPAddr string, ok bool) { - startTime := time.Now() - defer func() { - globals.VolumeNameToActivePeerPrivateIPAddrUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - activePeerPrivateIPAddr, ok = inode.VolumeNameToActivePeerPrivateIPAddr(volumeName) - return -} diff --git a/fs/api_internal.go b/fs/api_internal.go deleted file mode 100644 index 17a7c5fc..00000000 --- a/fs/api_internal.go +++ /dev/null @@ -1,4449 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package fs, sitting on top of the inode manager, defines the filesystem exposed by ProxyFS. -package fs - -import ( - "bytes" - "container/list" - "fmt" - "math" - "path" - "strings" - "syscall" - "time" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/dlm" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/utils" -) - -// Shorthand for our internal API debug log id; global to the package -var internalDebug = logger.DbgInternal - -type symlinkFollowState struct { - seen map[inode.InodeNumber]bool - traversed int -} - -// Let us sort an array of directory and file names -type dirAndFileName struct { - dirName string - fileName string -} - -// this has to be a named type to be a method receiver -type dirAndFileNameSlice []dirAndFileName - -func (coll dirAndFileNameSlice) Len() int { - return len(coll) -} - -func (coll dirAndFileNameSlice) Less(i int, j int) bool { - return coll[i].dirName < coll[j].dirName -} - -func (coll dirAndFileNameSlice) Swap(i int, j int) { - coll[i], coll[j] = coll[j], coll[i] -} - -// trackInFlightFileInodeData is called to ensure a timely Flush occurs. -// -// Only Write() will call this while holding a WriteLock on the fileInode -// either just before or just after its call to inode.Write(). -func (vS *volumeStruct) trackInFlightFileInodeData(inodeNumber inode.InodeNumber) { - var ( - inFlightFileInodeData *inFlightFileInodeDataStruct - ok bool - ) - - globals.Lock() - vS.dataMutex.Lock() - inFlightFileInodeData, ok = vS.inFlightFileInodeDataMap[inodeNumber] - if !ok { - inFlightFileInodeData = &inFlightFileInodeDataStruct{ - InodeNumber: inodeNumber, - volStruct: vS, - control: make(chan bool, inFlightFileInodeDataControlBuffering), - } - vS.inFlightFileInodeDataMap[inodeNumber] = inFlightFileInodeData - inFlightFileInodeData.globalsListElement = globals.inFlightFileInodeDataList.PushBack(inFlightFileInodeData) - inFlightFileInodeData.wg.Add(1) - go inFlightFileInodeData.inFlightFileInodeDataTracker() - } - vS.dataMutex.Unlock() - globals.Unlock() -} - -// untrackInFlightInodeData is called once it is known a Flush() is no longer needed -// or to actually request a Flush() [as would be the case during unmounting a volume]. -func (vS *volumeStruct) untrackInFlightFileInodeData(inodeNumber inode.InodeNumber, flushFirst bool) { - var ( - inFlightFileInodeData *inFlightFileInodeDataStruct - ok bool - ) - - globals.Lock() - vS.dataMutex.Lock() - inFlightFileInodeData, ok = vS.inFlightFileInodeDataMap[inodeNumber] - if !ok { - vS.dataMutex.Unlock() - globals.Unlock() - return - } - delete(vS.inFlightFileInodeDataMap, inodeNumber) - if nil != inFlightFileInodeData.globalsListElement { - _ = globals.inFlightFileInodeDataList.Remove(inFlightFileInodeData.globalsListElement) - inFlightFileInodeData.globalsListElement = nil - } - inFlightFileInodeData.control <- flushFirst - vS.dataMutex.Unlock() - globals.Unlock() - if flushFirst { - inFlightFileInodeData.wg.Wait() - } -} - -// untrackInFlightFileInodeDataAll is called to flush all current elements -// of vS.inFlightFileInodeDataMap (if any) during SIGHUP or Down(). -func (vS *volumeStruct) untrackInFlightFileInodeDataAll() { - var ( - inFlightFileInodeNumber inode.InodeNumber - inFlightFileInodeNumbers []inode.InodeNumber - inFlightFileInodeNumbersCapacity int - ) - - // Snapshot list of inode.InodeNumber's currently in vS.inFlightFileInodeDataMap - - vS.dataMutex.Lock() - inFlightFileInodeNumbersCapacity = len(vS.inFlightFileInodeDataMap) - if 0 == inFlightFileInodeNumbersCapacity { - vS.dataMutex.Unlock() - return - } - inFlightFileInodeNumbers = make([]inode.InodeNumber, 0, inFlightFileInodeNumbersCapacity) - for inFlightFileInodeNumber, _ = range vS.inFlightFileInodeDataMap { - inFlightFileInodeNumbers = append(inFlightFileInodeNumbers, inFlightFileInodeNumber) - } - vS.dataMutex.Unlock() - - // Now go flush each of those - - for _, inFlightFileInodeNumber = range inFlightFileInodeNumbers { - vS.untrackInFlightFileInodeData(inFlightFileInodeNumber, true) - } -} - -func (vS *volumeStruct) inFlightFileInodeDataFlusher(inodeNumber inode.InodeNumber) { - var ( - err error - inodeLock *dlm.RWLockStruct - stillExists bool - ) - - // Act as if a package fs client called Flush()... - - inodeLock, err = vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if nil != err { - logger.PanicfWithError(err, "InitInodeLock() for volume '%s' inode %v failed", vS.volumeName, inodeNumber) - } - err = inodeLock.WriteLock() - if nil != err { - logger.PanicfWithError(err, "dlm.Writelock() for volume '%s' inode %v failed", vS.volumeName, inodeNumber) - } - - stillExists = vS.inodeVolumeHandle.Access(inodeNumber, inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.F_OK, - inode.NoOverride) - if stillExists { - err = vS.inodeVolumeHandle.Flush(inodeNumber, false) - if nil == err { - vS.untrackInFlightFileInodeData(inodeNumber, false) - } else { - logger.ErrorfWithError(err, "Flush of file data failed on volume '%s' inode %v", vS.volumeName, inodeNumber) - } - } - - err = inodeLock.Unlock() - if nil != err { - logger.PanicfWithError(err, "dlm.Unlock() for volume '%s' inode %v failed", vS.volumeName, inodeNumber) - } -} - -func (inFlightFileInodeData *inFlightFileInodeDataStruct) inFlightFileInodeDataTracker() { - var ( - flushFirst bool - ) - - logger.Tracef("fs.inFlightFileInodeDataTracker(): waiting to flush volume '%s' inode %v", - inFlightFileInodeData.volStruct.volumeName, inFlightFileInodeData.InodeNumber) - - select { - case flushFirst = <-inFlightFileInodeData.control: - // All we needed was the value of flushFirst from control chan - case <-time.After(inFlightFileInodeData.volStruct.maxFlushTime): - flushFirst = true - } - - logger.Tracef("fs.inFlightFileInodeDataTracker(): flush starting for volume '%s' inode %v flushfirst %t", - inFlightFileInodeData.volStruct.volumeName, inFlightFileInodeData.InodeNumber, flushFirst) - - if flushFirst { - inFlightFileInodeData.volStruct.inFlightFileInodeDataFlusher(inFlightFileInodeData.InodeNumber) - } - - inFlightFileInodeData.wg.Done() -} - -func fetchVolumeHandleByAccountName(accountName string) (volumeHandle VolumeHandle, err error) { - var ( - ok bool - vS *volumeStruct - volumeName string - ) - - startTime := time.Now() - defer func() { - globals.FetchVolumeHandleUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.FetchVolumeHandleErrors.Add(1) - } - }() - - globals.Lock() - - volumeName, ok = inode.AccountNameToVolumeName(accountName) - if !ok { - err = fmt.Errorf("Unknown accountName passed to mountByAccountName(): \"%s\"", accountName) - err = blunder.AddError(err, blunder.NotFoundError) - globals.Unlock() - return - } - - vS, ok = globals.volumeMap[volumeName] - if !ok { - err = fmt.Errorf("Unknown volumeName computed by mountByAccountName(): \"%s\"", volumeName) - err = blunder.AddError(err, blunder.NotFoundError) - globals.Unlock() - return - } - - globals.Unlock() - - volumeHandle = vS - err = nil - - return -} - -func fetchVolumeHandleByVolumeName(volumeName string) (volumeHandle VolumeHandle, err error) { - var ( - ok bool - vS *volumeStruct - ) - - startTime := time.Now() - defer func() { - globals.FetchVolumeHandleUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.FetchVolumeHandleErrors.Add(1) - } - }() - - globals.Lock() - - vS, ok = globals.volumeMap[volumeName] - if !ok { - err = fmt.Errorf("Unknown volumeName passed to mountByVolumeName(): \"%s\"", volumeName) - err = blunder.AddError(err, blunder.NotFoundError) - globals.Unlock() - return - } - - globals.Unlock() - - volumeHandle = vS - err = nil - - return -} - -func (vS *volumeStruct) Access(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, accessMode inode.InodeMode) (accessReturn bool) { - startTime := time.Now() - defer func() { - globals.AccessUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - accessReturn = vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, accessMode, - inode.NoOverride) - return -} - -func (vS *volumeStruct) CallInodeToProvisionObject() (pPath string, err error) { - startTime := time.Now() - defer func() { - globals.CallInodeToProvisionObjectUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.CallInodeToProvisionObjectErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - pPath, err = vS.inodeVolumeHandle.ProvisionObject() - return -} - -func (vS *volumeStruct) Create(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, dirInodeNumber inode.InodeNumber, basename string, filePerm inode.InodeMode) (fileInodeNumber inode.InodeNumber, err error) { - startTime := time.Now() - defer func() { - globals.CreateUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.CreateErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - err = validateBaseName(basename) - if err != nil { - return 0, err - } - - // Lock the directory inode before doing the link - dirInodeLock, err := vS.inodeVolumeHandle.InitInodeLock(dirInodeNumber, nil) - if err != nil { - return 0, err - } - err = dirInodeLock.WriteLock() - if err != nil { - return 0, err - } - defer dirInodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(dirInodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - return 0, blunder.NewError(blunder.NotFoundError, "ENOENT") - } - if !vS.inodeVolumeHandle.Access(dirInodeNumber, userID, groupID, otherGroupIDs, inode.W_OK|inode.X_OK, - inode.NoOverride) { - return 0, blunder.NewError(blunder.PermDeniedError, "EACCES") - } - - // create the file and add it to the directory - fileInodeNumber, err = vS.inodeVolumeHandle.CreateFile(filePerm, userID, groupID) - if err != nil { - return 0, err - } - - err = vS.inodeVolumeHandle.Link(dirInodeNumber, basename, fileInodeNumber, false) - if err != nil { - destroyErr := vS.inodeVolumeHandle.Destroy(fileInodeNumber) - if destroyErr != nil { - logger.WarnfWithError(destroyErr, "couldn't destroy inode %v after failed Link() in fs.Create", fileInodeNumber) - } - return 0, err - } - - return fileInodeNumber, nil -} - -func (vS *volumeStruct) DefragmentFile(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, fileInodeNumber inode.InodeNumber) (err error) { - var ( - eofReached bool - fileOffset uint64 - inodeLock *dlm.RWLockStruct - inodeType inode.InodeType - ) - - startTime := time.Now() - defer func() { - globals.DefragmentFileUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.DefragmentFileErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - - inodeLock, err = vS.inodeVolumeHandle.InitInodeLock(fileInodeNumber, nil) - if nil != err { - vS.jobRWMutex.RUnlock() - return - } - err = inodeLock.WriteLock() - if nil != err { - vS.jobRWMutex.RUnlock() - return - } - - if !vS.inodeVolumeHandle.Access(fileInodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - _ = inodeLock.Unlock() - vS.jobRWMutex.RUnlock() - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(fileInodeNumber, userID, groupID, otherGroupIDs, inode.W_OK, - inode.OwnerOverride) { - _ = inodeLock.Unlock() - vS.jobRWMutex.RUnlock() - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - inodeType, err = vS.inodeVolumeHandle.GetType(fileInodeNumber) - if nil != err { - _ = inodeLock.Unlock() - vS.jobRWMutex.RUnlock() - logger.ErrorfWithError(err, "couldn't get type for inode %v", fileInodeNumber) - return - } - // Make sure the inode number is for a file inode - if inodeType != inode.FileType { - _ = inodeLock.Unlock() - vS.jobRWMutex.RUnlock() - err = fmt.Errorf("%s: expected inode %v to be a file inode, got %v", utils.GetFnName(), fileInodeNumber, inodeType) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFileError) - return - } - - fileOffset = 0 - - for { - fileOffset, eofReached, err = vS.inodeVolumeHandle.DefragmentFile(fileInodeNumber, fileOffset, vS.fileDefragmentChunkSize) - _ = inodeLock.Unlock() - vS.jobRWMutex.RUnlock() - if nil != err { - return - } - if eofReached { - return - } - time.Sleep(vS.fileDefragmentChunkDelay) - vS.jobRWMutex.RLock() - err = inodeLock.WriteLock() - if nil != err { - vS.jobRWMutex.RUnlock() - return - } - } -} - -func (vS *volumeStruct) FetchExtentMapChunk(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, fileInodeNumber inode.InodeNumber, fileOffset uint64, maxEntriesFromFileOffset int64, maxEntriesBeforeFileOffset int64) (extentMapChunk *inode.ExtentMapChunkStruct, err error) { - var ( - inodeLock *dlm.RWLockStruct - inodeType inode.InodeType - ) - - startTime := time.Now() - defer func() { - globals.FetchExtentMapChunkUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.FetchExtentMapChunkErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err = vS.inodeVolumeHandle.InitInodeLock(fileInodeNumber, nil) - if nil != err { - return - } - err = inodeLock.ReadLock() - if nil != err { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(fileInodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(fileInodeNumber, userID, groupID, otherGroupIDs, inode.R_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - inodeType, err = vS.inodeVolumeHandle.GetType(fileInodeNumber) - if nil != err { - logger.ErrorfWithError(err, "couldn't get type for inode %v", fileInodeNumber) - return - } - // Make sure the inode number is for a file inode - if inodeType != inode.FileType { - err = fmt.Errorf("%s: expected inode %v to be a file inode, got %v", utils.GetFnName(), fileInodeNumber, inodeType) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFileError) - return - } - - extentMapChunk, err = vS.inodeVolumeHandle.FetchExtentMapChunk(fileInodeNumber, fileOffset, maxEntriesFromFileOffset, maxEntriesBeforeFileOffset) - - return -} - -// doInlineCheckpointIfEnabled is called whenever we must guarantee that reported state changes -// are, indeed, persisted. Absent any sort of persistent transaction log, this means performing -// a checkpoint unfortunately. -// -// Currently, only explicitly invoked Flushes trigger this. But, actually, any Swift/S3 API call -// that modifies Objects or (what the client thinks are) Containers should also. -// -// TODO is to determine where else a call to this func should also be made. -// -func (vS *volumeStruct) doInlineCheckpointIfEnabled() { - var ( - err error - ) - - if !vS.doCheckpointPerFlush { - return - } - - err = vS.headhunterVolumeHandle.DoCheckpoint() - if nil != err { - logger.Fatalf("fs.doInlineCheckpoint() call to headhunter.DoCheckpoint() failed: %v", err) - } -} - -func (vS *volumeStruct) Flush(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (err error) { - startTime := time.Now() - defer func() { - globals.FlushUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.FlushErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - return blunder.NewError(blunder.NotFoundError, "ENOENT") - } - - // Note: We'd normally check EACCES here...but there are paths in FUSE (e.g. when files are - // closed) that end up calling Flush()...even though the file was "opened" ReadOnly. - // This is presumably to support updated of ATime and such. In any event, an EACCESS - // check would fail if the caller actually only had ReadOnly access to the Inode, so - // we won't be doing the check here. - - err = vS.inodeVolumeHandle.Flush(inodeNumber, false) - vS.untrackInFlightFileInodeData(inodeNumber, false) - - vS.doInlineCheckpointIfEnabled() - - return -} - -func (vS *volumeStruct) getFileLockList(inodeNumber inode.InodeNumber) (flockList *list.List) { - vS.dataMutex.Lock() - defer vS.dataMutex.Unlock() - - flockList, ok := vS.FLockMap[inodeNumber] - if !ok { - flockList = new(list.List) - vS.FLockMap[inodeNumber] = flockList - } - - return -} - -// Check for lock conflict with other Pids, if there is a conflict then it will return the first occurance of conflicting range. -func checkConflict(elm *FlockStruct, flock *FlockStruct) bool { - - if flock.Pid == elm.Pid { - return false - } - - if (elm.Start+elm.Len) <= flock.Start || (flock.Start+flock.Len) <= elm.Start { - return false - } - - if (flock.Type == syscall.F_WRLCK) || (elm.Type == syscall.F_WRLCK) { - return true - } - - return false -} - -func (vS *volumeStruct) verifyLock(inodeNumber inode.InodeNumber, flock *FlockStruct) (conflictLock *FlockStruct) { - flockList := vS.getFileLockList(inodeNumber) - - for e := flockList.Front(); e != nil; e = e.Next() { - elm := e.Value.(*FlockStruct) - - if checkConflict(elm, flock) == true { - return elm - } - } - - return nil -} - -// Insert a file lock range to corresponding lock list for the pid. -// Assumption: There is no lock conflict and the range that is being inserted has no conflict and is free. -func (vS *volumeStruct) fileLockInsert(inodeNumber inode.InodeNumber, inFlock *FlockStruct) (err error) { - err = nil - flockList := vS.getFileLockList(inodeNumber) - - overlapList := new(list.List) - var beforeElm *list.Element // Refers to the immediate element that starts before the start of the range. - var afterElm *list.Element // Refers to the immediate element that starts after the end of the range. - - // flockList is sorted by starting offset of the range. - // Inserting a range happens in two steps. 1) Check if there is any conflict and also identify the - // point in the list where the entry will be added (before and after elements) 2) Then check if - // the range can extend the before element, if so adjust it. 3) Simillarly, check if the after - // element can be collapsed if it forms a contiguous range. - - for e := flockList.Front(); e != nil; e = e.Next() { - elm := e.Value.(*FlockStruct) - - if (elm.Start + elm.Len) <= inFlock.Start { - beforeElm = e - continue - } - - if elm.Start > (inFlock.Start + inFlock.Len) { - afterElm = e - if overlapList.Len() == 0 { - flockList.InsertBefore(inFlock, e) - return - } - - break - } - - if checkConflict(elm, inFlock) { - err = blunder.AddError(nil, blunder.TryAgainError) - return - } - - if elm.Pid == inFlock.Pid { - overlapList.PushBack(e) - } - } - - if overlapList.Len() == 0 { - if beforeElm != nil { - elm := beforeElm.Value.(*FlockStruct) - if elm.Pid == inFlock.Pid && elm.Type == inFlock.Type && (elm.Start+elm.Len) == inFlock.Start { - elm.Len = inFlock.Start + inFlock.Len - elm.Len - } else { - flockList.InsertAfter(inFlock, beforeElm) - } - } else { - flockList.PushBack(inFlock) - } - - return - } - - // Look at the last element in the overlapping list - lastEnt := overlapList.Back() - e := lastEnt.Value.(*list.Element) - elm := e.Value.(*FlockStruct) - if (elm.Start + elm.Len) > (inFlock.Start + inFlock.Len) { - inFlock.Len = (elm.Start + elm.Len) - inFlock.Start - } - - // We can delete all the entries in the overlapping list. These entries are replaced by - // the range we are inserting. - for e := overlapList.Front(); e != nil; e = e.Next() { - entry := e.Value.(*list.Element) - flockList.Remove(entry) - } - - // Now adjust the before and after entries: - // First adjust the after: - if afterElm != nil { - elm := afterElm.Value.(*FlockStruct) - if elm.Pid == inFlock.Pid && elm.Type == inFlock.Type && (inFlock.Start+inFlock.Len) == elm.Start { - // We can collapse the entry: - elm.Len = elm.Start + elm.Len - inFlock.Start - elm.Start = inFlock.Start - - if beforeElm != nil { - belm := beforeElm.Value.(*FlockStruct) - if belm.Pid == elm.Pid && belm.Type == elm.Type && (belm.Start+belm.Len) == elm.Start { - belm.Len = elm.Start + elm.Len - belm.Start - flockList.Remove(afterElm) - } - } - - return - } - } - - if beforeElm != nil { - belm := beforeElm.Value.(*FlockStruct) - if belm.Pid == inFlock.Pid && belm.Type == inFlock.Type && (belm.Start+belm.Len) == inFlock.Start { - belm.Len = inFlock.Start + inFlock.Len - belm.Start - } - - flockList.InsertAfter(inFlock, beforeElm) - return - } - - if afterElm != nil { - flockList.InsertBefore(inFlock, afterElm) - } else { - flockList.PushBack(inFlock) - } - - return - -} - -// Unlock a given range. All locks held in this range by the process (identified by Pid) are removed. -func (vS *volumeStruct) fileUnlock(inodeNumber inode.InodeNumber, inFlock *FlockStruct) (err error) { - - flockList := vS.getFileLockList(inodeNumber) - if flockList == nil { - logger.Warnf("Unlock of a region not already locked - %+v", inFlock) - return - } - - start := inFlock.Start - len := inFlock.Len - - removeList := new(list.List) - - for e := flockList.Front(); e != nil; e = e.Next() { - elm := e.Value.(*FlockStruct) - - if elm.Pid != inFlock.Pid { - continue - } - - if (elm.Start + elm.Len) < start { - continue - } - - if elm.Start >= (start + len) { - break - } - - // If the lock falls completely in the range, delete it. - if elm.Start >= start && (elm.Start+elm.Len) <= (start+len) { - removeList.PushBack(e) - continue - } - - // This lock overlapps with the range - three possibalities 1) lock starts before the range, 2) end after range and 3) both. - - elmLen := elm.Start + elm.Len // Save the original length, it is required in case of #3 (both) - - if elm.Start < start { // Handle the first part - lock starts before the range. - elm.Len = start - elm.Start - } - - if elmLen > (start + len) { // Lock extends beyond the unlock range. - if elm.Start > start { // case #2 - // use the existing record - elm.Start = start + len - elm.Len = elmLen - elm.Start - break - } - - // Create a new record - handle case #3 both (starts before the range and extends beyond the range) - elmTail := new(FlockStruct) - elmTail.Start = start + len - elmTail.Len = elmLen - elm.Start - elmTail.Pid = elm.Pid - elmTail.Type = elm.Type - elmTail.Whence = elm.Whence - flockList.InsertAfter(elmTail, e) - break - } - } - - for e := removeList.Front(); e != nil; e = e.Next() { - elm := e.Value.(*list.Element) - flockList.Remove(elm) - } - - return -} - -// Implements file locking conforming to fcntl(2) locking description. F_SETLKW is not implemented. Supports F_SETLW and F_GETLW. -// whence: FS supports only SEEK_SET - starting from 0, since it does not manage file handles, caller is expected to supply the start and length relative to offset ZERO. -func (vS *volumeStruct) Flock(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, lockCmd int32, inFlock *FlockStruct) (outFlock *FlockStruct, err error) { - startTime := time.Now() - defer func() { - switch lockCmd { - - case syscall.F_GETLK: - globals.FlockGetUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.FlockGetErrors.Add(1) - } - - case syscall.F_SETLK: - if inFlock.Type == syscall.F_UNLCK { - globals.FlockUnlockUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.FlockUnlockErrors.Add(1) - } - - } else if inFlock.Type == syscall.F_WRLCK || inFlock.Type == syscall.F_RDLCK { - globals.FlockLockUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.FlockLockErrors.Add(1) - } - } else { - globals.FlockOtherErrors.Add(1) - } - - default: - globals.FlockOtherErrors.Add(1) - } - - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - outFlock = inFlock - - if lockCmd == syscall.F_SETLKW { - err = blunder.AddError(nil, blunder.NotSupportedError) - return - } - - // Make sure the inode does not go away, while we are applying the flock. - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.R_OK, inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - if inFlock.Len == 0 { // If length is ZERO means treat it as whole file. - inFlock.Len = ^uint64(0) - } - - switch lockCmd { - case syscall.F_GETLK: - conflictLock := vS.verifyLock(inodeNumber, inFlock) - if conflictLock != nil { - outFlock = conflictLock - err = blunder.AddError(nil, blunder.TryAgainError) - } else { - outFlock = inFlock - outFlock.Type = syscall.F_UNLCK - } - break - - case syscall.F_SETLK: - if inFlock.Type == syscall.F_UNLCK { - err = vS.fileUnlock(inodeNumber, inFlock) - - } else if inFlock.Type == syscall.F_WRLCK || inFlock.Type == syscall.F_RDLCK { - err = vS.fileLockInsert(inodeNumber, inFlock) - - } else { - err = blunder.NewError(blunder.InvalidArgError, "EINVAL") - return - } - break - - default: - err = blunder.NewError(blunder.InvalidArgError, "EINVAL") - return - } - - return -} - -func (vS *volumeStruct) getstatHelper(inodeNumber inode.InodeNumber, callerID dlm.CallerID) (stat Stat, err error) { - - lockID, err := vS.inodeVolumeHandle.MakeLockID(inodeNumber) - if err != nil { - return - } - if !dlm.IsLockHeld(lockID, callerID, dlm.ANYLOCK) { - err = fmt.Errorf("%s: inode %v lock must be held before calling", utils.GetFnName(), inodeNumber) - return nil, blunder.AddError(err, blunder.NotFoundError) - } - - stat, err = vS.getstatHelperWhileLocked(inodeNumber) - - return -} - -func (vS *volumeStruct) getstatHelperWhileLocked(inodeNumber inode.InodeNumber) (stat Stat, err error) { - var ( - metadata *inode.MetadataStruct - ) - - metadata, err = vS.inodeVolumeHandle.GetMetadata(inodeNumber) - if nil != err { - return - } - - stat = make(map[StatKey]uint64) - - stat[StatCRTime] = uint64(metadata.CreationTime.UnixNano()) - stat[StatMTime] = uint64(metadata.ModificationTime.UnixNano()) - stat[StatCTime] = uint64(metadata.AttrChangeTime.UnixNano()) - stat[StatATime] = uint64(metadata.AccessTime.UnixNano()) - stat[StatSize] = metadata.Size - stat[StatNLink] = metadata.LinkCount - stat[StatFType] = uint64(metadata.InodeType) - stat[StatINum] = uint64(inodeNumber) - stat[StatMode] = uint64(metadata.Mode) - stat[StatUserID] = uint64(metadata.UserID) - stat[StatGroupID] = uint64(metadata.GroupID) - stat[StatNumWrites] = metadata.NumWrites - - return -} - -func (vS *volumeStruct) Getstat(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (stat Stat, err error) { - startTime := time.Now() - defer func() { - globals.GetstatUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.GetstatErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - // Call getstat helper function to do the work - return vS.getstatHelper(inodeNumber, inodeLock.GetCallerID()) -} - -func (vS *volumeStruct) getTypeHelper(inodeNumber inode.InodeNumber, callerID dlm.CallerID) (inodeType inode.InodeType, err error) { - - lockID, err := vS.inodeVolumeHandle.MakeLockID(inodeNumber) - if err != nil { - return - } - if !dlm.IsLockHeld(lockID, callerID, dlm.ANYLOCK) { - err = fmt.Errorf("%s: inode %v lock must be held before calling.", utils.GetFnName(), inodeNumber) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - inodeType, err = vS.inodeVolumeHandle.GetType(inodeNumber) - if err != nil { - logger.ErrorWithError(err, "couldn't get inode type") - return inodeType, err - } - return inodeType, nil -} - -func (vS *volumeStruct) GetType(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (inodeType inode.InodeType, err error) { - startTime := time.Now() - defer func() { - globals.GetTypeUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.GetTypeErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - return vS.getTypeHelper(inodeNumber, inodeLock.GetCallerID()) -} - -func (vS *volumeStruct) GetXAttr(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, streamName string) (value []byte, err error) { - startTime := time.Now() - defer func() { - globals.GetXAttrUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.GetXAttrErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.R_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - value, err = vS.inodeVolumeHandle.GetStream(inodeNumber, streamName) - if err != nil { - // Did not find the requested stream. However this isn't really an error since - // samba will ask for acl-related streams and is fine with not finding them. - logger.TracefWithError(err, "Failed to get XAttr %v of inode %v", streamName, inodeNumber) - } - - return -} - -func (vS *volumeStruct) IsDir(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (inodeIsDir bool, err error) { - startTime := time.Now() - defer func() { - globals.IsDirUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.IsDirErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - lockID, err := vS.inodeVolumeHandle.MakeLockID(inodeNumber) - if err != nil { - return - } - if !dlm.IsLockHeld(lockID, inodeLock.GetCallerID(), dlm.ANYLOCK) { - err = fmt.Errorf("%s: inode %v lock must be held before calling", utils.GetFnName(), inodeNumber) - return false, blunder.AddError(err, blunder.NotFoundError) - } - - inodeType, err := vS.inodeVolumeHandle.GetType(inodeNumber) - if err != nil { - return false, err - } - return inodeType == inode.DirType, nil -} - -func (vS *volumeStruct) IsFile(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (inodeIsFile bool, err error) { - startTime := time.Now() - defer func() { - globals.IsFileUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.IsFileErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - inodeType, err := vS.inodeVolumeHandle.GetType(inodeNumber) - if err != nil { - return false, err - } - - return inodeType == inode.FileType, nil -} - -func (vS *volumeStruct) IsSymlink(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (inodeIsSymlink bool, err error) { - startTime := time.Now() - defer func() { - globals.IsSymlinkUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.IsSymlinkErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - inodeType, err := vS.inodeVolumeHandle.GetType(inodeNumber) - if err != nil { - return false, err - } - - return inodeType == inode.SymlinkType, nil -} - -func (vS *volumeStruct) Link(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, dirInodeNumber inode.InodeNumber, basename string, targetInodeNumber inode.InodeNumber) (err error) { - startTime := time.Now() - defer func() { - globals.LinkUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.LinkErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - var ( - inodeType inode.InodeType - ) - - err = validateBaseName(basename) - if err != nil { - return - } - - // We need both dirInodelock and the targetInode lock to make sure they - // don't go away and linkCount is updated correctly. - callerID := dlm.GenerateCallerID() - dirInodeLock, err := vS.inodeVolumeHandle.InitInodeLock(dirInodeNumber, callerID) - if err != nil { - return - } - - targetInodeLock, err := vS.inodeVolumeHandle.InitInodeLock(targetInodeNumber, callerID) - if err != nil { - return - } - - // Lock the target inode to check its type and insure its not a directory (if it is a - // directory then locking it after the target directory could result in deadlock). - err = targetInodeLock.WriteLock() - if err != nil { - return - } - - // make sure target inode is not a directory - inodeType, err = vS.inodeVolumeHandle.GetType(targetInodeNumber) - if err != nil { - targetInodeLock.Unlock() - // Because we know that GetType() has already "blunderized" the error, we just pass it on - logger.ErrorfWithError(err, "%s: couldn't get type for inode %v", utils.GetFnName(), targetInodeNumber) - return err - } - if inodeType == inode.DirType { - targetInodeLock.Unlock() - // no need to print an error when its a mistake by the client - err = fmt.Errorf("%s: inode %v cannot be a dir inode", utils.GetFnName(), targetInodeNumber) - return blunder.AddError(err, blunder.LinkDirError) - } - - // drop the target inode lock so we can get the directory lock then - // reget the target inode lock - targetInodeLock.Unlock() - - err = dirInodeLock.WriteLock() - if err != nil { - return - } - defer dirInodeLock.Unlock() - - err = targetInodeLock.WriteLock() - if err != nil { - return - } - defer targetInodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(dirInodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(targetInodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(dirInodeNumber, userID, groupID, otherGroupIDs, inode.W_OK|inode.X_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - err = vS.inodeVolumeHandle.Link(dirInodeNumber, basename, targetInodeNumber, false) - - // if the link was successful and this is a regular file then any - // pending data was flushed - if err == nil && inodeType == inode.FileType { - vS.untrackInFlightFileInodeData(targetInodeNumber, false) - } - - return err -} - -func (vS *volumeStruct) ListXAttr(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (streamNames []string, err error) { - startTime := time.Now() - defer func() { - globals.ListXAttrUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.ListXAttrErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.R_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - metadata, err := vS.inodeVolumeHandle.GetMetadata(inodeNumber) - if err != nil { - // Did not find the requested stream. However this isn't really an error since - // samba will ask for acl-related streams and is fine with not finding them. - logger.TracefWithError(err, "Failed to list XAttrs of inode %v", inodeNumber) - return - } - - streamNames = make([]string, len(metadata.InodeStreamNameSlice)) - copy(streamNames, metadata.InodeStreamNameSlice) - return -} - -func (vS *volumeStruct) Lookup(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, dirInodeNumber inode.InodeNumber, basename string) (inodeNumber inode.InodeNumber, err error) { - startTime := time.Now() - defer func() { - globals.LookupUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.LookupErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - dirInodeLock, err := vS.inodeVolumeHandle.InitInodeLock(dirInodeNumber, nil) - if err != nil { - return - } - dirInodeLock.ReadLock() - defer dirInodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(dirInodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(dirInodeNumber, userID, groupID, otherGroupIDs, inode.X_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - inodeNumber, err = vS.inodeVolumeHandle.Lookup(dirInodeNumber, basename) - return inodeNumber, err -} - -func (vS *volumeStruct) LookupPath(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, fullpath string) (inodeNumber inode.InodeNumber, err error) { - startTime := time.Now() - defer func() { - globals.LookupPathUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.LookupPathErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - // In the special case of a fullpath starting with "/", the path segment splitting above - // results in a first segment that still begins with "/". Because this is not recognized - // as a real path segment, by the underlying code, we have trouble looking it up. - // - // This is a hack to work around this case until I figure out a better way. - newfullpath := strings.TrimPrefix(fullpath, "/") - if strings.Compare(fullpath, newfullpath) != 0 { - fullpath = newfullpath - } - - pathSegments := strings.Split(path.Clean(fullpath), "/") - - cursorInodeNumber := inode.RootDirInodeNumber - for _, segment := range pathSegments { - cursorInodeLock, err1 := vS.inodeVolumeHandle.InitInodeLock(cursorInodeNumber, nil) - if err = err1; err != nil { - return - } - err = cursorInodeLock.ReadLock() - if err != nil { - return - } - - if !vS.inodeVolumeHandle.Access(cursorInodeNumber, userID, groupID, otherGroupIDs, inode.X_OK, - inode.NoOverride) { - cursorInodeLock.Unlock() - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - cursorInodeNumber, err = vS.inodeVolumeHandle.Lookup(cursorInodeNumber, segment) - cursorInodeLock.Unlock() - - if err != nil { - return cursorInodeNumber, err - } - } - - return cursorInodeNumber, nil -} - -func (vS *volumeStruct) MiddlewareCoalesce(destPath string, metaData []byte, elementPaths []string) ( - ino uint64, numWrites uint64, attrChangeTime uint64, modificationTime uint64, err error) { - - var ( - coalesceElementList []*inode.CoalesceElement - coalesceSize uint64 - ctime time.Time - destFileInodeNumber inode.InodeNumber - dirEntryBasename string - dirEntryInodeNumber inode.InodeNumber - dirInodeNumber inode.InodeNumber - elementPathIndex int - elementPathIndexAtChunkEnd int - elementPathIndexAtChunkStart int - heldLocks *heldLocksStruct - mtime time.Time - retryRequired bool - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - startTime := time.Now() - defer func() { - globals.MiddlewareCoalesceUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.MiddlewareCoalesceBytes.Add(coalesceSize) - if err != nil { - globals.MiddlewareCoalesceErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - // First create the destination file if necessary and ensure that it is empty - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -RestartDestinationFileCreation: - - tryLockBackoffContext.backoff() - - heldLocks = newHeldLocks() - - _, destFileInodeNumber, _, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - destPath, - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks| - resolvePathCreateMissingPathElements| - resolvePathRequireExclusiveLockOnDirEntryInode) - - if nil != err { - heldLocks.free() - return - } - - if retryRequired { - heldLocks.free() - goto RestartDestinationFileCreation - } - - vS.inodeVolumeHandle.SetSize(destFileInodeNumber, 0) - - heldLocks.free() - - // Now setup for looping through elementPaths with fresh locks - // every globals.coalesceElementChunkSize elements holding an - // Exclusive Lock on each FileInode and their containing DirInode - - elementPathIndexAtChunkStart = 0 - - for elementPathIndexAtChunkStart < len(elementPaths) { - elementPathIndexAtChunkEnd = elementPathIndexAtChunkStart + int(globals.coalesceElementChunkSize) - if elementPathIndexAtChunkEnd > len(elementPaths) { - elementPathIndexAtChunkEnd = len(elementPaths) - } - - // Coalesce elementPaths[elementPathIndexAtChunkStart:elementPathIndexAtChunkEnd) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - - RestartCoalesceChunk: - - tryLockBackoffContext.backoff() - - heldLocks = newHeldLocks() - - coalesceElementList = make([]*inode.CoalesceElement, 0, (elementPathIndexAtChunkEnd - elementPathIndexAtChunkStart)) - - for elementPathIndex = elementPathIndexAtChunkStart; elementPathIndex < elementPathIndexAtChunkEnd; elementPathIndex++ { - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - elementPaths[elementPathIndex], - heldLocks, - resolvePathFollowDirSymlinks| - resolvePathRequireExclusiveLockOnDirEntryInode| - resolvePathRequireExclusiveLockOnDirInode) - - if nil != err { - heldLocks.free() - return - } - - if retryRequired { - heldLocks.free() - goto RestartCoalesceChunk - } - - coalesceElementList = append(coalesceElementList, &inode.CoalesceElement{ - ContainingDirectoryInodeNumber: dirInodeNumber, - ElementInodeNumber: dirEntryInodeNumber, - ElementName: dirEntryBasename, - }) - } - - _, destFileInodeNumber, _, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - destPath, - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks| - resolvePathRequireExclusiveLockOnDirEntryInode) - - if nil != err { - heldLocks.free() - return - } - - if retryRequired { - heldLocks.free() - goto RestartCoalesceChunk - } - - ctime, mtime, numWrites, coalesceSize, err = vS.inodeVolumeHandle.Coalesce( - destFileInodeNumber, MiddlewareStream, metaData, coalesceElementList) - - heldLocks.free() - - if nil != err { - return - } - - elementPathIndexAtChunkStart = elementPathIndexAtChunkEnd - } - - // Regardless of err return, fill in other return values - - ino = uint64(destFileInodeNumber) - attrChangeTime = uint64(ctime.UnixNano()) - modificationTime = uint64(mtime.UnixNano()) - - return -} - -func (vS *volumeStruct) MiddlewareDelete(parentDir string, basename string) (err error) { - var ( - dirEntryBasename string - dirEntryInodeNumber inode.InodeNumber - dirInodeNumber inode.InodeNumber - doDestroy bool - heldLocks *heldLocksStruct - inodeType inode.InodeType - inodeVolumeHandle inode.VolumeHandle - linkCount uint64 - numDirEntries uint64 - retryRequired bool - toDestroyInodeNumber inode.InodeNumber - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - startTime := time.Now() - defer func() { - globals.MiddlewareDeleteUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.MiddlewareDeleteErrors.Add(1) - } - }() - - // Retry until done or failure (starting with ZERO backoff) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - // Perform backoff and update for each restart (starting with ZERO backoff of course) - - tryLockBackoffContext.backoff() - - // Construct fresh heldLocks for this restart - - heldLocks = newHeldLocks() - - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - parentDir+"/"+basename, - heldLocks, - resolvePathFollowDirSymlinks| - resolvePathRequireExclusiveLockOnDirEntryInode| - resolvePathRequireExclusiveLockOnDirInode) - - if nil != err { - heldLocks.free() - return - } - - if retryRequired { - heldLocks.free() - goto Restart - } - - // Check if Unlink() and Destroy() are doable - - inodeVolumeHandle = vS.inodeVolumeHandle - - inodeType, err = inodeVolumeHandle.GetType(dirEntryInodeNumber) - if nil != err { - heldLocks.free() - return - } - - if inode.DirType == inodeType { - numDirEntries, err = inodeVolumeHandle.NumDirEntries(dirEntryInodeNumber) - if nil != err { - heldLocks.free() - return - } - - if 2 != numDirEntries { - heldLocks.free() - err = blunder.NewError(blunder.NotEmptyError, "%s/%s not empty", parentDir, basename) - return - } - - doDestroy = true - } else { - linkCount, err = inodeVolumeHandle.GetLinkCount(dirEntryInodeNumber) - if nil != err { - heldLocks.free() - return - } - - doDestroy = (1 == linkCount) - } - - // Now perform the Unlink() and (potentially) Destroy() - - toDestroyInodeNumber, err = inodeVolumeHandle.Unlink(dirInodeNumber, dirEntryBasename, false) - if nil != err { - heldLocks.free() - return - } - - if doDestroy && (inode.InodeNumber(0) != toDestroyInodeNumber) { - err = inodeVolumeHandle.Destroy(toDestroyInodeNumber) - if nil != err { - logger.Errorf("fs.MiddlewareDelete() failed to Destroy dirEntryInodeNumber 0x%016X: %v", dirEntryInodeNumber, err) - } - } - - // Release heldLocks and exit with success (even if Destroy() failed earlier) - - heldLocks.free() - - err = nil - return -} - -func (vS *volumeStruct) middlewareReadDirHelper(path string, maxEntries uint64, prevBasename string) (pathDirInodeNumber inode.InodeNumber, dirEntrySlice []inode.DirEntry, moreEntries bool, err error) { - var ( - dirEntrySliceElement inode.DirEntry - heldLocks *heldLocksStruct - internalDirEntrySlice []inode.DirEntry - retryRequired bool - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - // Retry until done or failure (starting with ZERO backoff) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - // Perform backoff and update for each restart (starting with ZERO backoff of course) - - tryLockBackoffContext.backoff() - - // Construct fresh heldLocks for this restart - - heldLocks = newHeldLocks() - - _, pathDirInodeNumber, _, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - path, - heldLocks, - resolvePathFollowDirSymlinks) - - if nil != err { - heldLocks.free() - return - } - - if retryRequired { - heldLocks.free() - goto Restart - } - - // Now assemble response - - internalDirEntrySlice, moreEntries, err = vS.inodeVolumeHandle.ReadDir(pathDirInodeNumber, maxEntries, 0, prevBasename) - if nil != err { - heldLocks.free() - return - } - - // No need to hold any locks now... directory contents should be allowed to change while enumerating - heldLocks.free() - - dirEntrySlice = make([]inode.DirEntry, 0, len(internalDirEntrySlice)) - - for _, dirEntrySliceElement = range internalDirEntrySlice { - if ("." == dirEntrySliceElement.Basename) || (".." == dirEntrySliceElement.Basename) { - dirEntrySliceElement.Type = inode.DirType - } else { - dirEntrySliceElement.Type, err = vS.GetType(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirEntrySliceElement.InodeNumber) - if nil != err { - // It's ok to have an error here... it just means the directory we are iterating is changing - continue - } - } - dirEntrySlice = append(dirEntrySlice, dirEntrySliceElement) - } - - dirEntrySlice = dirEntrySlice[:len(dirEntrySlice)] - - err = nil - return -} - -func (vS *volumeStruct) MiddlewareGetAccount(maxEntries uint64, marker string, endmarker string) (accountEnts []AccountEntry, mtime uint64, ctime uint64, err error) { - var ( - dirEntrySlice []inode.DirEntry - dirEntrySliceElement inode.DirEntry - remainingMaxEntries uint64 - moreEntries bool - statResult Stat - ) - - statResult, err = vS.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber) - if nil != err { - return - } - mtime = statResult[StatMTime] - ctime = statResult[StatCTime] - - if 0 != maxEntries { - // Hard limit to number of DirInode Basenames to return - accountEnts = make([]AccountEntry, 0, maxEntries) - } - - remainingMaxEntries = maxEntries - - moreEntries = true - - for moreEntries { - _, dirEntrySlice, moreEntries, err = vS.middlewareReadDirHelper("/", remainingMaxEntries, marker) - if nil != err { - return - } - - if 0 == maxEntries { - // No limit to number of DirInode Basenames to return... so it must be <= len(dirEntrySlice) - accountEnts = make([]AccountEntry, 0, len(dirEntrySlice)) - // Note: moreEntries should be false so the "for moreEntries" loop should exit after 1st iteration - } - - for _, dirEntrySliceElement = range dirEntrySlice { - if ("" != endmarker) && (0 <= strings.Compare(dirEntrySliceElement.Basename, endmarker)) { - moreEntries = false - break - } - if ("." != dirEntrySliceElement.Basename) && (".." != dirEntrySliceElement.Basename) { - // So we've skipped "." & ".." - now also skip non-DirInodes - if inode.DirType == dirEntrySliceElement.Type { - statResult, err = vS.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirEntrySliceElement.InodeNumber) - if nil != err { - return - } - accountEnts = append(accountEnts, AccountEntry{ - Basename: dirEntrySliceElement.Basename, - ModificationTime: statResult[StatMTime], - AttrChangeTime: statResult[StatCTime], - }) - } - } - } - - if moreEntries && (0 != maxEntries) { - remainingMaxEntries = maxEntries - uint64(len(accountEnts)) - if 0 == remainingMaxEntries { - moreEntries = false - } - } - - if moreEntries { - // Adjust marker to fetch next dirEntrySlice - marker = dirEntrySlice[len(dirEntrySlice)-1].Basename - } - } - - accountEnts = accountEnts[:len(accountEnts)] - - return -} - -type dirEntrySliceStackElementStruct struct { - dirPath string - dirEntrySlice []inode.DirEntry - numConsumed int - moreEntries bool -} - -func (vS *volumeStruct) MiddlewareGetContainer(vContainerName string, maxEntries uint64, marker string, endmarker string, prefix string, delimiter string) (containerEnts []ContainerEntry, err error) { - var ( - containerEntry ContainerEntry - containerEntryBasename string // Misnamed... this is actually everything after ContainerName - containerEntryPath string - containerEntryPathSplit []string // Split on only the first '/' (to remove ContainerName from it) - doSingleDirectory bool - dirEntryInodeLock *dlm.RWLockStruct - dirEntryInodeNumber inode.InodeNumber - dirEntryInodeType inode.InodeType - dirEntryMetadata *inode.MetadataStruct - dirEntryPath string - dirEntrySlice []inode.DirEntry - dirEntrySliceElement inode.DirEntry - dirEntrySliceElementIndex int - dirEntrySliceElementToPrepend *inode.DirEntry - dirEntrySliceStack []*dirEntrySliceStackElementStruct - dirEntrySliceStackElement *dirEntrySliceStackElementStruct - dirEntrySliceToAppend []inode.DirEntry - dirInodeNumber inode.InodeNumber - dirPath string - dirPathSplit []string - dlmCallerID dlm.CallerID - endmarkerCanonicalized string - endmarkerPath []string - heldLocks *heldLocksStruct - initialDirEntryToMatch string // == "" if no initial path should be returned (i.e. in marker starting point case) - inodeVolumeHandle inode.VolumeHandle - markerCanonicalized string - markerPath []string - markerPathDirInodeIndex int - moreEntries bool - pathIndex int - prefixCanonicalized string - prefixPath []string - prefixPathDirInodeIndex int - prevReturned string - remainingMaxEntries uint64 - retryRequired bool - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - // Validate marker, endmarker, and prefix - - if "" == marker { - markerPath = []string{} - markerPathDirInodeIndex = -1 // Must be special cased below to ensure we don't look in markerPath - markerCanonicalized = "" // Actually never accessed - } else { - markerPath, markerPathDirInodeIndex, err = vS.canonicalizePathAndLocateLeafDirInode(vContainerName + "/" + marker) - if nil != err { - err = blunder.AddError(err, blunder.InvalidArgError) - return - } - - markerCanonicalized = strings.Join(markerPath, "/") - if strings.HasSuffix(marker, "/") { - markerCanonicalized += "/" - } - - if vContainerName+"/"+marker != markerCanonicalized { - err = blunder.NewError(blunder.InvalidArgError, "MiddlewareGetContainer() only supports a canonicalized marker") - return - } - } - - if "" == endmarker { - endmarkerPath = []string{} - endmarkerCanonicalized = "" // Actually never accessed - } else { - endmarkerPath, _, err = vS.canonicalizePathAndLocateLeafDirInode(vContainerName + "/" + endmarker) - if nil != err { - err = blunder.AddError(err, blunder.InvalidArgError) - return - } - - endmarkerCanonicalized = strings.Join(endmarkerPath, "/") - if strings.HasSuffix(endmarker, "/") { - endmarkerCanonicalized += "/" - } - - if vContainerName+"/"+endmarker != endmarkerCanonicalized { - err = blunder.NewError(blunder.InvalidArgError, "MiddlewareGetContainer() only supports a canonicalized endmarker") - return - } - } - - prefixPath, prefixPathDirInodeIndex, err = vS.canonicalizePathAndLocateLeafDirInode(vContainerName + "/" + prefix) - if nil != err { - err = blunder.AddError(err, blunder.InvalidArgError) - return - } - if prefixPathDirInodeIndex < 0 { - err = blunder.NewError(blunder.NotFoundError, "MiddlewareGetContainer() only supports querying an existing Container") - return - } - - prefixCanonicalized = strings.Join(prefixPath, "/") - if strings.HasSuffix(prefix, "/") { - prefixCanonicalized += "/" - } - - if (prefix != "") && (vContainerName+"/"+prefix != prefixCanonicalized) { - err = blunder.NewError(blunder.InvalidArgError, "MiddlewareGetContainer() only supports a canonicalized prefix") - return - } - - // Validate delimiter - - switch delimiter { - case "": - doSingleDirectory = false - case "/": - doSingleDirectory = true - default: - err = blunder.NewError(blunder.InvalidArgError, "MiddlewareGetContainer() only supports a delimiter of \"/\"") - return - } - - // Determine what DirInode from which to begin our enumeration - - pathIndex = 0 - - for { - if (pathIndex > markerPathDirInodeIndex) && (pathIndex > prefixPathDirInodeIndex) { - // Special (though probably typical) case where marker lands in prefix-indicated directory - - dirPath = strings.Join(prefixPath[:prefixPathDirInodeIndex+1], "/") - - if (1 == len(prefixPath)) || strings.HasSuffix(prefix, "/") { - if (markerPathDirInodeIndex + 1) == len(markerPath) { - prevReturned = "" - } else { - prevReturned = markerPath[markerPathDirInodeIndex+1] - } - initialDirEntryToMatch = "" - } else { - // Handle four remaining cases: - // marker & prefix both specified directories - // marker specified a directory, prefix did not - // prefix specified a directory, marker did not - // neither marker nor prefix specified a directory - - if (markerPathDirInodeIndex + 1) == len(markerPath) { - if (prefixPathDirInodeIndex + 1) == len(prefixPath) { - // Case where marker & prefix both specified directories - - prevReturned = "" - } else { - // Case where marker specified a directory, prefix did not - - prevReturned = prefixPath[prefixPathDirInodeIndex+1] - } - initialDirEntryToMatch = prevReturned - } else { // (markerPathDirInodeIndex + 1) != len(markerPath) - if (prefixPathDirInodeIndex + 1) == len(prefixPath) { - // Case where prefix specified a directory, marker did not - - prevReturned = markerPath[markerPathDirInodeIndex+1] - initialDirEntryToMatch = "" - } else { - // Case where neither marker nor prefix specified a directory - - if strings.Compare(prefixPath[prefixPathDirInodeIndex+1], markerPath[markerPathDirInodeIndex+1]) <= 0 { - prevReturned = markerPath[markerPathDirInodeIndex+1] - initialDirEntryToMatch = "" - } else { - prevReturned = prefixPath[prefixPathDirInodeIndex+1] - initialDirEntryToMatch = prevReturned - } - } - } - } - break - } - - if pathIndex > markerPathDirInodeIndex { - // Handle case where prefix is more constraining than marker - - if prefixPathDirInodeIndex == (len(prefixPath) - 1) { - if (1 == len(prefixPath)) || strings.HasSuffix(prefix, "/") { - dirPath = strings.Join(prefixPath[:prefixPathDirInodeIndex+1], "/") - prevReturned = "" - } else { - dirPath = strings.Join(prefixPath[:prefixPathDirInodeIndex], "/") - prevReturned = prefixPath[len(prefixPath)-1] - } - } else { - dirPath = strings.Join(prefixPath[:prefixPathDirInodeIndex+1], "/") - prevReturned = prefixPath[len(prefixPath)-1] - } - initialDirEntryToMatch = prevReturned - break - } - - if pathIndex > prefixPathDirInodeIndex { - // Handle case where marker is more constraining than prefix - - dirPath = strings.Join(markerPath[:markerPathDirInodeIndex+1], "/") - if markerPathDirInodeIndex == (len(markerPath) - 1) { - prevReturned = "" - } else { - prevReturned = markerPath[len(markerPath)-1] - } - initialDirEntryToMatch = "" - break - } - - switch strings.Compare(prefixPath[pathIndex], markerPath[pathIndex]) { - case -1: - dirPath = strings.Join(markerPath[:markerPathDirInodeIndex+1], "/") - if markerPathDirInodeIndex == (len(markerPath) - 1) { - prevReturned = "" - } else { - prevReturned = markerPath[len(markerPath)-1] - } - initialDirEntryToMatch = "" - break - case 0: - pathIndex++ - case 1: - if prefixPathDirInodeIndex == (len(prefixPath) - 1) { - if (1 == len(prefixPath)) || strings.HasSuffix(prefix, "/") { - dirPath = strings.Join(prefixPath[:prefixPathDirInodeIndex+1], "/") - prevReturned = "" - } else { - dirPath = strings.Join(prefixPath[:prefixPathDirInodeIndex], "/") - prevReturned = prefixPath[len(prefixPath)-1] - } - } else { - dirPath = strings.Join(prefixPath[:prefixPathDirInodeIndex+1], "/") - prevReturned = prefixPath[len(prefixPath)-1] - } - initialDirEntryToMatch = prevReturned - break - } - } - - // Setup shortcuts/contants - - dlmCallerID = dlm.GenerateCallerID() - inodeVolumeHandle = vS.inodeVolumeHandle - - // Compute initial response - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - tryLockBackoffContext.backoff() - - heldLocks = newHeldLocks() - - _, dirInodeNumber, _, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - dirPath, - heldLocks, - resolvePathDirEntryInodeMustBeDirectory) - if nil != err { - heldLocks.free() - return - } - if retryRequired { - heldLocks.free() - goto Restart - } - - containerEnts = make([]ContainerEntry, 0, maxEntries) - - if 0 == maxEntries { - heldLocks.free() - err = nil - return - } - - if "" == initialDirEntryToMatch { - dirEntrySliceElementToPrepend = nil - } else { - if "" == dirPath { - dirEntryPath = initialDirEntryToMatch - } else { - dirEntryPath = dirPath + "/" + initialDirEntryToMatch - } - if ("" != endmarker) && (strings.Compare(dirEntryPath, endmarkerCanonicalized) >= 0) { - heldLocks.free() - err = nil - return - } - dirEntryInodeNumber, err = inodeVolumeHandle.Lookup(dirInodeNumber, initialDirEntryToMatch) - if nil == err { - retryRequired = heldLocks.attemptSharedLock(inodeVolumeHandle, dlmCallerID, dirEntryInodeNumber) - if retryRequired { - heldLocks.free() - goto Restart - } - dirEntryInodeType, err = inodeVolumeHandle.GetType(dirEntryInodeNumber) - if nil == err { - dirEntrySliceElementToPrepend = &inode.DirEntry{ - InodeNumber: dirEntryInodeNumber, - Basename: initialDirEntryToMatch, - Type: dirEntryInodeType, - } - } else { - dirEntrySliceElementToPrepend = nil - } - heldLocks.unlock(dirEntryInodeNumber) - } else { - dirEntrySliceElementToPrepend = nil - } - } - - heldLocks.free() - - if 0 == maxEntries { - remainingMaxEntries = 0 - } else { - if nil == dirEntrySliceElementToPrepend { - remainingMaxEntries = maxEntries - } else { - remainingMaxEntries = maxEntries - 1 - } - } - - // At this point: - // no heldLocks - // containerEnts has been declared - // doSingleDirectory is set based on supplied delimiter - // if {marker,endmarker,prefix} asked to include an exact matched path that existed, it's in dirEntrySliceElementToPrepend - // prefixCanonicalized & endmarkerCanonicalized are set to terminate the ensuing treewalk - // remainingMaxEntries indicates how many more DirEntry's will fit in containerEnts (if capped) - // dirPath is pointing to the initial DirInode to read - // prevReturned indicates from where in the DirInode to start reading - - // Perform initial ReadDir and place in dirEntrySliceStack - - if nil == dirEntrySliceElementToPrepend { - _, dirEntrySlice, moreEntries, err = vS.middlewareReadDirHelper(dirPath, remainingMaxEntries, prevReturned) - if nil != err { - return - } - } else { - if 0 == remainingMaxEntries { - dirEntrySlice = []inode.DirEntry{*dirEntrySliceElementToPrepend} - moreEntries = false - } else { - _, dirEntrySliceToAppend, moreEntries, err = vS.middlewareReadDirHelper(dirPath, remainingMaxEntries, prevReturned) - if nil == err { - dirEntrySlice = make([]inode.DirEntry, 1, 1+len(dirEntrySliceToAppend)) - dirEntrySlice[0] = *dirEntrySliceElementToPrepend - dirEntrySlice = append(dirEntrySlice, dirEntrySliceToAppend...) - } else { - return - } - } - } - - dirEntrySliceStackElement = &dirEntrySliceStackElementStruct{ - dirPath: dirPath, - dirEntrySlice: dirEntrySlice, - numConsumed: 0, - moreEntries: moreEntries, - } - - dirEntrySliceStack = []*dirEntrySliceStackElementStruct{dirEntrySliceStackElement} - - containerEnts = make([]ContainerEntry, 0, len(dirEntrySlice)) - - // Now append appropriate ContainerEntry's until exit criteria is reached - - for uint64(len(containerEnts)) < maxEntries { - dirEntrySliceStackElement = dirEntrySliceStack[len(dirEntrySliceStack)-1] - - if dirEntrySliceStackElement.numConsumed == len(dirEntrySliceStackElement.dirEntrySlice) { - if dirEntrySliceStackElement.moreEntries { - dirPath = dirEntrySliceStackElement.dirPath - dirEntrySlice = dirEntrySliceStackElement.dirEntrySlice - dirEntrySliceElementIndex = len(dirEntrySlice) - 1 - dirEntrySliceElement = dirEntrySlice[dirEntrySliceElementIndex] - prevReturned = dirEntrySliceElement.Basename - - _, dirEntrySlice, moreEntries, err = vS.middlewareReadDirHelper(dirPath, remainingMaxEntries, prevReturned) - if (nil != err) || (0 == len(dirEntrySlice)) { - // Even though we thought there were moreEntries, there now are not for some reason - - if doSingleDirectory { - // Regardless of remaining contents of dirEntrySliceStack, we must be done - - err = nil - return - } - - // Navigate to parent directory - - dirEntrySliceStack = dirEntrySliceStack[:len(dirEntrySliceStack)-1] - continue - } - - // Restart this loop on current dirEntrySliceStackElement with new middlewareReadDirHelper() results - - dirEntrySliceStackElement.dirEntrySlice = dirEntrySlice - dirEntrySliceStackElement.numConsumed = 0 - dirEntrySliceStackElement.moreEntries = moreEntries - - continue - } else { - // We've reached the end of this DirInode - - if doSingleDirectory { - // Regardless of remaining contents of dirEntrySliceStack, we must be done - - err = nil - return - } - - // Navigate to parent directory (staying within this Container) - - if 1 == len(dirEntrySliceStack) { - // We are at the starting directory - - dirPathSplit = strings.Split(dirEntrySliceStackElement.dirPath, "/") - - if 1 == len(dirPathSplit) { - // We just finished Container-level directory, so we are done - - err = nil - return - } - - // Modify dirEntrySliceStackElement to point to parent directory as if we'd just processed the dirEntry of this directory - - dirPath = strings.Join(dirPathSplit[:len(dirPathSplit)-1], "/") - - if 0 == maxEntries { - remainingMaxEntries = 0 - } else { - remainingMaxEntries = maxEntries - uint64(len(containerEnts)) - } - - prevReturned = dirPathSplit[len(dirPathSplit)-1] - - _, dirEntrySlice, moreEntries, err = vS.middlewareReadDirHelper(dirPath, remainingMaxEntries, prevReturned) - if nil != err { - return - } - - dirEntrySliceStackElement.dirPath = dirPath - dirEntrySliceStackElement.dirEntrySlice = dirEntrySlice - dirEntrySliceStackElement.numConsumed = 0 - dirEntrySliceStackElement.moreEntries = moreEntries - } else { - // Parent directory already in dirEntrySliceStack... so just pop current ...Element - - dirEntrySliceStack = dirEntrySliceStack[:len(dirEntrySliceStack)-1] - } - - continue - } - } - - // Consume next dirEntrySliceElement - // ...skipping "." and ".." - // ...skipping if / <= marker - // ...recursing when encountering DirInode's if !doSingleDirectory - // ...terminating early if either: - // len(*containerEnts) reaches maxEntries - // / >= endmarker - // / does not start with prefix - - dirEntrySlice = dirEntrySliceStackElement.dirEntrySlice - dirEntrySliceElementIndex = dirEntrySliceStackElement.numConsumed - dirEntrySliceElement = dirEntrySlice[dirEntrySliceElementIndex] - - dirEntrySliceStackElement.numConsumed++ - - if ("." == dirEntrySliceElement.Basename) || (".." == dirEntrySliceElement.Basename) { - continue - } - - containerEntryPath = dirEntrySliceStackElement.dirPath + "/" + dirEntrySliceElement.Basename - - if ("" != marker) && (strings.Compare(containerEntryPath, markerCanonicalized) <= 0) { - err = nil - return - } - if ("" != endmarker) && (strings.Compare(containerEntryPath, endmarkerCanonicalized) >= 0) { - err = nil - return - } - if ("" != prefix) && !strings.HasPrefix(containerEntryPath, prefixCanonicalized) { - err = nil - return - } - - // Ok... so we actually want to append this entry to containerEnts - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - - Retry: - - tryLockBackoffContext.backoff() - - dirEntryInodeLock, err = inodeVolumeHandle.AttemptReadLock(dirEntrySliceElement.InodeNumber, dlmCallerID) - if nil != err { - goto Retry - } - - dirEntryMetadata, err = inodeVolumeHandle.GetMetadata(dirEntrySliceElement.InodeNumber) - if nil != err { - // Ok... so it must have disappeared... just skip it - - err = dirEntryInodeLock.Unlock() - if nil != err { - logger.Fatalf("Failure unlocking a held LockID %s: %v", dirEntryInodeLock.LockID, err) - } - - continue - } - - containerEntryPathSplit = strings.SplitN(containerEntryPath, "/", 2) - containerEntryBasename = containerEntryPathSplit[1] - - containerEntry = ContainerEntry{ - Basename: containerEntryBasename, - FileSize: dirEntryMetadata.Size, - ModificationTime: uint64(dirEntryMetadata.ModificationTime.UnixNano()), - AttrChangeTime: uint64(dirEntryMetadata.AttrChangeTime.UnixNano()), - IsDir: (dirEntrySliceElement.Type == inode.DirType), - NumWrites: dirEntryMetadata.NumWrites, - InodeNumber: uint64(dirEntrySliceElement.InodeNumber), - } - - containerEntry.Metadata, err = inodeVolumeHandle.GetStream(dirEntrySliceElement.InodeNumber, MiddlewareStream) - if nil != err { - if blunder.Is(err, blunder.StreamNotFound) { - // No MiddlewareStream... just make it appear empty - - containerEntry.Metadata = []byte{} - err = nil - } else { - // Ok... so it must have disappeared... just skip it - - err = dirEntryInodeLock.Unlock() - if nil != err { - logger.Fatalf("Failure unlocking a held LockID %s: %v", dirEntryInodeLock.LockID, err) - } - - continue - } - } - - // We can finally Unlock() this dirEntryInodeLock - - err = dirEntryInodeLock.Unlock() - if nil != err { - logger.Fatalf("Failure unlocking a held LockID %s: %v", dirEntryInodeLock.LockID, err) - } - - // If we reach here, we get to append this containerEntry to containerEnts - - containerEnts = append(containerEnts, containerEntry) - - // We must now descend into dirEntryInode descend into it if it's a DirInode and !doSingleDirectory - - if !doSingleDirectory && (dirEntrySliceElement.Type == inode.DirType) { - dirPath = dirEntrySliceStackElement.dirPath + "/" + dirEntrySliceElement.Basename - - if 0 == maxEntries { - remainingMaxEntries = 0 - } else { - remainingMaxEntries = maxEntries - uint64(len(containerEnts)) - } - - prevReturned = "" - - _, dirEntrySlice, moreEntries, err = vS.middlewareReadDirHelper(dirPath, remainingMaxEntries, prevReturned) - if nil != err { - return - } - - dirEntrySliceStackElement = &dirEntrySliceStackElementStruct{ - dirPath: dirPath, - dirEntrySlice: dirEntrySlice, - numConsumed: 0, - moreEntries: moreEntries, - } - - dirEntrySliceStack = append(dirEntrySliceStack, dirEntrySliceStackElement) - } - } - - // We will only reach here if we exhausted maxEntries before exhausing the tree/list of containerEntry's to append - - err = nil - return -} - -func (vS *volumeStruct) MiddlewareGetObject(containerObjectPath string, - readRangeIn []ReadRangeIn, readRangeOut *[]inode.ReadPlanStep) ( - response HeadResponse, err error) { - - var ( - dirEntryInodeNumber inode.InodeNumber - fileOffset uint64 - heldLocks *heldLocksStruct - inodeVolumeHandle inode.VolumeHandle - readPlan []inode.ReadPlanStep - readRangeInIndex int - retryRequired bool - stat Stat - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - startTime := time.Now() - defer func() { - var totalReadBytes uint64 - for _, step := range *readRangeOut { - totalReadBytes += step.Length - } - - globals.MiddlewareGetObjectUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.MiddlewareGetObjectBytes.Add(totalReadBytes) - if err != nil { - globals.MiddlewareGetObjectErrors.Add(1) - } - }() - - // Retry until done or failure (starting with ZERO backoff) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - // Perform backoff and update for each restart (starting with ZERO backoff of course) - - tryLockBackoffContext.backoff() - - // Construct fresh heldLocks for this restart - - heldLocks = newHeldLocks() - - _, dirEntryInodeNumber, _, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - containerObjectPath, - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks) - - if nil != err { - heldLocks.free() - return - } - - if retryRequired { - heldLocks.free() - goto Restart - } - - // Now assemble response - - stat, err = vS.getstatHelperWhileLocked(dirEntryInodeNumber) - if nil != err { - heldLocks.free() - return - } - - response.FileSize = stat[StatSize] - response.ModificationTime = stat[StatMTime] - response.AttrChangeTime = stat[StatCTime] - response.IsDir = (stat[StatFType] == uint64(inode.DirType)) - response.InodeNumber = dirEntryInodeNumber - response.NumWrites = stat[StatNumWrites] - - // Swift thinks all directories have a size of 0 (and symlinks as well) - if stat[StatFType] != uint64(inode.FileType) { - response.FileSize = 0 - } - - response.Metadata, err = vS.inodeVolumeHandle.GetStream(dirEntryInodeNumber, MiddlewareStream) - if nil != err { - if blunder.Is(err, blunder.StreamNotFound) { - response.Metadata = []byte{} - err = nil - } else { - heldLocks.free() - return - } - } - - // The only thing left is to construct a read plan and only regular - // files have read plans. If this is not a regular file then we're - // done. - if stat[StatFType] != uint64(inode.FileType) { - heldLocks.free() - return - } - - inodeVolumeHandle = vS.inodeVolumeHandle - if len(readRangeIn) == 0 { - // Get ReadPlan for entire file - - fileOffset = 0 - - readPlan, err = inodeVolumeHandle.GetReadPlan(dirEntryInodeNumber, &fileOffset, &response.FileSize) - if nil != err { - heldLocks.free() - return - } - - _ = appendReadPlanEntries(readPlan, readRangeOut) - } else { // len(readRangeIn) > 0 - // Append each computed range - - for readRangeInIndex = range readRangeIn { - readPlan, err = inodeVolumeHandle.GetReadPlan(dirEntryInodeNumber, readRangeIn[readRangeInIndex].Offset, readRangeIn[readRangeInIndex].Len) - if nil != err { - heldLocks.free() - return - } - - _ = appendReadPlanEntries(readPlan, readRangeOut) - } - } - - heldLocks.free() - - err = nil - return -} - -func (vS *volumeStruct) MiddlewareHeadResponse(entityPath string) (response HeadResponse, err error) { - var ( - dirEntryInodeNumber inode.InodeNumber - heldLocks *heldLocksStruct - retryRequired bool - stat Stat - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - startTime := time.Now() - defer func() { - globals.MiddlewareHeadResponseUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.MiddlewareHeadResponseErrors.Add(1) - } - }() - - // Retry until done or failure (starting with ZERO backoff) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - // Perform backoff and update for each restart (starting with ZERO backoff of course) - - tryLockBackoffContext.backoff() - - // Construct fresh heldLocks for this restart - - heldLocks = newHeldLocks() - - _, dirEntryInodeNumber, _, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - entityPath, - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks) - - if nil != err { - heldLocks.free() - return - } - - if retryRequired { - heldLocks.free() - goto Restart - } - - // Now assemble response - - stat, err = vS.getstatHelperWhileLocked(dirEntryInodeNumber) - if nil != err { - heldLocks.free() - return - } - - // since resolvePathFollowDirEntrySymlinks is set on the call to - // resolvePath(), above, we'll never see a symlink returned - response.ModificationTime = stat[StatMTime] - response.AttrChangeTime = stat[StatCTime] - response.FileSize = stat[StatSize] - response.IsDir = (stat[StatFType] == uint64(inode.DirType)) - response.InodeNumber = dirEntryInodeNumber - response.NumWrites = stat[StatNumWrites] - - // Swift thinks all directories have a size of 0 (and symlinks as well) - if stat[StatFType] != uint64(inode.FileType) { - response.FileSize = 0 - } - - response.Metadata, err = vS.inodeVolumeHandle.GetStream(dirEntryInodeNumber, MiddlewareStream) - if nil != err { - heldLocks.free() - response.Metadata = []byte{} - // If someone makes a directory or file via SMB/FUSE and then - // HEADs it via HTTP, we'll see this error. We treat it as - // though there is no metadata. The middleware is equipped to - // handle this case. - if blunder.Is(err, blunder.StreamNotFound) { - err = nil - } - return - } - - heldLocks.free() - return -} - -func (vS *volumeStruct) MiddlewarePost(parentDir string, baseName string, newMetaData []byte, oldMetaData []byte) (err error) { - var ( - dirEntryInodeNumber inode.InodeNumber - existingStreamData []byte - heldLocks *heldLocksStruct - retryRequired bool - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - startTime := time.Now() - defer func() { - globals.MiddlewarePostUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.MiddlewarePostBytes.Add(uint64(len(newMetaData))) - if err != nil { - globals.MiddlewarePostErrors.Add(1) - } - }() - - // Retry until done or failure (starting with ZERO backoff) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - // Perform backoff and update for each restart (starting with ZERO backoff of course) - - tryLockBackoffContext.backoff() - - // Construct fresh heldLocks for this restart - - heldLocks = newHeldLocks() - - _, dirEntryInodeNumber, _, _, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - parentDir+"/"+baseName, - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks| - resolvePathCreateMissingPathElements| - resolvePathRequireExclusiveLockOnDirEntryInode) - - if nil != err { - heldLocks.free() - return - } - - if retryRequired { - heldLocks.free() - goto Restart - } - - // Now apply MiddlewareStream update - - // Compare oldMetaData to existing existingStreamData to make sure that the HTTP metadata has not changed. - // If it has changed, then return an error since middleware has to handle it. - - existingStreamData, err = vS.inodeVolumeHandle.GetStream(dirEntryInodeNumber, MiddlewareStream) - if nil != err { - if blunder.Is(err, blunder.StreamNotFound) { - err = nil - existingStreamData = make([]byte, 0) - } else { - heldLocks.free() - return - } - } - - // Verify that the oldMetaData is the same as the one we think we are changing. - - if !bytes.Equal(existingStreamData, oldMetaData) { - heldLocks.free() - err = blunder.NewError(blunder.TryAgainError, "MiddlewarePost(): MetaData different - existingStreamData: %v OldMetaData: %v", existingStreamData, oldMetaData) - return - } - - // Change looks okay so make it. - - err = vS.inodeVolumeHandle.PutStream(dirEntryInodeNumber, MiddlewareStream, newMetaData) - if nil != err { - heldLocks.free() - return - } - - // PutStream() implicitly flushed... so, if it was a FileInode, we don't need to track it anymore - - vS.untrackInFlightFileInodeData(dirEntryInodeNumber, false) - - heldLocks.free() - return -} - -func (vS *volumeStruct) MiddlewarePutComplete(vContainerName string, vObjectPath string, pObjectPaths []string, pObjectLengths []uint64, pObjectMetadata []byte) (mtime uint64, ctime uint64, fileInodeNumber inode.InodeNumber, numWrites uint64, err error) { - var ( - containerName string - dirInodeNumber inode.InodeNumber - dirEntryInodeNumber inode.InodeNumber - dirEntryBasename string - dirEntryInodeType inode.InodeType - fileOffset uint64 - heldLocks *heldLocksStruct - inodeVolumeHandle inode.VolumeHandle = vS.inodeVolumeHandle - inodeWroteTime time.Time - numPObjects int - objectName string - pObjectIndex int - retryRequired bool - stat Stat - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - startTime := time.Now() - defer func() { - globals.MiddlewarePutCompleteUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.MiddlewarePutCompleteErrors.Add(1) - } - }() - - // Validate (pObjectPaths,pObjectLengths) args - - numPObjects = len(pObjectPaths) - - if numPObjects != len(pObjectLengths) { - blunder.NewError(blunder.InvalidArgError, "MiddlewarePutComplete() expects len(pObjectPaths) == len(pObjectLengths)") - return - } - - // Retry until done or failure (starting with ZERO backoff) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - // Perform backoff and update for each restart (starting with ZERO backoff of course) - - tryLockBackoffContext.backoff() - - // Construct fresh heldLocks for this restart - - heldLocks = newHeldLocks() - - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, dirEntryInodeType, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - vContainerName+"/"+vObjectPath, - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks| - resolvePathCreateMissingPathElements| - resolvePathRequireExclusiveLockOnDirInode| - resolvePathRequireExclusiveLockOnDirEntryInode) - if nil != err { - heldLocks.free() - return - } - if retryRequired { - heldLocks.free() - goto Restart - } - - // The semantics of PUT mean that the existing object is discarded; with - // a file we can just overwrite it, but symlinks or directories must be - // removed (if possible). - if dirEntryInodeType != inode.FileType { - - if dirEntryInodeType == inode.DirType { - - // try to unlink the directory (rmdir flushes the inodes) - err = vS.rmdirActual(dirInodeNumber, dirEntryBasename, dirEntryInodeNumber) - if err != nil { - // the directory was probably not empty - heldLocks.free() - return - - } - - } else { - // unlink the symlink (unlink flushes the inodes) - err = vS.unlinkActual(dirInodeNumber, dirEntryBasename, dirEntryInodeNumber) - if err != nil { - - // ReadOnlyError is my best guess for the failure - err = blunder.NewError(blunder.ReadOnlyError, - "MiddlewareMkdir(): vol '%s' failed to unlink '%s': %v", - vS.volumeName, vContainerName+"/"+vObjectPath, err) - heldLocks.free() - return - } - } - - // let resolvePath() create the file - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, dirEntryInodeType, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - vContainerName+"/"+vObjectPath, - heldLocks, - resolvePathFollowDirSymlinks| - resolvePathCreateMissingPathElements| - resolvePathDirEntryInodeMustBeFile| - resolvePathRequireExclusiveLockOnDirInode| - resolvePathRequireExclusiveLockOnDirEntryInode) - if nil != err { - heldLocks.free() - return - } - if retryRequired { - heldLocks.free() - goto Restart - } - } - - // Apply (pObjectPaths,pObjectLengths) to (erased) FileInode - - inodeWroteTime = time.Now() - - fileOffset = 0 - - for pObjectIndex = 0; pObjectIndex < numPObjects; pObjectIndex++ { - _, containerName, objectName, err = utils.PathToAcctContObj(pObjectPaths[pObjectIndex]) - if nil != err { - heldLocks.free() - logger.DebugfIDWithError(internalDebug, err, "MiddlewarePutComplete(): failed utils.PathToAcctContObj(\"%s\") for dirEntryInodeNumber 0x%016X", pObjectPaths[pObjectIndex], dirEntryInodeNumber) - return - } - - err = inodeVolumeHandle.Wrote( - dirEntryInodeNumber, - containerName, - objectName, - []uint64{fileOffset}, - []uint64{0}, - []uint64{pObjectLengths[pObjectIndex]}, - inodeWroteTime, - pObjectIndex > 0) // Initial pObjectIndex == 0 case will implicitly SetSize(,0) - if nil != err { - heldLocks.free() - logger.DebugfIDWithError(internalDebug, err, "MiddlewarePutComplete(): failed inode.Wrote() for dirEntryInodeNumber 0x%016X", dirEntryInodeNumber) - return - } - - fileOffset += pObjectLengths[pObjectIndex] - } - - // Apply pObjectMetadata to FileInode (this will flush it as well) - - err = inodeVolumeHandle.PutStream(dirEntryInodeNumber, MiddlewareStream, pObjectMetadata) - if err != nil { - heldLocks.free() - logger.DebugfIDWithError(internalDebug, err, "MiddlewarePutComplete(): failed PutStream() for dirEntryInodeNumber 0x%016X (pObjectMetadata: %v)", dirEntryInodeNumber, pObjectMetadata) - return - } - - stat, err = vS.getstatHelperWhileLocked(dirEntryInodeNumber) - if nil != err { - heldLocks.free() - return - } - - mtime = stat[StatMTime] - ctime = stat[StatCTime] - fileInodeNumber = dirEntryInodeNumber - numWrites = stat[StatNumWrites] - - heldLocks.free() - return -} - -func (vS *volumeStruct) MiddlewareMkdir(vContainerName string, vObjectPath string, metadata []byte) (mtime uint64, ctime uint64, inodeNumber inode.InodeNumber, numWrites uint64, err error) { - var ( - dirInodeNumber inode.InodeNumber - dirEntryInodeNumber inode.InodeNumber - dirEntryBasename string - dirEntryInodeType inode.InodeType - heldLocks *heldLocksStruct - retryRequired bool - stat Stat - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - startTime := time.Now() - defer func() { - globals.MiddlewareMkdirUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.MiddlewareMkdirErrors.Add(1) - } - }() - - // Retry until done or failure (starting with ZERO backoff) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - // Perform backoff and update for each restart (starting with ZERO backoff of course) - - tryLockBackoffContext.backoff() - - // Construct fresh heldLocks for this restart - - heldLocks = newHeldLocks() - - // Resolve the object, locking it and its parent directory exclusive - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, dirEntryInodeType, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - vContainerName+"/"+vObjectPath, - heldLocks, - resolvePathFollowDirSymlinks| - resolvePathCreateMissingPathElements| - resolvePathRequireExclusiveLockOnDirInode| - resolvePathRequireExclusiveLockOnDirEntryInode) - if nil != err { - heldLocks.free() - return - } - if retryRequired { - heldLocks.free() - goto Restart - } - - // The semantics of PUT for a directory object require that an existing - // file or symlink be discarded and be replaced with a directory (an - // existing directory is fine; it just has its headers overwritten). - if dirEntryInodeType != inode.DirType { - - // unlink the file or symlink (unlink flushes the inodes) - err = vS.unlinkActual(dirInodeNumber, dirEntryBasename, dirEntryInodeNumber) - if err != nil { - - // ReadOnlyError is my best guess for the failure - err = blunder.NewError(blunder.ReadOnlyError, - "MiddlewareMkdir(): vol '%s' failed to unlink '%s': %v", - vS.volumeName, vContainerName+"/"+vObjectPath, err) - heldLocks.free() - return - } - - // let resolvePath() make the directory - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, dirEntryInodeType, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - vContainerName+"/"+vObjectPath, - heldLocks, - resolvePathFollowDirSymlinks| - resolvePathCreateMissingPathElements| - resolvePathDirEntryInodeMustBeDirectory| - resolvePathRequireExclusiveLockOnDirInode| - resolvePathRequireExclusiveLockOnDirEntryInode) - if nil != err { - heldLocks.free() - return - } - if retryRequired { - heldLocks.free() - goto Restart - } - } - - err = vS.inodeVolumeHandle.PutStream(dirEntryInodeNumber, MiddlewareStream, metadata) - if err != nil { - heldLocks.free() - logger.DebugfIDWithError(internalDebug, err, "MiddlewareHeadResponse(): failed PutStream() for for dirEntryInodeNumber 0x%016X (pObjectMetadata: %v)", dirEntryInodeNumber, metadata) - return - } - - stat, err = vS.getstatHelperWhileLocked(dirEntryInodeNumber) - if nil != err { - heldLocks.free() - return - } - - mtime = stat[StatMTime] - ctime = stat[StatCTime] - inodeNumber = dirEntryInodeNumber - numWrites = stat[StatNumWrites] - - heldLocks.free() - return -} - -func (vS *volumeStruct) MiddlewarePutContainer(containerName string, oldMetadata []byte, newMetadata []byte) (err error) { - var ( - containerInodeLock *dlm.RWLockStruct - containerInodeNumber inode.InodeNumber - existingMetadata []byte - newDirInodeLock *dlm.RWLockStruct - newDirInodeNumber inode.InodeNumber - ) - - startTime := time.Now() - defer func() { - globals.MiddlewarePutContainerUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.MiddlewarePutContainerBytes.Add(uint64(len(newMetadata))) - if err != nil { - globals.MiddlewarePutContainerErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - // Yes, it's a heavy lock to hold on the root inode. However, we - // might need to add a new directory entry there, so there's not - // much else we can do. - rootInodeLock, err := vS.inodeVolumeHandle.GetWriteLock(inode.RootDirInodeNumber, nil) - if nil != err { - return - } - defer rootInodeLock.Unlock() - - containerInodeNumber, err = vS.inodeVolumeHandle.Lookup(inode.RootDirInodeNumber, containerName) - if err != nil && blunder.IsNot(err, blunder.NotFoundError) { - return - } else if err != nil { - // No such container, so we create it - err = validateBaseName(containerName) - if err != nil { - return - } - - newDirInodeNumber, err = vS.inodeVolumeHandle.CreateDir(inode.PosixModePerm, 0, 0) - if err != nil { - logger.ErrorWithError(err) - return - } - - newDirInodeLock, err = vS.inodeVolumeHandle.GetWriteLock(newDirInodeNumber, nil) - defer newDirInodeLock.Unlock() - - err = vS.inodeVolumeHandle.PutStream(newDirInodeNumber, MiddlewareStream, newMetadata) - if err != nil { - logger.ErrorWithError(err) - return - } - - err = vS.inodeVolumeHandle.Link(inode.RootDirInodeNumber, containerName, newDirInodeNumber, false) - - return - } - - containerInodeLock, err = vS.inodeVolumeHandle.GetWriteLock(containerInodeNumber, nil) - if err != nil { - return - } - defer containerInodeLock.Unlock() - - // Existing container: just update the metadata - existingMetadata, err = vS.inodeVolumeHandle.GetStream(containerInodeNumber, MiddlewareStream) - - // GetStream() will return an error if there is no "middleware" stream - if err != nil && blunder.IsNot(err, blunder.StreamNotFound) { - return - } else if err != nil { - existingMetadata = []byte{} - } - - // Only change it if the caller sent the current value - if !bytes.Equal(existingMetadata, oldMetadata) { - err = blunder.NewError(blunder.TryAgainError, "Metadata differs - actual: %v request: %v", existingMetadata, oldMetadata) - return - } - err = vS.inodeVolumeHandle.PutStream(containerInodeNumber, MiddlewareStream, newMetadata) - - return -} - -func (vS *volumeStruct) Mkdir(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, basename string, filePerm inode.InodeMode) (newDirInodeNumber inode.InodeNumber, err error) { - startTime := time.Now() - defer func() { - globals.MkdirUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.MkdirErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - // Make sure the file basename is not too long - err = validateBaseName(basename) - if err != nil { - return 0, err - } - - newDirInodeNumber, err = vS.inodeVolumeHandle.CreateDir(filePerm, userID, groupID) - if err != nil { - logger.ErrorWithError(err) - return 0, err - } - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - - destroyErr := vS.inodeVolumeHandle.Destroy(newDirInodeNumber) - if destroyErr != nil { - logger.WarnfWithError(destroyErr, "couldn't destroy inode %v after failed Access(F_OK) in fs.Mkdir", newDirInodeNumber) - } - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return 0, err - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK|inode.X_OK, - inode.NoOverride) { - - destroyErr := vS.inodeVolumeHandle.Destroy(newDirInodeNumber) - if destroyErr != nil { - logger.WarnfWithError(destroyErr, "couldn't destroy inode %v after failed Access(W_OK|X_OK) in fs.Mkdir", newDirInodeNumber) - } - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return 0, err - } - - err = vS.inodeVolumeHandle.Link(inodeNumber, basename, newDirInodeNumber, false) - if err != nil { - destroyErr := vS.inodeVolumeHandle.Destroy(newDirInodeNumber) - if destroyErr != nil { - logger.WarnfWithError(destroyErr, "couldn't destroy inode %v after failed Link() in fs.Mkdir", newDirInodeNumber) - } - return 0, err - } - - return newDirInodeNumber, nil -} - -func (vS *volumeStruct) RemoveXAttr(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, streamName string) (err error) { - startTime := time.Now() - defer func() { - globals.RemoveXAttrUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.RemoveXAttrErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - err = vS.inodeVolumeHandle.DeleteStream(inodeNumber, streamName) - if err != nil { - logger.ErrorfWithError(err, "Failed to delete XAttr %v of inode %v", streamName, inodeNumber) - } - - vS.untrackInFlightFileInodeData(inodeNumber, false) - - return -} - -func (vS *volumeStruct) workerForMoveAndRename(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, srcDirInodeNumber inode.InodeNumber, srcBasename string, dstDirInodeNumber inode.InodeNumber, dstBasename string) (toDestroyInodeNumber inode.InodeNumber, heldLocks *heldLocksStruct, err error) { - var ( - dirEntryBasename string - dirEntryInodeNumber inode.InodeNumber - dirInodeNumber inode.InodeNumber - retryRequired bool - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - err = validateBaseName(srcBasename) - if nil != err { - heldLocks = nil - return - } - - err = validateBaseName(dstBasename) - if nil != err { - heldLocks = nil - return - } - - // Retry until done or failure (starting with ZERO backoff) - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - // Perform backoff and update for each restart (starting with ZERO backoff of course) - - tryLockBackoffContext.backoff() - - // Construct fresh heldLocks for this restart - - heldLocks = newHeldLocks() - - // Acquire WriteLock on {srcDirInodeNumber,srcBasename} & perform Access Check - - dirInodeNumber, _, dirEntryBasename, _, retryRequired, err = - vS.resolvePath( - srcDirInodeNumber, - srcBasename, - heldLocks, - resolvePathRequireExclusiveLockOnDirEntryInode| - resolvePathRequireExclusiveLockOnDirInode) - - if nil != err { - heldLocks.free() - heldLocks = nil - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - if retryRequired { - heldLocks.free() - goto Restart - } - - if (dirInodeNumber != srcDirInodeNumber) || (dirEntryBasename != srcBasename) { - heldLocks.free() - heldLocks = nil - err = blunder.NewError(blunder.InvalidArgError, "EINVAL") - return - } - - if !vS.inodeVolumeHandle.Access(srcDirInodeNumber, userID, groupID, otherGroupIDs, inode.W_OK|inode.X_OK, inode.NoOverride) { - heldLocks.free() - heldLocks = nil - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - // Acquire WriteLock on dstDirInodeNumber & perform Access Check - - _, dirEntryInodeNumber, _, _, retryRequired, err = - vS.resolvePath( - dstDirInodeNumber, - ".", - heldLocks, - resolvePathDirEntryInodeMustBeDirectory| - resolvePathRequireExclusiveLockOnDirEntryInode) - - if nil != err { - heldLocks.free() - heldLocks = nil - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - if retryRequired { - heldLocks.free() - goto Restart - } - - if dirEntryInodeNumber != dstDirInodeNumber { - heldLocks.free() - heldLocks = nil - err = blunder.NewError(blunder.InvalidArgError, "EINVAL") - return - } - - if !vS.inodeVolumeHandle.Access(dstDirInodeNumber, userID, groupID, otherGroupIDs, inode.W_OK|inode.X_OK, inode.NoOverride) { - heldLocks.free() - heldLocks = nil - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - // Acquire WriteLock on dstBasename if it exists - - dirInodeNumber, _, dirEntryBasename, _, retryRequired, err = - vS.resolvePath( - dstDirInodeNumber, - dstBasename, - heldLocks, - resolvePathRequireExclusiveLockOnDirEntryInode) - - if nil == err { - if retryRequired { - heldLocks.free() - goto Restart - } - - if (dirInodeNumber != dstDirInodeNumber) || (dirEntryBasename != dstBasename) { - heldLocks.free() - heldLocks = nil - err = blunder.NewError(blunder.InvalidArgError, "EINVAL") - return - } - } else { - // This is actually OK... it means the target path of the Rename() isn't being potentially replaced - } - - // Locks held & Access Checks succeeded... time to do the Move - - toDestroyInodeNumber, err = vS.inodeVolumeHandle.Move(srcDirInodeNumber, srcBasename, dstDirInodeNumber, dstBasename) - - return // err returned from inode.Move() suffices here -} - -func (vS *volumeStruct) Rename(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, srcDirInodeNumber inode.InodeNumber, srcBasename string, dstDirInodeNumber inode.InodeNumber, dstBasename string) (err error) { - var ( - destroyErr error - heldLocks *heldLocksStruct - toDestroyInodeNumber inode.InodeNumber - ) - - startTime := time.Now() - defer func() { - globals.RenameUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.RenameErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - toDestroyInodeNumber, heldLocks, err = vS.workerForMoveAndRename(userID, groupID, otherGroupIDs, srcDirInodeNumber, srcBasename, dstDirInodeNumber, dstBasename) - - if (nil == err) && (inode.InodeNumber(0) != toDestroyInodeNumber) { - destroyErr = vS.inodeVolumeHandle.Destroy(toDestroyInodeNumber) - if nil != destroyErr { - logger.ErrorWithError(destroyErr) - } - } - - if nil != heldLocks { - heldLocks.free() - } - - return -} - -func (vS *volumeStruct) Move(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, srcDirInodeNumber inode.InodeNumber, srcBasename string, dstDirInodeNumber inode.InodeNumber, dstBasename string) (toDestroyInodeNumber inode.InodeNumber, err error) { - var ( - heldLocks *heldLocksStruct - ) - - startTime := time.Now() - defer func() { - globals.MoveUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.MoveErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - toDestroyInodeNumber, heldLocks, err = vS.workerForMoveAndRename(userID, groupID, otherGroupIDs, srcDirInodeNumber, srcBasename, dstDirInodeNumber, dstBasename) - - if nil != heldLocks { - heldLocks.free() - } - - return -} - -func (vS *volumeStruct) Destroy(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (err error) { - var ( - inodeLock *dlm.RWLockStruct - ) - - startTime := time.Now() - defer func() { - globals.DestroyUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.DestroyErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - - inodeLock, err = vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if nil != err { - vS.jobRWMutex.RUnlock() - return - } - err = inodeLock.WriteLock() - if nil != err { - vS.jobRWMutex.RUnlock() - return - } - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - _ = inodeLock.Unlock() - vS.jobRWMutex.RUnlock() - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK, - inode.OwnerOverride) { - _ = inodeLock.Unlock() - vS.jobRWMutex.RUnlock() - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - err = vS.inodeVolumeHandle.Destroy(inodeNumber) - - _ = inodeLock.Unlock() - vS.jobRWMutex.RUnlock() - - return -} - -func (vS *volumeStruct) Read(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, offset uint64, length uint64, profiler *utils.Profiler) (buf []byte, err error) { - startTime := time.Now() - defer func() { - globals.ReadUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.ReadBytes.Add(uint64(len(buf))) - if err != nil { - globals.ReadErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.R_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - inodeType, err := vS.inodeVolumeHandle.GetType(inodeNumber) - if err != nil { - logger.ErrorfWithError(err, "couldn't get type for inode %v", inodeNumber) - return buf, err - } - // Make sure the inode number is for a file inode - if inodeType != inode.FileType { - err = fmt.Errorf("%s: expected inode %v to be a file inode, got %v", utils.GetFnName(), inodeNumber, inodeType) - logger.ErrorWithError(err) - return buf, blunder.AddError(err, blunder.NotFileError) - } - - profiler.AddEventNow("before inode.Read()") - buf, err = vS.inodeVolumeHandle.Read(inodeNumber, offset, length, profiler) - profiler.AddEventNow("after inode.Read()") - if uint64(len(buf)) > length { - err = fmt.Errorf("%s: Buf length %v is greater than supplied length %v", utils.GetFnName(), uint64(len(buf)), length) - logger.ErrorWithError(err) - return buf, blunder.AddError(err, blunder.IOError) - } - - return buf, err -} - -func (vS *volumeStruct) readdirHelper(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, maxEntries uint64, prevReturned ...interface{}) (dirEntries []inode.DirEntry, statEntries []Stat, numEntries uint64, areMoreEntries bool, err error) { - var ( - dirEntryIndex uint64 - dlmCallerID dlm.CallerID - inodeLock *dlm.RWLockStruct - inodeVolumeHandle inode.VolumeHandle - internalErr error - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - dlmCallerID = dlm.GenerateCallerID() - inodeVolumeHandle = vS.inodeVolumeHandle - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - tryLockBackoffContext.backoff() - - inodeLock, err = inodeVolumeHandle.AttemptReadLock(inodeNumber, dlmCallerID) - if nil != err { - goto Restart - } - - if !inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, inode.NoOverride) { - internalErr = inodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("Failure unlocking a held LockID %s: %v", inodeLock.LockID, internalErr) - } - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.R_OK, inode.OwnerOverride) { - internalErr = inodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("Failure unlocking a held LockID %s: %v", inodeLock.LockID, internalErr) - } - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - dirEntries, areMoreEntries, err = inodeVolumeHandle.ReadDir(inodeNumber, maxEntries, 0, prevReturned...) - - internalErr = inodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("Failure unlocking a held LockID %s: %v", inodeLock.LockID, internalErr) - } - - if nil != err { - return - } - - // Now go back and fill in (dirEntries.Type and) statEntries - - numEntries = uint64(len(dirEntries)) - - statEntries = make([]Stat, numEntries, numEntries) - - for dirEntryIndex = 0; dirEntryIndex < numEntries; dirEntryIndex++ { - inodeLock, err = inodeVolumeHandle.AttemptReadLock(dirEntries[dirEntryIndex].InodeNumber, dlmCallerID) - if nil != err { - goto Restart - } - - statEntries[dirEntryIndex], err = vS.getstatHelperWhileLocked(dirEntries[dirEntryIndex].InodeNumber) - - internalErr = inodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("Failure unlocking a held LockID %s: %v", inodeLock.LockID, internalErr) - } - - if nil == err { - dirEntries[dirEntryIndex].Type = inode.InodeType(statEntries[dirEntryIndex][StatFType]) - } else { - logger.ErrorfWithError(err, "fs.readdirHelper(,,,inodeNumber:0x%016X,,...) couldn't `stat` %s:0x%016X... defaulting .Type to inode.DirType", inodeNumber, dirEntries[dirEntryIndex].Basename, dirEntries[dirEntryIndex].InodeNumber) - dirEntries[dirEntryIndex].Type = inode.DirType - err = nil - } - } - - return -} - -func (vS *volumeStruct) Readdir(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, maxEntries uint64, prevReturned ...interface{}) (entries []inode.DirEntry, numEntries uint64, areMoreEntries bool, err error) { - startTime := time.Now() - defer func() { - globals.ReaddirUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.ReaddirEntries.Add(uint64(len(entries))) - if err != nil { - globals.ReaddirErrors.Add(1) - } - }() - - entries, _, numEntries, areMoreEntries, err = vS.readdirHelper(userID, groupID, otherGroupIDs, inodeNumber, maxEntries, prevReturned...) - - return -} - -func (vS *volumeStruct) ReaddirPlus(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, maxEntries uint64, prevReturned ...interface{}) (dirEntries []inode.DirEntry, statEntries []Stat, numEntries uint64, areMoreEntries bool, err error) { - startTime := time.Now() - defer func() { - globals.ReaddirPlusUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.ReaddirPlusBytes.Add(uint64(len(dirEntries))) - if err != nil { - globals.ReaddirPlusErrors.Add(1) - } - }() - - dirEntries, statEntries, numEntries, areMoreEntries, err = vS.readdirHelper(userID, groupID, otherGroupIDs, inodeNumber, maxEntries, prevReturned...) - - return -} - -func (vS *volumeStruct) Readsymlink(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber) (target string, err error) { - startTime := time.Now() - defer func() { - globals.ReadsymlinkUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.ReadsymlinkErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.ReadLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.R_OK, - inode.NoOverride) { - - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - target, err = vS.inodeVolumeHandle.GetSymlink(inodeNumber) - - return target, err -} - -func (vS *volumeStruct) Resize(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, newSize uint64) (err error) { - startTime := time.Now() - defer func() { - globals.ResizeUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.ResizeErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK, - inode.OwnerOverride) { - - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - err = vS.inodeVolumeHandle.SetSize(inodeNumber, newSize) - vS.untrackInFlightFileInodeData(inodeNumber, false) - - return err -} - -func (vS *volumeStruct) Rmdir(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, basename string) (err error) { - startTime := time.Now() - defer func() { - globals.RmdirUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.RmdirErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - callerID := dlm.GenerateCallerID() - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, callerID) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK|inode.X_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - basenameInodeNumber, err := vS.inodeVolumeHandle.Lookup(inodeNumber, basename) - if nil != err { - return - } - - basenameInodeLock, err := vS.inodeVolumeHandle.InitInodeLock(basenameInodeNumber, callerID) - if err != nil { - return - } - err = basenameInodeLock.WriteLock() - if err != nil { - return - } - defer basenameInodeLock.Unlock() - - // no permissions are required on the target directory - - err = vS.rmdirActual(inodeNumber, basename, basenameInodeNumber) - return -} - -func (vS *volumeStruct) rmdirActual(inodeNumber inode.InodeNumber, basename string, basenameInodeNumber inode.InodeNumber) (err error) { - var ( - basenameInodeType inode.InodeType - dirEntries uint64 - toDestroyInodeNumber inode.InodeNumber - ) - - basenameInodeType, err = vS.inodeVolumeHandle.GetType(basenameInodeNumber) - if nil != err { - return - } - - if inode.DirType != basenameInodeType { - err = fmt.Errorf("Rmdir() called on non-Directory") - err = blunder.AddError(err, blunder.NotDirError) - return - } - - dirEntries, err = vS.inodeVolumeHandle.NumDirEntries(basenameInodeNumber) - if nil != err { - return - } - - if 2 != dirEntries { - err = fmt.Errorf("Directory not empty") - err = blunder.AddError(err, blunder.NotEmptyError) - return - } - - toDestroyInodeNumber, err = vS.inodeVolumeHandle.Unlink(inodeNumber, basename, false) - if nil != err { - return - } - - if inode.InodeNumber(0) != toDestroyInodeNumber { - err = vS.inodeVolumeHandle.Destroy(basenameInodeNumber) - if nil != err { - return - } - } - - return -} - -func (vS *volumeStruct) Setstat(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, stat Stat) (err error) { - startTime := time.Now() - defer func() { - globals.SetstatUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.SetstatErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.P_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotPermError, "EPERM") - return - } - - // perform all permissions checks before making any changes - // - // changing the filesize requires write permission - _, ok := stat[StatSize] - if ok { - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.NotPermError, "EPERM") - return - } - } - - // most other attributes can only be changed by the owner of the file - ownerOnly := []StatKey{StatCTime, StatCRTime, StatMTime, StatATime, StatMode, StatUserID, StatGroupID} - for _, key := range ownerOnly { - _, ok := stat[key] - if ok { - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.P_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotPermError, "EPERM") - return - } - break - } - } - - // the superuser (root) is the only one that can change the owner of the file to a - // different user, but the owner of the file can perform a no-op "change" in - // ownership - newUserID, settingUserID := stat[StatUserID] - if settingUserID && userID != inode.InodeRootUserID { - if userID != inode.InodeUserID(newUserID) { - err = blunder.NewError(blunder.NotPermError, "EPERM") - return - } - } - - // the group can only be changed to the current group or another group the owner - // is in (unless its the superuser asking) - newGroupID, settingGroupID := stat[StatGroupID] - if settingGroupID && groupID != inode.InodeGroupID(newGroupID) && userID != inode.InodeRootUserID { - - err = blunder.NewError(blunder.NotPermError, "EPERM") - for _, otherID := range otherGroupIDs { - if inode.InodeGroupID(newGroupID) == otherID { - err = nil - break - } - } - if err != nil { - return - } - } - - // sanity checks for invalid/illegal values - if settingUserID { - // Since we are using a uint64 to convey a uint32 value, make sure we didn't get something too big - if newUserID > uint64(math.MaxUint32) { - err = fmt.Errorf("%s: userID is too large - value is %v, max is %v.", utils.GetFnName(), newUserID, uint64(math.MaxUint32)) - err = blunder.AddError(err, blunder.InvalidUserIDError) - return - } - } - - if settingGroupID { - // Since we are using a uint64 to convey a uint32 value, make sure we didn't get something too big - if newGroupID > uint64(math.MaxUint32) { - err = fmt.Errorf("%s: groupID is too large - value is %v, max is %v.", utils.GetFnName(), newGroupID, uint64(math.MaxUint32)) - err = blunder.AddError(err, blunder.InvalidGroupIDError) - return - } - } - - filePerm, settingFilePerm := stat[StatMode] - if settingFilePerm { - // Since we are using a uint64 to convey a 12 bit value, make sure we didn't get something too big - if filePerm >= 1<<12 { - err = fmt.Errorf("%s: filePerm is too large - value is %v, max is %v.", utils.GetFnName(), - filePerm, 1<<12) - err = blunder.AddError(err, blunder.InvalidFileModeError) - return - } - } - - // get to work setting things - // - // Set permissions, if present in the map - if settingFilePerm { - err = vS.inodeVolumeHandle.SetPermMode(inodeNumber, inode.InodeMode(filePerm)) - if err != nil { - logger.ErrorWithError(err) - return err - } - } - - // set owner and/or group owner, if present in the map - err = nil - if settingUserID && settingGroupID { - err = vS.inodeVolumeHandle.SetOwnerUserIDGroupID(inodeNumber, inode.InodeUserID(newUserID), - inode.InodeGroupID(newGroupID)) - } else if settingUserID { - err = vS.inodeVolumeHandle.SetOwnerUserID(inodeNumber, inode.InodeUserID(newUserID)) - } else if settingGroupID { - err = vS.inodeVolumeHandle.SetOwnerGroupID(inodeNumber, inode.InodeGroupID(newGroupID)) - } - if err != nil { - logger.ErrorWithError(err) - return - } - - // Set crtime, if present in the map - crtime, ok := stat[StatCRTime] - if ok { - newCreationTime := time.Unix(0, int64(crtime)) - err = vS.inodeVolumeHandle.SetCreationTime(inodeNumber, newCreationTime) - if err != nil { - logger.ErrorWithError(err) - return err - } - } - - // Set mtime, if present in the map - mtime, ok := stat[StatMTime] - if ok { - newModificationTime := time.Unix(0, int64(mtime)) - err = vS.inodeVolumeHandle.SetModificationTime(inodeNumber, newModificationTime) - if err != nil { - logger.ErrorWithError(err) - return err - } - } - - // Set atime, if present in the map - atime, ok := stat[StatATime] - if ok { - newAccessTime := time.Unix(0, int64(atime)) - err = vS.inodeVolumeHandle.SetAccessTime(inodeNumber, newAccessTime) - if err != nil { - logger.ErrorWithError(err) - return err - } - } - - // ctime is used to reliably determine whether the contents of a file - // have changed so it cannot be altered by a client (some security - // software depends on this) - ctime, ok := stat[StatCTime] - if ok { - newAccessTime := time.Unix(0, int64(ctime)) - logger.Infof("%s: ignoring attempt to change ctime to %v on volume '%s' inode %v", - utils.GetFnName(), newAccessTime, vS.volumeName, inodeNumber) - } - - // Set size, if present in the map - size, ok := stat[StatSize] - if ok { - err = vS.inodeVolumeHandle.SetSize(inodeNumber, size) - if err != nil { - logger.ErrorWithError(err) - return err - } - } - - return -} - -func (vS *volumeStruct) SetXAttr(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, streamName string, value []byte, flags int) (err error) { - startTime := time.Now() - defer func() { - globals.SetXAttrUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.SetXAttrErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - switch flags { - case SetXAttrCreateOrReplace: - break - case SetXAttrCreate: - _, err = vS.GetXAttr(userID, groupID, otherGroupIDs, inodeNumber, streamName) - if err == nil { - return blunder.AddError(err, blunder.FileExistsError) - } - case SetXAttrReplace: - _, err = vS.GetXAttr(userID, groupID, otherGroupIDs, inodeNumber, streamName) - if err != nil { - return blunder.AddError(err, blunder.StreamNotFound) - } - default: - return blunder.AddError(err, blunder.InvalidArgError) - } - - err = vS.inodeVolumeHandle.PutStream(inodeNumber, streamName, value) - if err != nil { - logger.ErrorfWithError(err, "Failed to set XAttr %v to inode %v", streamName, inodeNumber) - } - - vS.untrackInFlightFileInodeData(inodeNumber, false) - - return -} - -func (vS *volumeStruct) StatVfs() (statVFS StatVFS, err error) { - startTime := time.Now() - defer func() { - globals.StatVfsUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.StatVfsErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - statVFS = make(map[StatVFSKey]uint64) - - statVFS[StatVFSFilesystemID] = vS.inodeVolumeHandle.GetFSID() - statVFS[StatVFSBlockSize] = vS.reportedBlockSize - statVFS[StatVFSFragmentSize] = vS.reportedFragmentSize - statVFS[StatVFSTotalBlocks] = vS.reportedNumBlocks - statVFS[StatVFSFreeBlocks] = vS.reportedNumBlocks - statVFS[StatVFSAvailBlocks] = vS.reportedNumBlocks - statVFS[StatVFSTotalInodes] = vS.reportedNumInodes - statVFS[StatVFSFreeInodes] = vS.reportedNumInodes - statVFS[StatVFSAvailInodes] = vS.reportedNumInodes - statVFS[StatVFSMountFlags] = 0 - statVFS[StatVFSMaxFilenameLen] = FileNameMax - - return statVFS, nil -} - -func (vS *volumeStruct) Symlink(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, basename string, target string) (symlinkInodeNumber inode.InodeNumber, err error) { - startTime := time.Now() - defer func() { - globals.SymlinkUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.SymlinkErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - err = validateBaseName(basename) - if err != nil { - return - } - - err = validateFullPath(target) - if err != nil { - return - } - - // Mode for symlinks defaults to rwxrwxrwx, i.e. inode.PosixModePerm - symlinkInodeNumber, err = vS.inodeVolumeHandle.CreateSymlink(target, inode.PosixModePerm, userID, groupID) - if err != nil { - return - } - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - - destroyErr := vS.inodeVolumeHandle.Destroy(symlinkInodeNumber) - if destroyErr != nil { - logger.WarnfWithError(destroyErr, "couldn't destroy inode %v after failed Access(F_OK) in fs.Symlink", symlinkInodeNumber) - } - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK|inode.X_OK, - inode.NoOverride) { - - destroyErr := vS.inodeVolumeHandle.Destroy(symlinkInodeNumber) - if destroyErr != nil { - logger.WarnfWithError(destroyErr, "couldn't destroy inode %v after failed Access(W_OK|X_OK) in fs.Symlink", symlinkInodeNumber) - } - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - err = vS.inodeVolumeHandle.Link(inodeNumber, basename, symlinkInodeNumber, false) - if err != nil { - destroyErr := vS.inodeVolumeHandle.Destroy(symlinkInodeNumber) - if destroyErr != nil { - logger.WarnfWithError(destroyErr, "couldn't destroy inode %v after failed Link() in fs.Symlink", symlinkInodeNumber) - } - return - } - - return -} - -func (vS *volumeStruct) Unlink(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, basename string) (err error) { - startTime := time.Now() - defer func() { - globals.UnlinkUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.UnlinkErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - callerID := dlm.GenerateCallerID() - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, callerID) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK|inode.X_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - basenameInodeNumber, err := vS.inodeVolumeHandle.Lookup(inodeNumber, basename) - if nil != err { - return - } - - basenameInodeLock, err := vS.inodeVolumeHandle.InitInodeLock(basenameInodeNumber, callerID) - if err != nil { - return - } - err = basenameInodeLock.WriteLock() - if err != nil { - return - } - defer basenameInodeLock.Unlock() - - err = vS.unlinkActual(inodeNumber, basename, basenameInodeNumber) - return -} - -func (vS *volumeStruct) unlinkActual(inodeNumber inode.InodeNumber, basename string, basenameInodeNumber inode.InodeNumber) (err error) { - var ( - basenameInodeType inode.InodeType - toDestroyInodeNumber inode.InodeNumber - ) - - basenameInodeType, err = vS.inodeVolumeHandle.GetType(basenameInodeNumber) - if nil != err { - return - } - - if inode.DirType == basenameInodeType { - err = fmt.Errorf("Unlink() called on a Directory") - err = blunder.AddError(err, blunder.IsDirError) - return - } - - toDestroyInodeNumber, err = vS.inodeVolumeHandle.Unlink(inodeNumber, basename, false) - if nil != err { - return - } - - if inode.InodeNumber(0) != toDestroyInodeNumber { - vS.untrackInFlightFileInodeData(basenameInodeNumber, false) - err = vS.inodeVolumeHandle.Destroy(toDestroyInodeNumber) - } - - return -} - -func (vS *volumeStruct) VolumeName() (volumeName string) { - startTime := time.Now() - - volumeName = vS.volumeName - globals.VolumeNameUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - return -} - -func (vS *volumeStruct) Write(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, offset uint64, buf []byte, profiler *utils.Profiler) (size uint64, err error) { - startTime := time.Now() - defer func() { - globals.WriteUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.WriteBytes.Add(size) - if err != nil { - globals.WriteErrors.Add(1) - } - }() - - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - logger.Tracef("fs.Write(): starting volume '%s' inode %v offset %v len %v", - vS.volumeName, inodeNumber, offset, len(buf)) - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - profiler.AddEventNow("before inode.Write()") - err = vS.inodeVolumeHandle.Write(inodeNumber, offset, buf, profiler) - profiler.AddEventNow("after inode.Write()") - // write to Swift presumably succeeds or fails as a whole - if err != nil { - return 0, err - } - - logger.Tracef("fs.Write(): tracking write volume '%s' inode %v", vS.volumeName, inodeNumber) - vS.trackInFlightFileInodeData(inodeNumber) - size = uint64(len(buf)) - - return -} - -func (vS *volumeStruct) Wrote(userID inode.InodeUserID, groupID inode.InodeGroupID, otherGroupIDs []inode.InodeGroupID, inodeNumber inode.InodeNumber, containerName string, objectName string, fileOffset []uint64, objectOffset []uint64, length []uint64, wroteTime uint64) (err error) { - vS.jobRWMutex.RLock() - defer vS.jobRWMutex.RUnlock() - - inodeLock, err := vS.inodeVolumeHandle.InitInodeLock(inodeNumber, nil) - if err != nil { - return - } - err = inodeLock.WriteLock() - if err != nil { - return - } - defer inodeLock.Unlock() - - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.F_OK, - inode.NoOverride) { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - return - } - if !vS.inodeVolumeHandle.Access(inodeNumber, userID, groupID, otherGroupIDs, inode.W_OK, - inode.OwnerOverride) { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - return - } - - err = vS.inodeVolumeHandle.Flush(inodeNumber, false) - vS.untrackInFlightFileInodeData(inodeNumber, false) - - inodeWroteTime := time.Unix(0, int64(wroteTime)) - - err = vS.inodeVolumeHandle.Wrote(inodeNumber, containerName, objectName, fileOffset, objectOffset, length, inodeWroteTime, true) - - return // err, as set by inode.Wrote(), is sufficient -} - -func validateBaseName(baseName string) (err error) { - // Make sure the file baseName is not too long - baseLen := len(baseName) - if baseLen > FileNameMax { - err = fmt.Errorf("%s: basename is too long. Length %v, max %v", utils.GetFnName(), baseLen, FileNameMax) - logger.ErrorWithError(err) - return blunder.AddError(err, blunder.NameTooLongError) - } - return -} - -func validateFullPath(fullPath string) (err error) { - pathLen := len(fullPath) - if pathLen > FilePathMax { - err = fmt.Errorf("%s: fullpath is too long. Length %v, max %v", utils.GetFnName(), pathLen, FilePathMax) - logger.ErrorWithError(err) - return blunder.AddError(err, blunder.NameTooLongError) - } - return -} - -func revSplitPath(fullpath string) []string { - // TrimPrefix avoids empty [0] element in pathSegments - trimmed := strings.TrimPrefix(fullpath, "/") - if trimmed == "" { - // path.Clean("") = ".", which is not useful - return []string{} - } - - segments := strings.Split(path.Clean(trimmed), "/") - slen := len(segments) - for i := 0; i < slen/2; i++ { - segments[i], segments[slen-i-1] = segments[slen-i-1], segments[i] - } - return segments -} - -// Utility function to unlink, but not destroy, a particular file or empty subdirectory. -// -// This function checks that the directory is empty. -// -// The caller of this function must hold appropriate locks. -// -// obstacleInodeNumber must refer to an existing file or directory -// that is (a) already part of the directory tree and (b) not the root -// directory. -func (vS *volumeStruct) removeObstacleToObjectPut(callerID dlm.CallerID, dirInodeNumber inode.InodeNumber, obstacleName string, obstacleInodeNumber inode.InodeNumber) (err error) { - var ( - fileType inode.InodeType - numEntries uint64 - statResult Stat - toDestroyInodeNumber inode.InodeNumber - ) - - statResult, err = vS.getstatHelper(obstacleInodeNumber, callerID) - if err != nil { - return - } - - fileType = inode.InodeType(statResult[StatFType]) - if fileType == inode.FileType || fileType == inode.SymlinkType { - // Files and symlinks can always, barring errors, be unlinked - toDestroyInodeNumber, err = vS.inodeVolumeHandle.Unlink(dirInodeNumber, obstacleName, false) - if err != nil { - return - } - } else if fileType == inode.DirType { - numEntries, err = vS.inodeVolumeHandle.NumDirEntries(obstacleInodeNumber) - if err != nil { - return - } - if numEntries >= 3 { - // We're looking at a pre-existing, user-visible directory - // that's linked into the directory structure, so we've - // got at least two entries, namely "." and ".." - // - // If there's a third, then the directory is non-empty. - err = blunder.NewError(blunder.NotEmptyError, "%s is a non-empty directory", obstacleName) - return - } else { - // We don't want to call Rmdir() here since - // that function (a) grabs locks, (b) checks - // that it's a directory and is empty, then - // (c) calls Unlink() and Destroy(). - // - // We already have the locks and we've already - // checked that it's empty, so let's just get - // down to it. - toDestroyInodeNumber, err = vS.inodeVolumeHandle.Unlink(dirInodeNumber, obstacleName, false) - if err != nil { - return - } - } - } - - if inode.InodeNumber(0) != toDestroyInodeNumber { - err = vS.inodeVolumeHandle.Destroy(toDestroyInodeNumber) - } - - return -} - -// Utility function to append entries to reply -func appendReadPlanEntries(readPlan []inode.ReadPlanStep, readRangeOut *[]inode.ReadPlanStep) (numEntries uint64) { - for i := range readPlan { - entry := inode.ReadPlanStep{ObjectPath: readPlan[i].ObjectPath, Offset: readPlan[i].Offset, Length: readPlan[i].Length} - *readRangeOut = append(*readRangeOut, entry) - numEntries++ - } - return -} diff --git a/fs/api_test.go b/fs/api_test.go deleted file mode 100644 index 62c94453..00000000 --- a/fs/api_test.go +++ /dev/null @@ -1,1358 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fs - -import ( - "bytes" - "math" - "strings" - "syscall" - "testing" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/inode" -) - -// TODO: Enhance this to do a stat() as well and check number of files -func expectDirectory(t *testing.T, userID inode.InodeUserID, groupID inode.InodeGroupID, inodeNum inode.InodeNumber, expectedEntries []string) { - readdirEntries, numEntries, moreEntries, err := testVolumeStruct.Readdir(userID, groupID, nil, inodeNum, 0, "") - if nil != err { - t.Fatalf("Readdir() [#1] returned error: %v", err) - } - if uint64(len(expectedEntries)) != numEntries { - t.Fatalf("Readdir() [#1] returned unexpected number of entries (%v) - should have been %v", numEntries, len(expectedEntries)) - } - if moreEntries { - t.Fatalf("Readdir() [#1] returned moreEntries == true... should have been false") - } - - entriesFound := make(map[string]bool) - for i := uint64(0); i < numEntries; i++ { - entriesFound[readdirEntries[i].Basename] = true - } - - for i := 0; i < len(expectedEntries); i++ { - expected := expectedEntries[i] - _, found := entriesFound[expected] - if !found { - t.Errorf("Expected entry %s not found in readdirEntries", expected) - } - } -} - -func createTestDirectory(t *testing.T, dirname string) (dirInode inode.InodeNumber) { - var err error - - // Get root dir inode number - rootDirInodeNumber := inode.RootDirInodeNumber - - dirInode, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, dirname, inode.PosixModePerm) - if nil != err { - t.Fatalf("Mkdir() returned error: %v", err) - } - - return dirInode -} - -// TODO: Ultimately, each of these tests should at least run in their own directory -// a la createTestDirectory(), or preferably some stronger effort should be -// made to insulate them from each other. -func TestCreateAndLookup(t *testing.T) { - testSetup(t, false) - - rootDirInodeNumber := inode.RootDirInodeNumber - basename := "create_lookup.test" - - createdFileInodeNumber, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename, inode.PosixModePerm) - if err != nil { - t.Fatalf("Unexpectedly couldn't create file: %v", err) - } - - foundFileInodeNumber, err := testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename) - if err != nil { - t.Fatalf("Unexpectedly failed to look up %v", basename) - } - - if createdFileInodeNumber != foundFileInodeNumber { - t.Fatalf("Expected created inode number %v to equal found inode number %v", createdFileInodeNumber, foundFileInodeNumber) - } - - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename) - if nil != err { - t.Fatalf("Unlink() returned error: %v", err) - } - - testTeardown(t) -} - -func TestGetstat(t *testing.T) { - testSetup(t, false) - - rootDirInodeNumber := inode.RootDirInodeNumber - basename := "getstat.test" - timeBeforeCreation := uint64(time.Now().UnixNano()) - - inodeNumber, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename, inode.PosixModePerm) - if err != nil { - t.Fatalf("couldn't create file: %v", err) - } - - stat, err := testVolumeStruct.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber) - if err != nil { - t.Fatalf("couldn't stat inode %v: %v", inodeNumber, err) - } - - if !(math.Abs(float64(int64(timeBeforeCreation)-int64(stat[StatCRTime]))) < 0.1*1000000000) { // nanoseconds - t.Errorf("unexpectedly skewed StatCRTime %v is not close to %v", stat[StatCRTime], timeBeforeCreation) - } - if !(math.Abs(float64(int64(timeBeforeCreation)-int64(stat[StatMTime]))) < 0.1*1000000000) { // nanoseconds - t.Errorf("unexpectedly skewed StatMTime %v is not close to %v", stat[StatMTime], timeBeforeCreation) - } - if stat[StatSize] != 0 { - t.Errorf("expected size to be 0") - } - if stat[StatNLink] != 1 { - t.Errorf("expected number of links to be one, got %v", stat[StatNLink]) - } - - // TODO: perform a write, check that size has changed accordingly - // TODO: make and delete hardlinks, check that link count has changed accordingly - - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename) - if nil != err { - t.Fatalf("Unlink() returned error: %v", err) - } - - testTeardown(t) -} - -// TestAllAPIPositiveCases() follows the following "positive" test steps: -// -// Mount A : mount the specified test Volume (must be empty) -// Mkdir A/B/ : create a subdirectory within Volume directory -// Create #1 A/C : create and open a normal file within Volume directory -// Lookup #1 A/C : fetch the inode name of the just created normal file -// Write A/C : write something to normal file -// Read A/C : read back what was just written to normal file -// FetchExtentMapChunk A/C : fetch extentMapChunk for entire file -// Getstat #1 A/C : check the current size of the normal file -// Resize A/C : truncate the file -// Getstat #2 A/C : verify the size of the normal file is now zero -// Symlink A/D->A/C : create a symlink to the normal file -// Lookup #2 A/D : fetch the inode name of the just created symlink -// Readsymlink A/D : read the symlink to ensure it points to the normal file -// Lookup #3 A/B/ : fetch the inode name of the subdirectory -// Create #2 A/B/E : create a normal file within subdirectory -// Readdir #1 A/B/ (prev == "", max_entries == 0) : ensure we get only ".", "..", and "E" -// Statfs A : should report A has 4 "files" (normal & symlink) and 1 directory "ideally" -// Unlink #1 A/B/E : delete the normal file within the subdirectory -// Readdir #2 A/ (prev == "", max_entries == 3) : ensure we get only ".", ".." & "B" -// Readdir #3 A/ (prev == "B", max_entries == 3) : ensure we get only "C" & "D" -// Unlink #2 A/D : delete the symlink -// Unlink #3 A/C : delete the normal file -// Unlink #4 A/B : delete the subdirectory -// Unmount A : unmount the Volume -// -// TODO: Rename(), Link() tests - -var tempVolumeName string // TODO: This is currently the local file system full path - -func TestAllAPIPositiveCases(t *testing.T) { - var ( - err error - ) - - testSetup(t, false) - - // Get root dir inode number - rootDirInodeNumber := inode.RootDirInodeNumber - - // Mkdir A/B/ : create a subdirectory within Volume directory - _, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, "TestSubDirectory", inode.PosixModePerm) - // newDirInodeNum, err := testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, "TestSubDirectory") - if nil != err { - t.Fatalf("Mkdir() returned error: %v", err) - } - - // Create #1 A/C : create and open a normal file within Volume directory - basename := "TestNormalFile" - createdFileInodeNumber, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename, inode.PosixModePerm) - if err != nil { - t.Fatalf("Create() [#1] returned error: %v", err) - } - - // Lookup #1 A/C : fetch the inode name of the just created normal file - foundFileInodeNumber, err := testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename) - if err != nil { - t.Fatalf("Lookup() [#1] returned error: %v", err) - } - if createdFileInodeNumber != foundFileInodeNumber { - t.Fatalf("Expected created inode number %v to equal found inode number %v", createdFileInodeNumber, foundFileInodeNumber) - } - - // Write A/C : write something to normal file - bufToWrite := []byte{0x41, 0x42, 0x43} - write_rspSize, err := testVolumeStruct.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, createdFileInodeNumber, 0, bufToWrite, nil) - if nil != err { - t.Fatalf("Write() returned error: %v", err) - } - if uint64(len(bufToWrite)) != write_rspSize { - t.Fatalf("Write() expected to write %v bytes but actually wrote %v bytes", len(bufToWrite), write_rspSize) - } - - // don't forget to flush - err = testVolumeStruct.Flush(inode.InodeRootUserID, inode.InodeGroupID(0), nil, createdFileInodeNumber) - if err != nil { - t.Fatalf("Flush() returned error: %v", err) - } - - // Read A/C : read back what was just written to normal file - read_buf, err := testVolumeStruct.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, createdFileInodeNumber, 0, uint64(len(bufToWrite)), nil) - if nil != err { - t.Fatalf("Read() returned error: %v", err) - } - if len(bufToWrite) != len(read_buf) { - t.Fatalf("Read() expected to read %v bytes but actually read %v bytes", len(bufToWrite), len(read_buf)) - } - if 0 != bytes.Compare(bufToWrite, read_buf) { - t.Fatalf("Read() returned data different from what was written") - } - - extent_map_chunk, err := testVolumeStruct.FetchExtentMapChunk(inode.InodeRootUserID, inode.InodeGroupID(0), nil, createdFileInodeNumber, uint64(0), int64(1), int64(0)) - if nil != err { - t.Fatalf("FetchExtentMapChunk() returned error: %v", err) - } - if 0 != extent_map_chunk.FileOffsetRangeStart { - t.Fatalf("FetchExtentMapChunk() returned unexpected FileOffsetRangeStart: %v (should be 0)", extent_map_chunk.FileOffsetRangeStart) - } - if uint64(len(bufToWrite)) != extent_map_chunk.FileOffsetRangeEnd { - t.Fatalf("FetchExtentMapChunk() returned unexpected FileOffsetRangeEnd: %v (should be %v)", len(bufToWrite), extent_map_chunk.FileOffsetRangeEnd) - } - if uint64(len(bufToWrite)) != extent_map_chunk.FileSize { - t.Fatalf("FetchExtentMapChunk() returned unexpected FileSize: %v (should be %v)", len(bufToWrite), extent_map_chunk.FileSize) - } - if 1 != len(extent_map_chunk.ExtentMapEntry) { - t.Fatalf("FetchExtentMapChunk() returned unexpected len(ExtentMapEntry slice): %v (should be 1)", len(extent_map_chunk.ExtentMapEntry)) - } - if uint64(0) != extent_map_chunk.ExtentMapEntry[0].FileOffset { - t.Fatalf("FetchExtentMapChunk() returned unexpected ExtentMapEntry[0].FileOffset: %v (should be 0)", extent_map_chunk.ExtentMapEntry[0].FileOffset) - } - if uint64(len(bufToWrite)) != extent_map_chunk.ExtentMapEntry[0].Length { - t.Fatalf("FetchExtentMapChunk() returned unexpected ExtentMapEntry[0].Length: %v (should be %v)", extent_map_chunk.ExtentMapEntry[0].Length, uint64(len(bufToWrite))) - } - if uint64(0) != extent_map_chunk.ExtentMapEntry[0].LogSegmentOffset { - t.Fatalf("FetchExtentMapChunk() returned unexpected ExtentMapEntry[0].LogSegmentOffset: %v (should be 0)", extent_map_chunk.ExtentMapEntry[0].LogSegmentOffset) - } - - // Getstat #1 A/C : check the current size of the normal file - getstat_1_rspStat, err := testVolumeStruct.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, foundFileInodeNumber) - if nil != err { - t.Fatalf("Getstat() returned error: %v", err) - } - getstat_1_size, getstat_1_size_ok := getstat_1_rspStat[StatSize] - if !getstat_1_size_ok { - t.Fatalf("Getstat() returned no StatSize") - } - if uint64(len(bufToWrite)) != getstat_1_size { - t.Fatalf("Getstat() returned StatSize == %v instead of the expected %v", getstat_1_size, len(bufToWrite)) - } - - // Resize A/C : truncate the file - err = testVolumeStruct.Resize(inode.InodeRootUserID, inode.InodeGroupID(0), nil, foundFileInodeNumber, 0) - if nil != err { - t.Fatalf("Resize() returned error: %v", err) - } - - // Getstat #2 A/C : verify the size of the normal file is now zero - getstat_2_rspStat, err := testVolumeStruct.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, foundFileInodeNumber) - if nil != err { - t.Fatalf("Getstat() [#2] returned error: %v", err) - } - getstat_2_size, getstat_2_size_ok := getstat_2_rspStat[StatSize] - if !getstat_2_size_ok { - t.Fatalf("Getstat() [#2] returned no StatSize") - } - if 0 != getstat_2_size { - t.Fatalf("Getstat() [#2] returned StatSize == %v instead of the expected %v", getstat_2_size, 0) - } - - // Symlink A/D->A/C : create a symlink to the normal file - createdSymlinkInodeNumber, err := testVolumeStruct.Symlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, "TestSymlink", "TestNormalFile") - if nil != err { - t.Fatalf("Symlink() returned error: %v", err) - } - - // Lookup #2 A/D : fetch the inode name of the just created symlink - lookup_2_inodeHandle, err := testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, "TestSymlink") - if nil != err { - t.Fatalf("Lookup() [#2] returned error: %v", err) - } - if lookup_2_inodeHandle != createdSymlinkInodeNumber { - t.Fatalf("Lookup() [#2] returned unexpected InodeNumber") - } - - // Readsymlink A/D : read the symlink to ensure it points to the normal file - readsymlink_target, err := testVolumeStruct.Readsymlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lookup_2_inodeHandle) - if nil != err { - t.Fatalf("Readsymlink() returned error: %v", err) - } - if 0 != strings.Compare("TestNormalFile", readsymlink_target) { - t.Fatalf("Readsymlink() data different from what was written") - } - - // Lookup #3 A/B/ : fetch the inode name of the subdirectory - lookup_3_inodeHandle, err := testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, "TestSubDirectory") - if nil != err { - t.Fatalf("Lookup() [#3] returned error: %v", err) - } - - // Create #2 A/B/E : create a normal file within subdirectory - testSubDirectoryFileInode, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lookup_3_inodeHandle, "TestSubDirectoryFile", inode.PosixModePerm) - if nil != err { - t.Fatalf("Create() [#2] returned error: %v", err) - } - - // Readdir and examine contents - entriesExpected := []string{".", "..", "TestSubDirectoryFile"} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), lookup_3_inodeHandle, entriesExpected) - - // Link A/B/E - err = testVolumeStruct.Link(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lookup_3_inodeHandle, "TestSubDirectoryFileHardLink", testSubDirectoryFileInode) - if nil != err { - t.Fatalf("Link() returned error: %v", err) - } - - entriesExpected = []string{".", "..", "TestSubDirectoryFile", "TestSubDirectoryFileHardLink"} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), lookup_3_inodeHandle, entriesExpected) - - // Unlink #1 A/B/E : delete the normal file within the subdirectory - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lookup_3_inodeHandle, "TestSubDirectoryFile") - if nil != err { - t.Fatalf("Unlink() [#1] returned error: %v", err) - } - - entriesExpected = []string{".", "..", "TestSubDirectoryFileHardLink"} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), lookup_3_inodeHandle, entriesExpected) - - // Unlink #1.5 A/B/E : delete the normal file within the subdirectory - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lookup_3_inodeHandle, "TestSubDirectoryFileHardLink") - if nil != err { - t.Fatalf("Unlink() [#1.5] returned error: %v", err) - } - - entriesExpected = []string{".", "..", "TestSymlink", "TestNormalFile", "TestSubDirectory"} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), rootDirInodeNumber, entriesExpected) - - // Unlink #2 A/D : delete the symlink - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, "TestSymlink") - - if nil != err { - t.Fatalf("Unlink() [#2] returned error: %v", err) - } - - // Unlink #3 A/C : delete the normal file - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, "TestNormalFile") - if nil != err { - t.Fatalf("Unlink() [#3] returned error: %v", err) - } - - // Rmdir #4 A/B : delete the subdirectory - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, "TestSubDirectory") - if nil != err { - t.Fatalf("Unlink() [#4] returned error: %v", err) - } - - entriesExpected = []string{".", ".."} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), rootDirInodeNumber, entriesExpected) - - testTeardown(t) -} - -// TODO: flesh this out with other boundary condition testing for Link -func TestBadLinks(t *testing.T) { - testSetup(t, false) - - testDirInode := createTestDirectory(t, "BadLinks") - - validFile := "PerfectlyValidFile" - validFileInodeNumber, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, validFile, inode.PosixModePerm) - if err != nil { - t.Fatalf("Create() returned error: %v", err) - } - - nameTooLong := strings.Repeat("x", FileNameMax+1) - err = testVolumeStruct.Link(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, nameTooLong, validFileInodeNumber) - if nil != err { - if blunder.IsNot(err, blunder.NameTooLongError) { - t.Fatalf("Link() returned error %v, expected %v(%d).", blunder.Errno(err), blunder.NameTooLongError, blunder.NameTooLongError.Value()) - } - } else { - t.Fatal("Link() unexpectedly succeeded on too-long filename!") - } - - entriesExpected := []string{".", "..", validFile} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), testDirInode, entriesExpected) - - testTeardown(t) -} - -func TestMkdir(t *testing.T) { - testSetup(t, false) - - testDirInode := createTestDirectory(t, "Mkdir") - longButLegalFilename := strings.Repeat("x", FileNameMax) - nameTooLong := strings.Repeat("x", FileNameMax+1) - - _, err := testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, nameTooLong, inode.PosixModePerm) - if nil != err { - if blunder.IsNot(err, blunder.NameTooLongError) { - t.Fatalf("Mkdir() returned error %v, expected %v(%d).", blunder.Errno(err), blunder.NameTooLongError, blunder.NameTooLongError.Value()) - } - } else { - t.Fatal("Mkdir() unexpectedly succeeded on too-long filename!") - } - - _, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, longButLegalFilename, inode.PosixModePerm) - if err != nil { - t.Fatalf("Mkdir() returned error: %v", err) - } - - entriesExpected := []string{".", "..", longButLegalFilename} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), testDirInode, entriesExpected) - - longButLegalFullPath := "/Mkdir/" + longButLegalFilename - ino, err := testVolumeStruct.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, longButLegalFullPath) - if err != nil { - t.Fatalf("LookupPath() returned error: %v", err) - } - - _, err = testVolumeStruct.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(ino)) - if err != nil { - t.Fatalf("GetStat() returned error: %v", err) - } - - // trying to make the directory a second time should fail with EEXIST - _, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInode, longButLegalFilename, inode.PosixModePerm) - if err == nil { - t.Fatalf("Mkdir() of existing entry returned success") - } - if blunder.IsNot(err, blunder.FileExistsError) { - t.Fatalf("Mkdir() of existing entry should return FileExistsError, but got %v", err) - } - - testTeardown(t) -} - -func TestRmdir(t *testing.T) { - testSetup(t, false) - defer testTeardown(t) - - testDirInode := createTestDirectory(t, "Rmdir") - - _, err := testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInode, "test1", inode.PosixModePerm) - if err != nil { - t.Fatalf("Mkdir(\"test1\") returned error: %v", err) - } - - // the test directory can't be removed until its empty - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - inode.RootDirInodeNumber, "Rmdir") - if err == nil { - t.Fatalf("Rmdir() [#0] should have failed") - } - if blunder.IsNot(err, blunder.NotEmptyError) { - t.Fatalf("Rmdir() [#0] should have returned 'NotEmptyError', err: %v", err) - } - - // empty the test directory - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInode, "test1") - if err != nil { - t.Fatalf("Rmdir() [#1] returned error: %v", err) - } - - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - inode.RootDirInodeNumber, "Rmdir") - if err != nil { - t.Fatalf("Rmdir() [#2] returned error: %v", err) - } -} - -// TODO: flesh this out with other boundary condition testing for Rename -func TestBadRename(t *testing.T) { - testSetup(t, false) - - testDirInode := createTestDirectory(t, "BadRename") - nameTooLong := strings.Repeat("x", FileNameMax+1) - - validFile := "PerfectlyValidFile" - _, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, validFile, inode.PosixModePerm) - if nil != err { - t.Fatalf("Create() returned error: %v", err) - } - - // Try to rename a valid file to a name that is too long - err = testVolumeStruct.Rename(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, validFile, testDirInode, nameTooLong) - if nil != err { - if blunder.IsNot(err, blunder.NameTooLongError) { - t.Fatalf("Link() returned error %v, expected %v(%d).", blunder.Errno(err), blunder.NameTooLongError, blunder.NameTooLongError.Value()) - } - } else { - t.Fatal("Link() unexpectedly succeeded on too-long filename!") - } - - entriesExpected := []string{".", "..", validFile} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), testDirInode, entriesExpected) - - // Try to rename a nonexistent file with a name that is too long - err = testVolumeStruct.Rename(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, nameTooLong, testDirInode, "AlsoAGoodFilename") - if nil != err { - if blunder.IsNot(err, blunder.NameTooLongError) { - t.Fatalf("Link() returned error %v, expected %v(%d).", blunder.Errno(err), blunder.NameTooLongError, blunder.NameTooLongError.Value()) - } - } else { - t.Fatal("Link() unexpectedly succeeded on too-long filename!") - } - - entriesExpected = []string{".", "..", validFile} - expectDirectory(t, inode.InodeRootUserID, inode.InodeGroupID(0), testDirInode, entriesExpected) - - testTeardown(t) -} - -func TestBadChownChmod(t *testing.T) { - var ( - err error - ) - - testSetup(t, false) - - // Get root dir inode number - rootDirInodeNumber := inode.RootDirInodeNumber - - // Create file to play with - basename := "TestFile" - createdFileInodeNumber, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename, inode.PosixModePerm) - if err != nil { - t.Fatalf("Create() %v returned error: %v", basename, err) - } - - // Since we are playing some games with size of mode/userid/groupid, make sure that we - // correctly handle cases where the value is > uint32 - var tooBigForUint32 uint64 = math.MaxUint32 + 7<<48 - - // Validate too-big Mode - stat := make(Stat) - stat[StatMode] = tooBigForUint32 - err = testVolumeStruct.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, createdFileInodeNumber, stat) - if blunder.IsNot(err, blunder.InvalidFileModeError) { - t.Fatalf("Setstat() %v returned error %v, expected %v(%d).", basename, blunder.Errno(err), blunder.InvalidFileModeError, blunder.InvalidFileModeError.Value()) - } - delete(stat, StatMode) - - // Validate too-big UserID - stat[StatUserID] = tooBigForUint32 - err = testVolumeStruct.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, createdFileInodeNumber, stat) - if blunder.Errno(err) != int(blunder.InvalidFileModeError) { - t.Fatalf("Setstat() %v returned error %v, expected %v(%d).", basename, blunder.Errno(err), blunder.InvalidFileModeError, blunder.InvalidFileModeError.Value()) - } - delete(stat, StatUserID) - - // Validate too-big GroupID - stat[StatGroupID] = tooBigForUint32 - err = testVolumeStruct.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, createdFileInodeNumber, stat) - if blunder.Errno(err) != int(blunder.InvalidFileModeError) { - t.Fatalf("Setstat() %v returned error %v, expected %v(%d).", basename, blunder.Errno(err), blunder.InvalidFileModeError, blunder.InvalidFileModeError.Value()) - } - delete(stat, StatGroupID) - - testTeardown(t) -} - -func TestFlock(t *testing.T) { - var ( - err error - ) - - testSetup(t, false) - - rootDirInodeNumber := inode.RootDirInodeNumber - - // Create file to play with - basename := "TestLockFile" - lockFileInodeNumber, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename, inode.PosixModePerm) - if err != nil { - t.Fatalf("Create() %v returned error: %v", basename, err) - } - - // Resize the file to a 1M so that we can apply byte range locks: - err = testVolumeStruct.Resize(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, 1024*1024) - if err != nil { - t.Fatalf("Resize() %v returned error: %v", basename, err) - } - - // Write lock test: - var lock FlockStruct - lock.Type = syscall.F_WRLCK - lock.Start = 0 - lock.Len = 0 - lock.Pid = 1 - - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock) - if err != nil { - t.Fatalf("Write lock on file failed: %v", err) - } - - lock.Type = syscall.F_UNLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock) - if err != nil { - t.Fatalf("Unlock on file failed: %v", blunder.Errno(err)) - } - - lock.Type = syscall.F_WRLCK - lock.Pid = 1 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock) - if err != nil { - t.Fatalf("Write lock on file failed: %v", err) - } - - // Try another write lock from a different pid, it should fail: - var lock1 FlockStruct - lock1 = lock - lock1.Pid = 2 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock1) - if blunder.Errno(err) != int(blunder.TryAgainError) { - t.Fatalf("Write lock on a locked file should fail with EAGAIN instead got : %v", err) - } - - // Lock again from pid1, it should succeed: - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock) - if err != nil { - t.Fatalf("Relocking from same PID on file failed: %v", err) - } - - lock.Type = syscall.F_UNLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock) - if err != nil { - t.Fatalf("Unlock failed : %v", err) - } - - // Read lock test: - lock.Type = syscall.F_RDLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock) - if err != nil { - t.Fatalf("Read lock pid - 1 failed: %v", err) - } - - lock1.Type = syscall.F_RDLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock1) - if err != nil { - t.Fatalf("Read lock pid - 2 failed: %v", err) - } - - // Try write lock it should fail: - lock3 := lock - lock3.Type = syscall.F_WRLCK - lock3.Pid = 3 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock3) - if blunder.Errno(err) != int(blunder.TryAgainError) { - t.Fatalf("Write lock should have failed with EAGAIN instead got - %v", err) - } - - lock11 := lock1 - lock11.Type = syscall.F_UNLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock11) - if err != nil { - t.Fatalf("Unlock of (readlock) - 2 failed: %v", err) - } - - lock01 := lock - lock01.Type = syscall.F_UNLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock01) - if err != nil { - t.Fatalf("Unlock of (readlock) - 1 failed: %v", err) - } - - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock3) - if err != nil { - t.Fatalf("Write lock should have succeeded instead got - %v", err.Error()) - } - - lock3.Type = syscall.F_UNLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock3) - if err != nil { - t.Fatalf("Unlock of (write after read) failed: %v", err) - } - - // Multiple Range lock testing: - - var lock10 FlockStruct - lock10.Pid = 1 - lock10.Start = 100 - lock10.Len = 100 - lock10.Type = syscall.F_WRLCK - lock10.Whence = 0 - - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock10) - if err != nil { - t.Fatalf("Range test failed to lock range (100 - 200), err %v", err) - } - - lock201 := lock10 - lock201.Pid = 2 - lock201.Type = syscall.F_RDLCK - lock201.Start = 10 - lock201.Len = 10 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock201) - if err != nil { - t.Fatalf("Range test failed to read lock range (10 - 20) by pid2, err %v", err) - } - - lock202 := lock201 - lock202.Start = 90 - lock202.Len = 10 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock202) - if err != nil { - t.Fatalf("Range test failed to read lock range (90 - 100) by pid2, err %v", err) - } - - lock203 := lock202 - lock203.Start = 80 - lock203.Len = 40 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock203) - if err == nil { - t.Fatalf("Range test read lock of range (80 - 120) should have failed for pid2 err %v", err) - } - - lock204 := lock203 - lock204.Start = 180 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock204) - if err == nil { - t.Fatalf("Range test read lock of range (180 - 220) should have failed for pid2 err %v", err) - } - - lock205 := lock204 - lock205.Start = 200 - lock205.Len = 10 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock205) - if err != nil { - t.Fatalf("Range test read lock of range (200 - 210) should have succeeded for pid2 err %v", err) - } - - lock206 := lock205 - lock206.Start = 240 - lock206.Len = 10 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock206) - if err != nil { - t.Fatalf("Range test read lock of range (240 - 250) should have succeeded for pid2 err %v", err) - } - - lock101 := lock10 - lock101.Type = syscall.F_RDLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock101) - if err != nil { - t.Fatalf("Range test converting write lock to read lock of pid1 range 100 - 200 failed, err %v", err) - } - - // Now, lock 203 and 204 should succceed. - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock203) - if err != nil { - t.Fatalf("Range test read lock of range (80 - 120) should have succeeded for pid2 err %v", err) - } - - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock204) - if err != nil { - t.Fatalf("Range test read lock of range (180 - 220) should have succeeded for pid2 err %v", err) - } - - lock30 := lock10 - lock30.Pid = 3 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock30) - if err == nil { - t.Fatalf("Range test write lock of range 100 - 200 should have failed for pid3 err %v", err) - } - - lock102 := lock10 - lock102.Type = syscall.F_UNLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock102) - if err != nil { - t.Fatalf("Range test unlock of range 100 - 200 for pid1 should have succeeded, err - %v", err) - } - - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock30) - if err == nil { - t.Fatalf("Range test write lock of range 100 - 200 should have failed for pid3 err %v", err) - } - - lock207 := lock10 - lock207.Type = syscall.F_UNLCK - lock207.Pid = 2 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock207) - if err != nil { - t.Fatalf("Range test unlock of range 100 - 200 for pid2 should have succeeded, err - %v", err) - } - - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock30) - if err != nil { - t.Fatalf("Range test write lock of range 100 - 200 should have succeeded for pid3 err %v", err) - } - - lock301 := lock30 - lock301.Type = syscall.F_UNLCK - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock301) - if err != nil { - t.Fatalf("Range test unlock of range 100 - 200 should have succeeded for pid3 err %v", err) - } - - lock2u1 := lock201 - lock2u1.Type = syscall.F_UNLCK - lock2u1.Start = 0 - lock2u1.Len = 150 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock2u1) - if err != nil { - t.Fatalf("Range test unlock of range 0 - 150 should have succeeded for pid2 err %v", err) - } - - lock2u2 := lock2u1 - lock2u2.Start = 150 - lock2u2.Len = 150 - _, err = testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_SETLK, &lock2u2) - if err != nil { - t.Fatalf("Range test unlock of range 150 - 300 should have succeeded for pid2 err %v", err) - } - - lock30.Start = 0 - lock30.Len = 250 - lock30.Type = syscall.F_WRLCK - lockHeld, err := testVolumeStruct.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, lockFileInodeNumber, syscall.F_GETLK, &lock30) - if err != nil { - t.Fatalf("Range test GET write lock of range 0 - 250 should have succeeded for pid3 err %v lockHeld %+v", err, lockHeld) - } - - if lock30.Type != syscall.F_UNLCK { - t.Fatalf("GetLock should have succeeded for range 0 - 250 for pid 3, err %v", err) - } - - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, rootDirInodeNumber, basename) - if err != nil { - t.Fatalf("Unlink() %v returned error: %v", basename, err) - } - - testTeardown(t) -} - -// Verify that the file system API works correctly with stale inode numbers, -// as can happen if an NFS client cache gets out of sync because another NFS -// client as removed a file or directory. -func TestStaleInodes(t *testing.T) { - var ( - rootDirInodeNumber inode.InodeNumber = inode.RootDirInodeNumber - testDirname string = "stale_inodes_test" - testFileName string = "valid_file" - staleDirName string = "dir" - staleFileName string = "file" - testDirInodeNumber inode.InodeNumber - testFileInodeNumber inode.InodeNumber - staleDirInodeNumber inode.InodeNumber - staleFileInodeNumber inode.InodeNumber - err error - ) - - testSetup(t, false) - - // scratchpad directory for testing - testDirInodeNumber, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - rootDirInodeNumber, testDirname, 0755) - if nil != err { - t.Fatalf("Mkdir() '%s' returned error: %v", testDirname, err) - } - - // create a valid test file - testFileInodeNumber, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, testFileName, 0644) - if nil != err { - t.Fatalf("Create() '%s' returned error: %v", testFileName, err) - } - - // get an inode number that used to belong to a dirctory - _, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, staleDirName, 0755) - if nil != err { - t.Fatalf("Mkdir() '%s' returned error: %v", testDirname, err) - } - staleDirInodeNumber, err = testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, staleDirName) - if err != nil { - t.Fatalf("Unexpectedly failed to look up of '%s': %v", testDirname, err) - } - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, staleDirName) - if nil != err { - t.Fatalf("Rmdir() of '%s' returned error: %v", staleDirName, err) - } - - // get an inode number that used to belong to a file (it shouldn't - // really matter which type of file the inode used to be, but it doesn't - // hurt to have two to play with) - _, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, staleFileName, 0644) - if nil != err { - t.Fatalf("Mkdir() '%s' returned error: %v", testDirname, err) - } - staleFileInodeNumber, err = testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, staleFileName) - if err != nil { - t.Fatalf("Unexpectedly failed to look up of '%s': %v", testDirname, err) - } - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, staleFileName) - if nil != err { - t.Fatalf("Unlink() of '%s' returned error: %v", staleFileName, err) - } - - // Stat - _, err = testVolumeStruct.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, staleFileInodeNumber) - if nil == err { - t.Fatalf("Getstat() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Getstat() should have failed with NotFoundError, instead got: %v", err) - } - - // Mkdir - _, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleDirInodeNumber, "TestSubDirectory", 0755) - if nil == err { - t.Fatalf("Mkdir() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Mkdir() should have failed with NotFoundError, instead got: %v", err) - } - - // Rmdir - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleDirInodeNumber, "fubar") - if nil == err { - t.Fatalf("Rmdir() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Rmdir() should have failed with NotFoundError, instead got: %v", err) - } - - // Create - _, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleDirInodeNumber, "fubar", 0644) - if nil == err { - t.Fatalf("Create() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Create() should have failed with NotFoundError, instead got: %v", err) - } - - // Lookup - _, err = testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleDirInodeNumber, "fubar") - if nil == err { - t.Fatalf("Lookup() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Lookup() should have failed with NotFoundError, instead got: %v", err) - } - - // Write - bufToWrite := []byte{0x41, 0x42, 0x43} - _, err = testVolumeStruct.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleFileInodeNumber, 0, bufToWrite, nil) - if nil == err { - t.Fatalf("Write() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Write() should have failed with NotFoundError, instead got: %v", err) - } - - // Read - _, err = testVolumeStruct.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleFileInodeNumber, 0, uint64(len(bufToWrite)), nil) - if nil == err { - t.Fatalf("Read() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Read() should have failed with NotFoundError, instead got: %v", err) - } - - // Trunc - err = testVolumeStruct.Resize(inode.InodeRootUserID, inode.InodeGroupID(0), nil, staleFileInodeNumber, 77) - if nil == err { - t.Fatalf("Resize() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Resize() should have failed with NotFoundError, instead got: %v", err) - } - - // Symlink - _, err = testVolumeStruct.Symlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleDirInodeNumber, "TestSymlink", "fubar") - if nil == err { - t.Fatalf("Symlink() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Symlink() should have failed with NotFoundError, instead got: %v", err) - } - - // Readsymlink (that we didn't create) - _, err = testVolumeStruct.Readsymlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, staleFileInodeNumber) - if nil == err { - t.Fatalf("Readsymlink() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Readsymlink() should have failed with NotFoundError, instead got: %v", err) - } - - // Readdir - _, _, _, err = testVolumeStruct.Readdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, staleDirInodeNumber, 0, "") - if nil == err { - t.Fatalf("Readdir() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Readdir() should have failed with NotFoundError, instead got: %v", err) - } - - // Link -- two cases, one with stale directory and one with stale file - err = testVolumeStruct.Link(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleDirInodeNumber, "fubar", testFileInodeNumber) - if nil == err { - t.Fatalf("Link(1) should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Link(1) should have failed with NotFoundError, instead got: %v", err) - } - - err = testVolumeStruct.Link(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, testFileName, staleFileInodeNumber) - if nil == err { - t.Fatalf("Link(2) should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Link(2) should have failed with NotFoundError, instead got: %v", err) - } - - // Unlink - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleDirInodeNumber, "fubar") - if nil == err { - t.Fatalf("Unlink() should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Unlink() should have failed with NotFoundError, instead got: %v", err) - } - - // Rename -- two cases, one with stale src directory and one with stale dest - err = testVolumeStruct.Rename(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, "fubar", staleDirInodeNumber, "barfu") - if nil == err { - t.Fatalf("Rename(1) should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Rename(1) should have failed with NotFoundError, instead got: %v", err) - } - - err = testVolumeStruct.Rename(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - staleDirInodeNumber, "fubar", testDirInodeNumber, "barfu") - if nil == err { - t.Fatalf("Rename(2) should not have returned success") - } - if blunder.IsNot(err, blunder.NotFoundError) { - t.Fatalf("Rename(2) should have failed with NotFoundError, instead got: %v", err) - } - - // cleanup test file and directory - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - testDirInodeNumber, testFileName) - if nil != err { - t.Fatalf("Unlink() of '%s' returned error: %v", testFileName, err) - } - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - rootDirInodeNumber, testDirname) - if nil != err { - t.Fatalf("Rmdir() of '%s' returned error: %v", testDirname, err) - } - - testTeardown(t) -} - -func TestMiddlewareGetContainer(t *testing.T) { - var ents []ContainerEntry - testSetup(t, false) - - testDirInode := createTestDirectory(t, "container") - - marker1 := "a_marker" - _, err := testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, marker1, inode.PosixModePerm) - if err != nil { - t.Fatalf("Create() returned error: %v", err) - } - marker2 := "b_marker" - _, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testDirInode, marker2, inode.PosixModePerm) - if err != nil { - t.Fatalf("Create() returned error: %v", err) - } - - ents, err = testVolumeStruct.MiddlewareGetContainer("container", 10, "a", "", "", "") - if nil != err { - t.Fatalf("got some error: %v", err) - } - if 2 != len(ents) { - t.Fatalf("marker a gave wrong number of entries: %v", ents) - } - - ents, err = testVolumeStruct.MiddlewareGetContainer("container", 10, "b", "", "", "") - if nil != err { - t.Fatalf("got some error: %v", err) - } - if 1 != len(ents) { - t.Fatalf("marker b gave wrong number of entries: %v", ents) - } - - ents, err = testVolumeStruct.MiddlewareGetContainer("container", 10, "a_marker", "", "", "") - if nil != err { - t.Fatalf("got some error: %v", err) - } - if 1 != len(ents) { - t.Fatalf("marker a_marker gave wrong number of entries: %v", ents) - } - - testTeardown(t) -} - -// Verify that the metadata for the object at containerObjPath, as returned by -// MiddlewareHeadResponse(), matches the metadata in opMetdata. opMetadata is -// presumably returned by some middleware operation named opName, but it could -// come from somewhere else. -// -// Throw testing errors if anything doesn't match. stepName and opName are used -// in the error strings. -func verifyMetadata(t *testing.T, containerObjPath string, - stepName string, opName string, opMeta *HeadResponse) { - - // how to print time stamps (the date is dropped) - timeFormat := "15:04:05.000000000" - - var ( - headMeta HeadResponse - err error - ) - - // fetch the current metadata (implicit and explicit) - headMeta, err = testVolumeStruct.MiddlewareHeadResponse(containerObjPath) - if err != nil { - t.Errorf("MiddlewareHeadResponse() for '%s' op %s '%s' failed: %v", - containerObjPath, opName, stepName, err) - return - } - - // validate the "explicit" metadata - if !bytes.Equal(opMeta.Metadata, headMeta.Metadata) { - t.Errorf("object '%s' op %s '%s' op metadata '%s' does not match stat metadata '%s'", - containerObjPath, opName, stepName, opMeta.Metadata, headMeta.Metadata) - } - - // check the rest of the attributes and quit after the first mismatch - if opMeta.IsDir != headMeta.IsDir { - t.Errorf("object '%s' op %s '%s' op IsDir '%v' does not match stat IsDir '%v'", - containerObjPath, opName, stepName, opMeta.IsDir, headMeta.IsDir) - return - } - if opMeta.FileSize != headMeta.FileSize { - t.Errorf("object '%s' op %s '%s' op FileSize '%d' does not match stat FileSize '%d'", - containerObjPath, opName, stepName, opMeta.FileSize, headMeta.FileSize) - return - } - if opMeta.NumWrites != headMeta.NumWrites { - t.Errorf("object '%s' op %s '%s' op NumWrites '%d' does not match stat NumWrites '%d'", - containerObjPath, opName, stepName, opMeta.NumWrites, headMeta.NumWrites) - return - } - if opMeta.InodeNumber != headMeta.InodeNumber { - t.Errorf("object '%s' op %s '%s' op InodeNumber '%d' does not match stat InodeNumber '%d'", - containerObjPath, opName, stepName, opMeta.InodeNumber, headMeta.InodeNumber) - return - } - - if opMeta.AttrChangeTime != headMeta.AttrChangeTime { - t.Errorf("object '%s' op %s '%s' op AttrChangeTime '%s' does not match stat AttrChangeTime '%s'", - containerObjPath, opName, stepName, - time.Unix(0, int64(opMeta.AttrChangeTime)).Format(timeFormat), - time.Unix(0, int64(headMeta.AttrChangeTime)).Format(timeFormat)) - return - } - if opMeta.ModificationTime != headMeta.ModificationTime { - t.Errorf("object '%s' op %s '%s' op ModificationTime '%s' does not match stat ModificationTime '%s'", - containerObjPath, opName, stepName, - time.Unix(0, int64(opMeta.ModificationTime)).Format(timeFormat), - time.Unix(0, int64(headMeta.ModificationTime)).Format(timeFormat)) - return - } - return -} - -// Test MiddlewareMkdir() and MiddlewarePutComplete(). -// -// A Swift PUT operation can create a file or a directory, depending on the -// arguments. Further, in Swift a PUT on an object deletes the object and -// replaces it with a new object with the new metadata. -// -// We interpret the "replacement rule" to mean that a PUT can delete a file and -// replace it with a directory or vice versa, but it cannot cause the delete of -// a directory that is not empty. (Its not clear how symlinks figure into this, -// but they should probably follow the same rule.) -// -// The code in pfs_middleware determines whether a PUT request intends to -// create a file or a directory. -// -// Test other behaviors, like automatically creating a path to the specified -// object. This does not test concatenating 1 or more objects to make up the -// contents of the file. -func TestMiddlewarePuts(t *testing.T) { - - testSetup(t, false) - defer testTeardown(t) - - initialMetadata := []byte("initial metadata") - updatedMetadata := []byte("updated metadata") - updatedMetadata2 := []byte("really new metadata") - containerName := "MiddlewarePuts" - objectPath := "dir0/dir1/dir2/file0" - containerObjectPath := containerName + "/" + objectPath - - var ( - opMeta HeadResponse - err error - ) - - // make a container for testing and verify the explicit metadata - err = testVolumeStruct.MiddlewarePutContainer(containerName, []byte(""), initialMetadata) - if err != nil { - t.Fatalf("MiddlewarePutContainer() failed: %v", err) - } - opMeta, err = testVolumeStruct.MiddlewareHeadResponse(containerName) - if err != nil { - t.Fatalf("MiddlewareHeadResponse() for container '%s' failed: %v", containerName, err) - } - if !bytes.Equal(opMeta.Metadata, initialMetadata) { - t.Errorf("MiddlewareHeadResponse() for container '%s' metadata '%s' does not match '%s'", - containerName, opMeta.Metadata, initialMetadata) - } - - // create a file object and then verify the explicit metadata and - // returned attributes are correct - opMeta.ModificationTime, opMeta.AttrChangeTime, opMeta.InodeNumber, opMeta.NumWrites, err = - testVolumeStruct.MiddlewarePutComplete(containerName, objectPath, nil, nil, initialMetadata) - if err != nil { - t.Errorf("MiddlewarePutComplete() for container '%s' object '%s' failed: %v", - containerName, objectPath, err) - } else { - opMeta.IsDir = false - opMeta.FileSize = 0 - opMeta.Metadata = initialMetadata - - verifyMetadata(t, containerObjectPath, "step 0", "MiddlewarePutComplete", &opMeta) - } - - // replace the file object with a directory object then verify the - // explicit metadata and returned attributes - opMeta.ModificationTime, opMeta.AttrChangeTime, opMeta.InodeNumber, opMeta.NumWrites, err = - testVolumeStruct.MiddlewareMkdir(containerName, objectPath, updatedMetadata) - if err != nil { - t.Errorf("MiddlewareMkdir() for container '%s' object '%s' failed: %v", - containerName, objectPath, err) - } else { - opMeta.IsDir = true - opMeta.FileSize = 0 - opMeta.Metadata = updatedMetadata - - verifyMetadata(t, containerObjectPath, "step 1", "MiddlewareMkdir", &opMeta) - } - - // verify the metadata (explicit and implicit) returned by - // MiddlewareGetObject() matches MiddlewareHeadResponse() for a - // directory - opMeta, err = testVolumeStruct.MiddlewareGetObject(containerObjectPath, - []ReadRangeIn{}, &[]inode.ReadPlanStep{}) - if err != nil { - t.Errorf("MiddlewareGetObject() for object '%s' failed: %v", containerObjectPath, err) - } else { - verifyMetadata(t, containerObjectPath, "step 1.5", "MiddlewareGetObject", &opMeta) - } - - // change the directory object back to a file object and verify the - // explicit metadata and returned attributes - opMeta.ModificationTime, opMeta.AttrChangeTime, opMeta.InodeNumber, opMeta.NumWrites, err = - testVolumeStruct.MiddlewarePutComplete(containerName, objectPath, nil, nil, updatedMetadata2) - if err != nil { - t.Errorf("MiddlewarePutComplete() for container '%s' object '%s' failed: %v", - containerName, objectPath, err) - } else { - opMeta.IsDir = false - opMeta.FileSize = 0 - opMeta.Metadata = updatedMetadata2 - - verifyMetadata(t, containerObjectPath, "step 2", "MiddlewarePutComplete", &opMeta) - } - - // verify the metadata (explicit and implicit) returned by - // MiddlewareGetObject() matches MiddlewareHeadResponse() - opMeta, err = testVolumeStruct.MiddlewareGetObject(containerObjectPath, - []ReadRangeIn{}, &[]inode.ReadPlanStep{}) - if err != nil { - t.Errorf("MiddlewareGetObject() for object '%s' failed: %v", containerObjectPath, err) - } else { - verifyMetadata(t, containerObjectPath, "step 3", "MiddlewareGetObject", &opMeta) - } - - // save the file's metadata for later validation - file0Meta := opMeta - - // a "file" PUT to a directory object that is not empty should fail - // (because we cannot convert a non-empty directory to a file object) - pathComponents := strings.Split(objectPath, "/") - dirPath := strings.Join(pathComponents[:len(pathComponents)-1], "/") - containerDirPath := containerName + "/" + dirPath - - opMeta.ModificationTime, opMeta.AttrChangeTime, opMeta.InodeNumber, opMeta.NumWrites, err = - testVolumeStruct.MiddlewarePutComplete(containerName, dirPath, nil, nil, initialMetadata) - if err == nil { - t.Errorf("MiddlewarePutComplete() for container '%s' non-empty object '%s' should have failed", - containerName, objectPath) - } else if blunder.IsNot(err, blunder.NotEmptyError) { - t.Errorf("MiddlewarePutComplete() for container '%s' non-empty object '%s' should have failed "+ - "with error '%s' but failed with error '%s' (%s)", - containerName, objectPath, - blunder.FsError(int(unix.ENOTEMPTY)), - blunder.FsError(blunder.Errno(err)), - blunder.ErrorString(err)) - } - - // a "directory PUT" to a directory object that is not empty should - // succeed and update the explicit metadata (but should not delete - // existing directory entries) - opMeta.ModificationTime, opMeta.AttrChangeTime, opMeta.InodeNumber, opMeta.NumWrites, err = - testVolumeStruct.MiddlewareMkdir(containerName, dirPath, updatedMetadata) - if err != nil { - t.Errorf("MiddlewareMkdir() for object '%s' failed: %v", containerDirPath, err) - } else { - opMeta.IsDir = true - opMeta.FileSize = 0 - opMeta.Metadata = updatedMetadata - - verifyMetadata(t, containerDirPath, "step 4", "MiddlewareMkdir", &opMeta) - } - - // verify that the file (child of the directory) is unchanged - verifyMetadata(t, containerObjectPath, "step 5", "verify", &file0Meta) -} diff --git a/fs/config.go b/fs/config.go deleted file mode 100644 index 32af7807..00000000 --- a/fs/config.go +++ /dev/null @@ -1,377 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fs - -import ( - "container/list" - "fmt" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -type inFlightFileInodeDataStruct struct { - inode.InodeNumber // Indicates the InodeNumber of a fileInode with unflushed - volStruct *volumeStruct // Synchronized via volStruct's sync.Mutex - control chan bool // Signal with true to flush (and exit), false to simply exit - wg sync.WaitGroup // Client can know when done - globalsListElement *list.Element // Back-pointer to wrapper used to insert into globals.inFlightFileInodeDataList -} - -// inFlightFileInodeDataControlBuffering specifies the inFlightFileInodeDataStruct.control channel buffer size -// Note: There are potentially multiple initiators of this signal -const inFlightFileInodeDataControlBuffering = 100 - -type volumeStruct struct { - dataMutex trackedlock.Mutex - volumeName string - doCheckpointPerFlush bool - maxFlushTime time.Duration - fileDefragmentChunkSize uint64 - fileDefragmentChunkDelay time.Duration - reportedBlockSize uint64 - reportedFragmentSize uint64 - reportedNumBlocks uint64 // Used for Total, Free, and Avail - reportedNumInodes uint64 // Used for Total, Free, and Avail - FLockMap map[inode.InodeNumber]*list.List - inFlightFileInodeDataMap map[inode.InodeNumber]*inFlightFileInodeDataStruct - jobRWMutex trackedlock.RWMutex - inodeVolumeHandle inode.VolumeHandle - headhunterVolumeHandle headhunter.VolumeHandle -} - -type tryLockBackoffContextStruct struct { - sync.WaitGroup - backoffsCompleted uint64 // Note that tryLockBackoffContextStruct{} sets this to zero -} - -type globalsStruct struct { - trackedlock.Mutex - - tryLockBackoffMin time.Duration - tryLockBackoffMax time.Duration - tryLockSerializationThreshhold uint64 - symlinkMax uint16 - coalesceElementChunkSize uint16 - - volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName - - inFlightFileInodeDataList *list.List - serializedBackoffList *list.List - - AccessUsec bucketstats.BucketLog2Round - CreateUsec bucketstats.BucketLog2Round - DestroyUsec bucketstats.BucketLog2Round - FlushUsec bucketstats.BucketLog2Round - FlockGetUsec bucketstats.BucketLog2Round - FlockLockUsec bucketstats.BucketLog2Round - FlockUnlockUsec bucketstats.BucketLog2Round - GetstatUsec bucketstats.BucketLog2Round - GetTypeUsec bucketstats.BucketLog2Round - GetXAttrUsec bucketstats.BucketLog2Round - IsDirUsec bucketstats.BucketLog2Round - IsFileUsec bucketstats.BucketLog2Round - IsSymlinkUsec bucketstats.BucketLog2Round - LinkUsec bucketstats.BucketLog2Round - ListXAttrUsec bucketstats.BucketLog2Round - LookupUsec bucketstats.BucketLog2Round - LookupPathUsec bucketstats.BucketLog2Round - MkdirUsec bucketstats.BucketLog2Round - MoveUsec bucketstats.BucketLog2Round - RemoveXAttrUsec bucketstats.BucketLog2Round - RenameUsec bucketstats.BucketLog2Round - ReadUsec bucketstats.BucketLog2Round - ReadBytes bucketstats.BucketLog2Round - ReaddirUsec bucketstats.BucketLog2Round - ReaddirEntries bucketstats.BucketLog2Round - ReaddirOneUsec bucketstats.BucketLog2Round - ReaddirOnePlusUsec bucketstats.BucketLog2Round - ReaddirPlusUsec bucketstats.BucketLog2Round - ReaddirPlusBytes bucketstats.BucketLog2Round - ReadsymlinkUsec bucketstats.BucketLog2Round - ResizeUsec bucketstats.BucketLog2Round - RmdirUsec bucketstats.BucketLog2Round - SetstatUsec bucketstats.BucketLog2Round - SetXAttrUsec bucketstats.BucketLog2Round - StatVfsUsec bucketstats.BucketLog2Round - SymlinkUsec bucketstats.BucketLog2Round - UnlinkUsec bucketstats.BucketLog2Round - VolumeNameUsec bucketstats.BucketLog2Round - WriteUsec bucketstats.BucketLog2Round - WriteBytes bucketstats.BucketLog2Round - - CreateErrors bucketstats.Total - DefragmentFileErrors bucketstats.Total - DestroyErrors bucketstats.Total - FetchExtentMapChunkErrors bucketstats.Total - FlushErrors bucketstats.Total - FlockOtherErrors bucketstats.Total - FlockGetErrors bucketstats.Total - FlockLockErrors bucketstats.Total - FlockUnlockErrors bucketstats.Total - GetstatErrors bucketstats.Total - GetTypeErrors bucketstats.Total - GetXAttrErrors bucketstats.Total - IsDirErrors bucketstats.Total - IsFileErrors bucketstats.Total - IsSymlinkErrors bucketstats.Total - LinkErrors bucketstats.Total - ListXAttrErrors bucketstats.Total - LookupErrors bucketstats.Total - LookupPathErrors bucketstats.Total - MkdirErrors bucketstats.Total - MoveErrors bucketstats.Total - RemoveXAttrErrors bucketstats.Total - RenameErrors bucketstats.Total - ReadErrors bucketstats.Total - ReaddirErrors bucketstats.Total - ReaddirOneErrors bucketstats.Total - ReaddirOnePlusErrors bucketstats.Total - ReaddirPlusErrors bucketstats.Total - ReadsymlinkErrors bucketstats.Total - ResizeErrors bucketstats.Total - RmdirErrors bucketstats.Total - SetstatErrors bucketstats.Total - SetXAttrErrors bucketstats.Total - StatVfsErrors bucketstats.Total - SymlinkErrors bucketstats.Total - UnlinkErrors bucketstats.Total - WriteErrors bucketstats.Total - - DefragmentFileUsec bucketstats.BucketLog2Round - FetchExtentMapChunkUsec bucketstats.BucketLog2Round - CallInodeToProvisionObjectUsec bucketstats.BucketLog2Round - MiddlewareCoalesceUsec bucketstats.BucketLog2Round - MiddlewareCoalesceBytes bucketstats.BucketLog2Round - MiddlewareDeleteUsec bucketstats.BucketLog2Round - MiddlewareGetAccountUsec bucketstats.BucketLog2Round - MiddlewareGetContainerUsec bucketstats.BucketLog2Round - MiddlewareGetObjectUsec bucketstats.BucketLog2Round - MiddlewareGetObjectBytes bucketstats.BucketLog2Round - MiddlewareHeadResponseUsec bucketstats.BucketLog2Round - MiddlewareMkdirUsec bucketstats.BucketLog2Round - MiddlewarePostUsec bucketstats.BucketLog2Round - MiddlewarePostBytes bucketstats.BucketLog2Round - MiddlewarePutCompleteUsec bucketstats.BucketLog2Round - MiddlewarePutCompleteBytes bucketstats.BucketLog2Round - MiddlewarePutContainerUsec bucketstats.BucketLog2Round - MiddlewarePutContainerBytes bucketstats.BucketLog2Round - - CallInodeToProvisionObjectErrors bucketstats.Total - MiddlewareCoalesceErrors bucketstats.Total - MiddlewareDeleteErrors bucketstats.Total - MiddlewareGetAccountErrors bucketstats.Total - MiddlewareGetContainerErrors bucketstats.Total - MiddlewareGetObjectErrors bucketstats.Total - MiddlewareHeadResponseErrors bucketstats.Total - MiddlewareMkdirErrors bucketstats.Total - MiddlewarePostErrors bucketstats.Total - MiddlewarePutCompleteErrors bucketstats.Total - MiddlewarePutContainerErrors bucketstats.Total - - FetchVolumeHandleUsec bucketstats.BucketLog2Round - FetchVolumeHandleErrors bucketstats.BucketLog2Round - ValidateVolumeUsec bucketstats.BucketLog2Round - ScrubVolumeUsec bucketstats.BucketLog2Round - ValidateBaseNameUsec bucketstats.BucketLog2Round - ValidateBaseNameErrors bucketstats.Total - ValidateFullPathUsec bucketstats.BucketLog2Round - ValidateFullPathErrors bucketstats.Total - AccountNameToVolumeNameUsec bucketstats.BucketLog2Round - VolumeNameToActivePeerPrivateIPAddrUsec bucketstats.BucketLog2Round -} - -var globals globalsStruct - -func init() { - transitions.Register("fs", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - globals.tryLockBackoffMin, err = confMap.FetchOptionValueDuration("FSGlobals", "TryLockBackoffMin") - if nil != err { - globals.tryLockBackoffMin = time.Duration(10 * time.Millisecond) // TODO: Eventually, just return - } - globals.tryLockBackoffMax, err = confMap.FetchOptionValueDuration("FSGlobals", "TryLockBackoffMax") - if nil != err { - globals.tryLockBackoffMax = time.Duration(50 * time.Millisecond) // TODO: Eventually, just return - } - globals.tryLockSerializationThreshhold, err = confMap.FetchOptionValueUint64("FSGlobals", "TryLockSerializationThreshhold") - if nil != err { - globals.tryLockSerializationThreshhold = 5 // TODO: Eventually, just return - } - globals.symlinkMax, err = confMap.FetchOptionValueUint16("FSGlobals", "SymlinkMax") - if nil != err { - globals.symlinkMax = 32 // TODO: Eventually, just return - } - globals.coalesceElementChunkSize, err = confMap.FetchOptionValueUint16("FSGlobals", "CoalesceElementChunkSize") - if nil != err { - globals.coalesceElementChunkSize = 16 // TODO: Eventually, just return - } - - globals.volumeMap = make(map[string]*volumeStruct) - - globals.inFlightFileInodeDataList = list.New() - globals.serializedBackoffList = list.New() - - bucketstats.Register("proxyfs.fs", "", &globals) - - err = nil - return nil -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} - -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - replayLogFileName string - volume *volumeStruct - volumeSectionName string - ) - - volume = &volumeStruct{ - volumeName: volumeName, - FLockMap: make(map[inode.InodeNumber]*list.List), - inFlightFileInodeDataMap: make(map[inode.InodeNumber]*inFlightFileInodeDataStruct), - } - - volumeSectionName = "Volume:" + volumeName - - replayLogFileName, err = confMap.FetchOptionValueString(volumeSectionName, "ReplayLogFileName") - if nil == err { - volume.doCheckpointPerFlush = ("" == replayLogFileName) - } else { - volume.doCheckpointPerFlush = true - } - - logger.Infof("Checkpoint per Flush for volume %v is %v", volume.volumeName, volume.doCheckpointPerFlush) - - volume.maxFlushTime, err = confMap.FetchOptionValueDuration(volumeSectionName, "MaxFlushTime") - if nil != err { - return - } - - volume.fileDefragmentChunkSize, err = confMap.FetchOptionValueUint64(volumeSectionName, "FileDefragmentChunkSize") - if nil != err { - volume.fileDefragmentChunkSize = 10485760 // TODO: Eventually, just return - } - volume.fileDefragmentChunkDelay, err = confMap.FetchOptionValueDuration(volumeSectionName, "FileDefragmentChunkDelay") - if nil != err { - volume.fileDefragmentChunkDelay = time.Duration(10 * time.Millisecond) // TODO: Eventually, just return - } - - volume.reportedBlockSize, err = confMap.FetchOptionValueUint64(volumeSectionName, "ReportedBlockSize") - if nil != err { - volume.reportedBlockSize = DefaultReportedBlockSize // TODO: Eventually, just return - } - volume.reportedFragmentSize, err = confMap.FetchOptionValueUint64(volumeSectionName, "ReportedFragmentSize") - if nil != err { - volume.reportedFragmentSize = DefaultReportedFragmentSize // TODO: Eventually, just return - } - volume.reportedNumBlocks, err = confMap.FetchOptionValueUint64(volumeSectionName, "ReportedNumBlocks") - if nil != err { - volume.reportedNumBlocks = DefaultReportedNumBlocks // TODO: Eventually, just return - } - volume.reportedNumInodes, err = confMap.FetchOptionValueUint64(volumeSectionName, "ReportedNumInodes") - if nil != err { - volume.reportedNumInodes = DefaultReportedNumInodes // TODO: Eventually, just return - } - - volume.inodeVolumeHandle, err = inode.FetchVolumeHandle(volumeName) - if nil != err { - return - } - volume.headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(volumeName) - if nil != err { - return - } - - globals.volumeMap[volumeName] = volume - - err = nil - return -} - -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - ok bool - volume *volumeStruct - ) - - volume, ok = globals.volumeMap[volumeName] - - if !ok { - err = nil - return - } - - volume.untrackInFlightFileInodeDataAll() - - delete(globals.volumeMap, volumeName) - - err = nil - return -} - -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - return nil -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - var ( - volume *volumeStruct - ) - - if 0 != len(globals.volumeMap) { - err = fmt.Errorf("fs.Down() called with 0 != len(globals.volumeMap") - return - } - if 0 != globals.inFlightFileInodeDataList.Len() { - err = fmt.Errorf("fs.Down() called with 0 != globals.inFlightFileInodeDataList.Len()") - return - } - - for _, volume = range globals.volumeMap { - volume.untrackInFlightFileInodeDataAll() - } - - if 0 < globals.inFlightFileInodeDataList.Len() { - logger.Fatalf("fs.Down() has completed all un-mount's... but found non-empty globals.inFlightFileInodeDataList") - } - - bucketstats.UnRegister("proxyfs.fs", "") - - err = nil - return -} diff --git a/fs/metadata_stress_test.go b/fs/metadata_stress_test.go deleted file mode 100644 index a4e9ee26..00000000 --- a/fs/metadata_stress_test.go +++ /dev/null @@ -1,709 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fs - -import ( - "strconv" - "sync" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/utils" -) - -// -// Code related to multiple test threads -// - -const testDirName = "MetaDataStressTestDir" - -var testDirInodeNumber inode.InodeNumber - -func testSetupForStress(t *testing.T, starvationMode bool) { - var err error - testSetup(t, starvationMode) - testDirInodeNumber, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, testDirName, inode.PosixModePerm) - if nil != err { - t.Fatalf("Failed to create %s: %v", testDirName, err) - } -} - -// -// Metadata stress tests -// - -func TestStressMetaDataOpsWhileStarved(t *testing.T) { - testStressMetaDataOps(t, true) -} - -func TestStressMetaDataOpsWhileNotStarved(t *testing.T) { - testStressMetaDataOps(t, false) -} - -func testStressMetaDataOps(t *testing.T, starvationMode bool) { - if testing.Short() { - t.Skip("skipping stress test.") - } - - globalSyncPt = make(chan testRequest) - - testSetupForStress(t, starvationMode) - testTwoThreadsCreateUnlink(t) - testTeardown(t) - testSetupForStress(t, starvationMode) - testTwoThreadsCreateCreate(t) - testTeardown(t) - testSetupForStress(t, starvationMode) - testTeardown(t) - testSetupForStress(t, starvationMode) - testMultiThreadCreateAndLookup(t) - testTeardown(t) - testSetupForStress(t, starvationMode) - testMultiThreadCreateAndReaddir(t) - testTeardown(t) - testSetupForStress(t, starvationMode) - testCreateReWriteNoFlush(t) - testTeardown(t) - testSetupForStress(t, starvationMode) - testCreateSeqWriteNoFlush(t) - testTeardown(t) -} - -type testOpTyp int - -const ( - nilTestOp testOpTyp = iota - createTestOp - createLoopTestOp - lookupPathLoopTestOp - mkdirTestOp - readdirLoopTestOp - getContainerLoopTestOp - rmdirTestOp - stopThreadTestOp - unlinkTestOp - unlinkLoopTestOp - reWriteNoFlushLoopTestOp - seqWriteNoFlushLoopTestOp -) - -type testRequest struct { - opType testOpTyp // Operation type - name1 string - loopCount int // Number of times to do operation. 0 means enforce min/max - minimumLoopCount int // Minimum number of times to do infinite operation. - maximumLoopCount int // Maximum number of times to do infinite operation. - loopDelay time.Duration - inodeNumber inode.InodeNumber - bufPtr *[]byte - offset uint64 - length uint64 - t *testing.T -} - -type testResponse struct { - err error - inodeNumber inode.InodeNumber -} - -// Per thread structure storing channel information -type threadInfo struct { - sync.Mutex - startedNode chan bool - requestForThread chan *testRequest - operationStatus chan *testResponse - endLoop bool // Flag used to signal an infinite loop test to stop -} - -var globalSyncPt chan testRequest // Channel used to synchronize test threads to simulate multiple threads - -// Map of threads and channels used for communication -var threadMap map[int]*threadInfo - -// Setup thread stuctures based on number of threads test wants -func setupThreadMap(threadCount int) { - threadMap = make(map[int]*threadInfo) - - for i := 0; i < threadCount; i++ { - thread := &threadInfo{startedNode: make(chan bool), requestForThread: make(chan *testRequest), operationStatus: make(chan *testResponse)} - threadMap[i] = thread - } -} - -func setupThreads(threadCount int) { - setupThreadMap(threadCount) - - // Start threads and wait for them - for i := range threadMap { - go threadNode(i) - _ = <-threadMap[i].startedNode - } -} - -func stopThreads(t *testing.T) { - for i := range threadMap { - - // Tell thread to exit - request := &testRequest{opType: stopThreadTestOp, t: t, inodeNumber: testDirInodeNumber} - sendRequestToThread(i, t, request) - } -} - -func loopOp(fileRequest *testRequest, threadID int, inodeNumber inode.InodeNumber) (err error) { - var ( - areMoreEntries bool - containerEnts []ContainerEntry - dirEnts []inode.DirEntry - fName string - infiniteLoopCount int // Useful for debugging - lastBasename string - localLoopCount int - loopCount int - loopDelay time.Duration - maxEntries uint64 - minimumLoopCount int - maximumLoopCount int - more bool - name1 string - numEntries uint64 - offset uint64 - totalEntriesRead uint64 - ) - - name1 = fileRequest.name1 - loopCount = fileRequest.loopCount - minimumLoopCount = fileRequest.minimumLoopCount - maximumLoopCount = fileRequest.maximumLoopCount - loopDelay = fileRequest.loopDelay - - // Loop doing operation loopCount times. If it is an infinite loop we loop until signaled to stop. - // - // minimumLoopCount is used with infiniteLoop to make sure the loop executes at least minimumLoopCount times - // before returning. - for { - fName = name1 + "-" + strconv.Itoa(localLoopCount) - switch fileRequest.opType { - case createLoopTestOp: - _, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, fName, inode.PosixModePerm) - case lookupPathLoopTestOp: - _, err = testVolumeStruct.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fName) - case readdirLoopTestOp: - areMoreEntries = true - lastBasename = "" - maxEntries = 10 - totalEntriesRead = 0 // Useful for debugging - for areMoreEntries { - dirEnts, numEntries, more, err = testVolumeStruct.Readdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, maxEntries, lastBasename) - if nil != err { - return - } - lastBasename = dirEnts[len(dirEnts)-1].Basename - areMoreEntries = more - totalEntriesRead += numEntries - } - case getContainerLoopTestOp: - areMoreEntries = true - lastBasename = "" - maxEntries = 10 - totalEntriesRead = 0 // Useful for debugging - for areMoreEntries { - containerEnts, err = testVolumeStruct.MiddlewareGetContainer(testDirName, maxEntries, lastBasename, "", "", "") - if nil != err { - return - } - if 0 == len(containerEnts) { - areMoreEntries = false - } else { - lastBasename = containerEnts[len(containerEnts)-1].Basename - totalEntriesRead += uint64(len(containerEnts)) - } - } - case unlinkLoopTestOp: - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, fName) - case reWriteNoFlushLoopTestOp: - _, _ = testVolumeStruct.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, fileRequest.offset, *fileRequest.bufPtr, nil) - case seqWriteNoFlushLoopTestOp: - offset = fileRequest.length * uint64(localLoopCount) - _, _ = testVolumeStruct.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, offset, *fileRequest.bufPtr, nil) - } - localLoopCount++ - infiniteLoopCount++ - - // The infinite loop case breaks when control thread signals this thread to stop - // and we have at least hit our minimumLoopCount. - if 0 == loopCount { - if 0 == maximumLoopCount { - if localLoopCount >= minimumLoopCount { - threadMap[threadID].Lock() - if threadMap[threadID].endLoop == true { - threadMap[threadID].Unlock() - break - } - threadMap[threadID].Unlock() - } - } else { - if localLoopCount == maximumLoopCount { - break - } else { - if localLoopCount >= minimumLoopCount { - threadMap[threadID].Lock() - if threadMap[threadID].endLoop == true { - threadMap[threadID].Unlock() - break - } - threadMap[threadID].Unlock() - } - } - } - } else { - if localLoopCount == loopCount { - break - } - } - - time.Sleep(loopDelay) - } - - return -} - -// Test thread. Just waits on channel and does operation requested. -func threadNode(threadID int) { - - // Tell control thread we are up and set channel to read. - threadMap[threadID].startedNode <- true - var request chan *testRequest - request = threadMap[threadID].requestForThread - - // Wait for an operation - for { - fileRequest := <-request - name1 := fileRequest.name1 - inodeNumber := fileRequest.inodeNumber - - switch fileRequest.opType { - case stopThreadTestOp: - return - - case createTestOp: - response := &testResponse{} - response.inodeNumber, response.err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, - name1, inode.PosixModePerm) - threadMap[threadID].operationStatus <- response - - case createLoopTestOp: - // Loop creating files loopCount times. - err := loopOp(fileRequest, threadID, inodeNumber) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - - case lookupPathLoopTestOp: - // Loop doing LookupPath of files loopCount times. - err := loopOp(fileRequest, threadID, inodeNumber) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - - case mkdirTestOp: - newInodeNumber, err := testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, name1, inode.PosixModePerm) - response := &testResponse{err: err, inodeNumber: newInodeNumber} - threadMap[threadID].operationStatus <- response - - case readdirLoopTestOp: - // Loop doing readdir of files loopCount times. - err := loopOp(fileRequest, threadID, inodeNumber) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - - case getContainerLoopTestOp: - // Loop doing MiddlewareGetContainer of objects loopCount times. - err := loopOp(fileRequest, threadID, inodeNumber) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - - case rmdirTestOp: - err := testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, name1) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - - case unlinkTestOp: - err := testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inodeNumber, name1) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - - case unlinkLoopTestOp: - // Loop unlinking files loopCount times. - err := loopOp(fileRequest, threadID, inodeNumber) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - - case reWriteNoFlushLoopTestOp: - // Loop writing and rewriting a file loopCount times. - err := loopOp(fileRequest, threadID, inodeNumber) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - - case seqWriteNoFlushLoopTestOp: - // Loop writing and rewriting a file loopCount times. - err := loopOp(fileRequest, threadID, inodeNumber) - response := &testResponse{err: err} - threadMap[threadID].operationStatus <- response - } - } -} - -// Set flag telling thread doing infinite loop to exit. -func setEndLoopFlag(threadID int) { - threadMap[threadID].Lock() - threadMap[threadID].endLoop = true - threadMap[threadID].Unlock() -} - -func sendRequestToThread(threadID int, t *testing.T, request *testRequest) { - // Clear endLoop flag before sending request - threadMap[threadID].Lock() - threadMap[threadID].endLoop = false - threadMap[threadID].Unlock() - - threadMap[threadID].requestForThread <- request - - // We do not wait until the operation completes before returning. -} - -// Test that two threads can grab a lock *exclusive* and the second thread -// only gets lock after first one has done Unlock(). -func testTwoThreadsCreateUnlink(t *testing.T) { - var numThreads = 2 - - // Initialize worker threads - setupThreads(numThreads) - - // Tell thread 0 to loop creating files of the pattern "testfile*" - request := &testRequest{opType: createLoopTestOp, t: t, name1: "testfile", - inodeNumber: testDirInodeNumber} - sendRequestToThread(0, t, request) - - // Create the file from thread 1 - request = &testRequest{opType: createTestOp, t: t, name1: "TestNormalFile", - inodeNumber: testDirInodeNumber} - sendRequestToThread(1, t, request) - _ = <-threadMap[1].operationStatus - - // Unlink the file from thread 1 - request = &testRequest{opType: unlinkTestOp, t: t, name1: "TestNormalFile", - inodeNumber: testDirInodeNumber} - sendRequestToThread(1, t, request) - _ = <-threadMap[1].operationStatus - - // Tell thread 0 to stop creating files - setEndLoopFlag(0) - _ = <-threadMap[0].operationStatus - - // Stop worker threads - stopThreads(t) -} - -// Test that two threads can grab a lock *exclusive* and the second thread -// only gets lock after first one has done Unlock(). -func testTwoThreadsCreateCreate(t *testing.T) { - var numThreads = 2 - - // Initialize worker threads - setupThreads(numThreads) - - for i := 0; i < numThreads; i++ { - // Tell thread 0 to loop creating files of the pattern "testfile*" - request := &testRequest{opType: createLoopTestOp, t: t, name1: "testfile-" + strconv.Itoa(i), - inodeNumber: testDirInodeNumber} - sendRequestToThread(i, t, request) - } - - time.Sleep(100 * time.Millisecond) - - // Tell threads to stop creating files - for i := 0; i < numThreads; i++ { - setEndLoopFlag(i) - _ = <-threadMap[i].operationStatus - } - - // Stop worker threads - stopThreads(t) -} - -// Test that two threads can grab a lock *exclusive* and the second thread -// only gets lock after first one has done Unlock(). -func testMultiThreadCreate(t *testing.T) { - var numThreads = 3 - nameOfTest := utils.GetFnName() - - // Initialize worker threads - setupThreads(numThreads) - - // Unlink existing files - for i := 0; i < numThreads; i++ { - request := &testRequest{opType: unlinkLoopTestOp, t: t, name1: nameOfTest + "-" + strconv.Itoa(i), - inodeNumber: testDirInodeNumber, loopCount: 5} - sendRequestToThread(i, t, request) - } - // Wait for unlinkLoopTestOp to complete - for i := 0; i < numThreads; i++ { - _ = <-threadMap[i].operationStatus - } - - // Create files - for i := 0; i < numThreads; i++ { - request := &testRequest{opType: createLoopTestOp, t: t, name1: nameOfTest + "-" + strconv.Itoa(i), - inodeNumber: testDirInodeNumber, loopCount: 5} - sendRequestToThread(i, t, request) - } - // Wait for createLoopTestOp to complete - for i := 0; i < numThreads; i++ { - _ = <-threadMap[i].operationStatus - } - - // Now unlink the files - for i := 0; i < numThreads; i++ { - request := &testRequest{opType: unlinkLoopTestOp, t: t, name1: nameOfTest + "-" + strconv.Itoa(i), - inodeNumber: testDirInodeNumber, loopCount: 5} - sendRequestToThread(i, t, request) - } - // Wait for unlinkLoopTestOp to complete - for i := 0; i < numThreads; i++ { - _ = <-threadMap[i].operationStatus - } - - // Stop worker threads - stopThreads(t) -} - -// Test one thread doing Create() in loop and two threads -// doing Lookup() -func testMultiThreadCreateAndLookup(t *testing.T) { - var numThreads = 3 - nameOfTest := utils.GetFnName() - - // Initialize worker threads - setupThreads(numThreads) - - // Create a subdirectory to use - request1 := &testRequest{opType: mkdirTestOp, t: t, name1: nameOfTest + "-subdir", - inodeNumber: testDirInodeNumber} - sendRequestToThread(0, t, request1) - mkdirResponse := <-threadMap[0].operationStatus - - // Tell thread 0 to loop creating files of the pattern nameOfTest - request2 := &testRequest{opType: createLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, loopCount: 10} - sendRequestToThread(0, t, request2) - - // Tell thread 1 to loop doing 35 Lookups - request3 := &testRequest{opType: lookupPathLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, loopCount: 35} - sendRequestToThread(1, t, request3) - - // Tell thread 2 to loop doing 35 Lookups - request4 := &testRequest{opType: lookupPathLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, loopCount: 35} - sendRequestToThread(2, t, request4) - - // Wait for threads to complete - for i := 0; i < numThreads; i++ { - _ = <-threadMap[i].operationStatus - } - - // Tell thread 0 to loop unlinking test files created during test - // and wait for it to complete - request5 := &testRequest{opType: unlinkLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, loopCount: 10} - sendRequestToThread(0, t, request5) - _ = <-threadMap[0].operationStatus - - // Remove subdirectory - request6 := &testRequest{opType: rmdirTestOp, t: t, name1: nameOfTest + "-subdir", - inodeNumber: testDirInodeNumber} - sendRequestToThread(0, t, request6) - _ = <-threadMap[0].operationStatus - - // Stop worker threads - stopThreads(t) -} - -// Test one thread doing Create() in loop and nine other threads doing Readdir -func testMultiThreadCreateAndReaddir(t *testing.T) { - var numThreads = 10 - nameOfTest := utils.GetFnName() - - // Initialize worker threads - setupThreads(numThreads) - - // Create a subdirectory to use - request1 := &testRequest{opType: mkdirTestOp, t: t, name1: nameOfTest + "-subdir", - inodeNumber: testDirInodeNumber} - sendRequestToThread(0, t, request1) - mkdirResponse := <-threadMap[0].operationStatus - - // Tell thread 0 to loop creating files of the pattern nameOfTest in the subdirectory. - // Create a minimum of at least 200 before stopping. - request2 := &testRequest{opType: createLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, minimumLoopCount: 200, maximumLoopCount: 400, loopDelay: 5 * time.Microsecond} - sendRequestToThread(0, t, request2) - - // Pause a few milliseconds between operations - time.Sleep(10 * time.Millisecond) - - // Tell threads 1 to numThreads to loop doing 35 readdirs - for i := 1; i < numThreads; i++ { - request3 := &testRequest{opType: readdirLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, loopCount: 35} - sendRequestToThread(i, t, request3) - } - - // Wait until threads 1 to numThreads complete - for i := 1; i < numThreads; i++ { - _ = <-threadMap[i].operationStatus - } - - // Tell threads 1 to numThreads to loop doing 15 getContainers - for i := 1; i < numThreads; i++ { - request3 := &testRequest{opType: getContainerLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, loopCount: 15} - sendRequestToThread(i, t, request3) - } - - // Wait until threads 1 to numThreads complete - for i := 1; i < numThreads; i++ { - _ = <-threadMap[i].operationStatus - } - - // Tell thread 0 to stop doing Creates in an infinite loop - setEndLoopFlag(0) - - // Wait for thread 0 to complete - _ = <-threadMap[0].operationStatus - - // Now tell thread 1 to do one more readdirLoopTestOp to make sure we can read 200 entries - request4 := &testRequest{opType: readdirLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, loopCount: 1} - sendRequestToThread(1, t, request4) - _ = <-threadMap[1].operationStatus - - // Now tell thread 1 to do one more getContainerLoopTestOp to make sure we can read 200 entries - request5 := &testRequest{opType: getContainerLoopTestOp, t: t, name1: nameOfTest, - inodeNumber: mkdirResponse.inodeNumber, loopCount: 1} - sendRequestToThread(1, t, request5) - _ = <-threadMap[1].operationStatus - - // Stop worker threads - stopThreads(t) -} - -// Test numThreads doing create(), loop doing rewrites() of same offset and location and no flush -func testCreateReWriteNoFlush(t *testing.T) { - // NOTE: This test uses a lot of memory and will cause a OOM. Be careful - // increasing numThreads, size of write buffer and number of overwrites. - var numThreads = 50 - - fileInodes := make([]inode.InodeNumber, numThreads) // Map to store each inode created - nameOfTest := utils.GetFnName() - - // Initialize worker threads - setupThreads(numThreads) - - // Create a subdirectory to use - request4 := &testRequest{opType: mkdirTestOp, t: t, name1: nameOfTest + "-subdir", - inodeNumber: testDirInodeNumber} - sendRequestToThread(0, t, request4) - mkdirResponse := <-threadMap[0].operationStatus - - // Create files used for writes - for i := 0; i < numThreads; i++ { - request5 := &testRequest{opType: createTestOp, t: t, name1: nameOfTest + "-" + strconv.Itoa(i), - inodeNumber: mkdirResponse.inodeNumber, loopCount: 5} - sendRequestToThread(i, t, request5) - } - // Wait for createTestOp to complete and store inode number created - for i := 0; i < numThreads; i++ { - response := <-threadMap[i].operationStatus - fileInodes[i] = response.inodeNumber - } - - var bufLen uint64 = 11 * 1024 * 1024 - bufToWrite := make([]byte, bufLen, bufLen) - - // Write to files without doing a flush. We write 11MB starting from offset 0. - // We rewrite the same location numOverWrites times. - numOverWrites := 1 - minNumberOfLoops := 1 - writeOffset := uint64(0) - for i := 0; i < numThreads; i++ { - request6 := &testRequest{opType: reWriteNoFlushLoopTestOp, t: t, name1: nameOfTest + "-" + strconv.Itoa(i), - inodeNumber: fileInodes[i], loopCount: numOverWrites, minimumLoopCount: minNumberOfLoops, - offset: writeOffset, length: bufLen, bufPtr: &bufToWrite} - sendRequestToThread(i, t, request6) - } - - // Wait until threads complete - for i := 0; i < numThreads; i++ { - _ = <-threadMap[i].operationStatus - } - - // Stop worker threads - stopThreads(t) -} - -// Test numThreads doing create, loop doing sequential 1MB writes and no flush. -// NOTE: The writes are not ordered - they may not actually happen sequentially. -func testCreateSeqWriteNoFlush(t *testing.T) { - // NOTE: This test uses a lot of memory and will cause a OOM. Be careful - // increasing numThreads, size of write buffer and number of overwrites. - var numThreads = 25 - - fileInodes := make([]inode.InodeNumber, numThreads) // Map to store each inode created - nameOfTest := utils.GetFnName() - - // Initialize worker threads - setupThreads(numThreads) - - // Create a subdirectory to use - request4 := &testRequest{opType: mkdirTestOp, t: t, name1: nameOfTest + "-subdir", - inodeNumber: testDirInodeNumber} - sendRequestToThread(0, t, request4) - mkdirResponse := <-threadMap[0].operationStatus - - // Create files used for writes - for i := 0; i < numThreads; i++ { - request5 := &testRequest{opType: createTestOp, t: t, name1: nameOfTest + "-" + strconv.Itoa(i), - inodeNumber: mkdirResponse.inodeNumber, loopCount: 11} - sendRequestToThread(i, t, request5) - } - // Wait for createTestOp to complete and store inode number created - for i := 0; i < numThreads; i++ { - response := <-threadMap[i].operationStatus - fileInodes[i] = response.inodeNumber - } - - // Each write will be 1MB - var bufLen uint64 = 1 * 1024 * 1024 - bufToWrite := make([]byte, bufLen, bufLen) - - // Write to files without doing a flush. We issue 1MB writes sequentially - // although they can finish in any order. - numOfWrites := 11 - minNumberOfLoops := 11 - writeOffset := uint64(0) - for i := 0; i < numThreads; i++ { - request6 := &testRequest{opType: seqWriteNoFlushLoopTestOp, t: t, name1: nameOfTest + "-" + strconv.Itoa(i), - inodeNumber: fileInodes[i], loopCount: numOfWrites, minimumLoopCount: minNumberOfLoops, - offset: writeOffset, length: bufLen, bufPtr: &bufToWrite} - sendRequestToThread(i, t, request6) - } - - // Wait until threads complete - for i := 0; i < numThreads; i++ { - _ = <-threadMap[i].operationStatus - } - - // Stop worker threads - stopThreads(t) -} diff --git a/fs/resolve_path.go b/fs/resolve_path.go deleted file mode 100644 index 0de667c0..00000000 --- a/fs/resolve_path.go +++ /dev/null @@ -1,1023 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fs - -import ( - "fmt" - "strings" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/dlm" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/utils" -) - -type resolvePathOption uint32 - -const ( - resolvePathFollowDirEntrySymlinks resolvePathOption = 1 << iota - resolvePathFollowDirSymlinks // Follow symlink in final pathanme component - resolvePathCreateMissingPathElements // Defaults created DirEntry to be a File - resolvePathDirEntryInodeMustBeDirectory // - resolvePathDirEntryInodeMustBeFile // - resolvePathDirEntryInodeMustBeSymlink // - resolvePathRequireExclusiveLockOnDirEntryInode // - resolvePathRequireExclusiveLockOnDirInode // Presumably only useful if resolvePathRequireExclusiveLockOnDirEntryInode also specified - resolvePathRequireSharedLockOnDirInode // Not valid if resolvePathRequireExclusiveLockOnDirInode also specified -) - -func resolvePathOptionsCheck(optionsRequested resolvePathOption, optionsToCheckFor resolvePathOption) ( - optionsRequestedIncludesCheckFor bool) { - - optionsRequestedIncludesCheckFor = (optionsToCheckFor == (optionsToCheckFor & optionsRequested)) - return -} - -type heldLocksStruct struct { - exclusive map[inode.InodeNumber]*dlm.RWLockStruct - shared map[inode.InodeNumber]*dlm.RWLockStruct -} - -func newHeldLocks() (heldLocks *heldLocksStruct) { - heldLocks = &heldLocksStruct{ - exclusive: make(map[inode.InodeNumber]*dlm.RWLockStruct), - shared: make(map[inode.InodeNumber]*dlm.RWLockStruct), - } - return -} - -func (heldLocks *heldLocksStruct) attemptExclusiveLock(inodeVolumeHandle inode.VolumeHandle, dlmCallerID dlm.CallerID, inodeNumber inode.InodeNumber) (retryRequired bool) { - var ( - err error - inodeLock *dlm.RWLockStruct - ok bool - ) - - _, ok = heldLocks.exclusive[inodeNumber] - if ok { - retryRequired = false - return - } - - inodeLock, ok = heldLocks.shared[inodeNumber] - if ok { - err = inodeLock.Unlock() - if nil != err { - logger.Fatalf("Failure unlocking a held LockID %s: %v", inodeLock.LockID, err) - } - - delete(heldLocks.shared, inodeNumber) - } - - inodeLock, err = inodeVolumeHandle.AttemptWriteLock(inodeNumber, dlmCallerID) - if nil != err { - retryRequired = true - return - } - - heldLocks.exclusive[inodeNumber] = inodeLock - - retryRequired = false - return -} - -func (heldLocks *heldLocksStruct) attemptSharedLock(inodeVolumeHandle inode.VolumeHandle, dlmCallerID dlm.CallerID, inodeNumber inode.InodeNumber) (retryRequired bool) { - var ( - err error - inodeLock *dlm.RWLockStruct - ok bool - ) - - _, ok = heldLocks.exclusive[inodeNumber] - if ok { - retryRequired = false - return - } - _, ok = heldLocks.shared[inodeNumber] - if ok { - retryRequired = false - return - } - - inodeLock, err = inodeVolumeHandle.AttemptReadLock(inodeNumber, dlmCallerID) - if nil != err { - retryRequired = true - return - } - - heldLocks.shared[inodeNumber] = inodeLock - - retryRequired = false - return -} - -func (heldLocks *heldLocksStruct) unlock(inodeNumber inode.InodeNumber) { - var ( - err error - heldLock *dlm.RWLockStruct - ok bool - ) - - heldLock, ok = heldLocks.exclusive[inodeNumber] - if ok { - err = heldLock.Unlock() - if nil != err { - logger.Fatalf("Failure unlocking a held LockID %s: %v", heldLock.LockID, err) - } - delete(heldLocks.exclusive, inodeNumber) - return - } - - heldLock, ok = heldLocks.shared[inodeNumber] - if ok { - err = heldLock.Unlock() - if nil != err { - logger.Fatalf("Failure unlocking a held LockID %s: %v", heldLock.LockID, err) - } - delete(heldLocks.shared, inodeNumber) - return - } - - logger.Fatalf("Attempt to unlock a non-held Lock on inodeNumber 0x%016X", inodeNumber) -} - -func (heldLocks *heldLocksStruct) free() { - var ( - err error - heldLock *dlm.RWLockStruct - ) - - for _, heldLock = range heldLocks.exclusive { - err = heldLock.Unlock() - if nil != err { - logger.Fatalf("Failure unlocking a held LockID %s: %v", heldLock.LockID, err) - } - } - - for _, heldLock = range heldLocks.shared { - err = heldLock.Unlock() - if nil != err { - logger.Fatalf("Failure unlocking a held LockID %s: %v", heldLock.LockID, err) - } - } -} - -// resolvePath is used to walk the supplied path starting at the supplied DirInode and locate -// the "leaf" {Dir|File}Inode. Various options allow for traversing encountered SymlinkInodes as -// well as optionally creating missing Inodes along the way. The caller is allowed to hold locks -// at entry that will be used (and possibly upgraded from "shared" to "exclusive") as well as -// extended to include any additional locks obtained. All lock requests are attempted so as to -// always avoid deadlock and if attempts fail, retryRequired == TRUE will be returned and the -// caller will be responsible for releasing all held locks, backing off (via a delay), before -// restarting the sequence. -// -// The pattern should look something like this: -// -// tryLockBackoffContext = &tryLockBackoffContextStruct{} -// -// Restart: -// -// tryLockBackoffContext.backoff() -// -// heldLocks = newHeldLocks() -// -// dirInode, dirEntryInode, retryRequired, err = -// resolvePath(inode.RootDirInodeNumber, path, heldLocks, resolvePathFollowSymlnks|...) -// -// if retryRequired { -// heldLocks.free() -// goto Restart -// } -// -// // Do whatever needed to be done with returned [dirInode and] dirEntryInode -// -// heldLocks.free() -// -func (vS *volumeStruct) resolvePath(startingInodeNumber inode.InodeNumber, path string, heldLocks *heldLocksStruct, options resolvePathOption) (dirInodeNumber inode.InodeNumber, dirEntryInodeNumber inode.InodeNumber, dirEntryBasename string, dirEntryInodeType inode.InodeType, retryRequired bool, err error) { - var ( - dirEntryInodeLock *dlm.RWLockStruct - dirEntryInodeLockAlreadyExclusive bool - dirEntryInodeLockAlreadyHeld bool - dirEntryInodeLockAlreadyShared bool - dirInodeLock *dlm.RWLockStruct - dirInodeLockAlreadyExclusive bool - dirInodeLockAlreadyHeld bool - dirInodeLockAlreadyShared bool - dlmCallerID dlm.CallerID - followSymlink bool - inodeVolumeHandle inode.VolumeHandle - internalErr error - pathSplit []string - pathSplitPart string - pathSplitPartIndex int - symlinkCount uint16 - symlinkTarget string - ) - - // Setup default returns - - dirInodeNumber = inode.InodeNumber(0) - dirEntryInodeNumber = inode.InodeNumber(0) - dirEntryBasename = "" - retryRequired = false - err = nil - - // Validate options - - if resolvePathOptionsCheck(options, resolvePathRequireExclusiveLockOnDirInode) && resolvePathOptionsCheck(options, resolvePathRequireSharedLockOnDirInode) { - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,,,options) includes both resolvePathRequireExclusiveLockOnDirInode & resolvePathRequireSharedLockOnDirInode") - return - } - - if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeDirectory) { - if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeFile) || resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeSymlink) { - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,,,options) cannot include more than one {resolvePathDirEntryInodeMustBeDirectory, resolvePathDirEntryInodeMustBeFile, resolvePathDirEntryInodeMustBeSymlink}") - return - } - } else if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeFile) && resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeSymlink) { - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,,,options) cannot include more than one {resolvePathDirEntryInodeMustBeDirectory, resolvePathDirEntryInodeMustBeFile, resolvePathDirEntryInodeMustBeSymlink}") - return - } - - // Setup shortcuts/contants - - dlmCallerID = dlm.GenerateCallerID() - inodeVolumeHandle = vS.inodeVolumeHandle - - // Prepare for SymlinkInode-restart handling on canonicalized path - - symlinkCount = 0 - - pathSplit, internalErr = canonicalizePath(path) - -RestartAfterFollowingSymlink: - - if nil != internalErr { - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,\"%s\",,) invalid", path) - return - } - - // Start loop from a ReadLock on startingInodeNumber - - dirInodeNumber = inode.InodeNumber(0) - - dirEntryInodeNumber = startingInodeNumber - - dirEntryInodeLock, dirEntryInodeLockAlreadyExclusive = heldLocks.exclusive[dirEntryInodeNumber] - if dirEntryInodeLockAlreadyExclusive { - dirEntryInodeLockAlreadyHeld = true - dirEntryInodeLockAlreadyShared = false - } else { - dirEntryInodeLock, dirEntryInodeLockAlreadyShared = heldLocks.shared[dirEntryInodeNumber] - if dirEntryInodeLockAlreadyShared { - dirEntryInodeLockAlreadyHeld = true - } else { - dirEntryInodeLockAlreadyHeld = false - dirEntryInodeLock, internalErr = inodeVolumeHandle.AttemptReadLock(dirEntryInodeNumber, dlmCallerID) - if nil != internalErr { - retryRequired = true - return - } - } - } - - if 0 == len(pathSplit) { - // Special case where path resolves to "/" - // Note: dirEntryInodeNumber == inode.RootDirInodeNumber - - // Reject invalid options for the "/" case - - if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeFile) || - resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeSymlink) || - resolvePathOptionsCheck(options, resolvePathRequireExclusiveLockOnDirInode) || - resolvePathOptionsCheck(options, resolvePathRequireSharedLockOnDirInode) { - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(inode.RootDirInodeNumber,\"/\",,options) cannot satisfy options: 0x%08X", options) - return - } - - // Attempt to obtain requested lock on "/" - - if resolvePathOptionsCheck(options, resolvePathRequireExclusiveLockOnDirEntryInode) { - if !dirEntryInodeLockAlreadyExclusive { - if dirEntryInodeLockAlreadyShared { - // Promote heldLocks.shared dirEntryInodeLock to .exclusive - - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - - delete(heldLocks.shared, dirEntryInodeNumber) - dirEntryInodeLockAlreadyHeld = false - dirEntryInodeLockAlreadyShared = false - - dirEntryInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirEntryInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - retryRequired = true - return - } - - heldLocks.exclusive[dirEntryInodeNumber] = dirEntryInodeLock - dirEntryInodeLockAlreadyExclusive = true - dirEntryInodeLockAlreadyHeld = true - } else { - // Promote temporary ReadLock dirEntryInodeLock to .exclusive - - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - - dirEntryInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirEntryInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - retryRequired = true - return - } - - heldLocks.exclusive[dirEntryInodeNumber] = dirEntryInodeLock - dirEntryInodeLockAlreadyExclusive = true - dirEntryInodeLockAlreadyHeld = true - } - } - } else { - if !dirEntryInodeLockAlreadyHeld { - // Promote temporary ReadLock dirEntryInodeLock to heldLocks.shared - - heldLocks.shared[dirEntryInodeNumber] = dirEntryInodeLock - dirEntryInodeLockAlreadyHeld = true - dirEntryInodeLockAlreadyShared = true - } - } - - return - } - - // Now loop for each pathSplit part - - for pathSplitPartIndex, pathSplitPart = range pathSplit { - // Shift dirEntryInode to dirInode as we recurse down pathSplit - - if inode.InodeNumber(0) != dirInodeNumber { - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - } - - dirInodeNumber = dirEntryInodeNumber - dirInodeLock = dirEntryInodeLock - dirInodeLockAlreadyExclusive = dirEntryInodeLockAlreadyExclusive - dirInodeLockAlreadyHeld = dirEntryInodeLockAlreadyHeld - dirInodeLockAlreadyShared = dirEntryInodeLockAlreadyShared - - // Lookup dirEntry - - dirEntryBasename = pathSplitPart // In case this is the last pathSplitPart that needs to be returned - - dirEntryInodeNumber, internalErr = inodeVolumeHandle.Lookup(dirInodeNumber, pathSplitPart) - - if nil == internalErr { - // Lookup() succeeded... ensure we have at least a ReadLock on dirEntryInode - - dirEntryInodeLock, dirEntryInodeLockAlreadyExclusive = heldLocks.exclusive[dirEntryInodeNumber] - if dirEntryInodeLockAlreadyExclusive { - dirEntryInodeLockAlreadyHeld = true - dirEntryInodeLockAlreadyShared = false - } else { - dirEntryInodeLock, dirEntryInodeLockAlreadyShared = heldLocks.shared[dirEntryInodeNumber] - if dirEntryInodeLockAlreadyShared { - dirEntryInodeLockAlreadyHeld = true - } else { - dirEntryInodeLockAlreadyHeld = false - dirEntryInodeLock, internalErr = inodeVolumeHandle.AttemptReadLock(dirEntryInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - // But first, free locks not recorded in heldLocks (if any) - - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - retryRequired = true - return - } - } - } - - // Handle SymlinkInode case if requested - - dirEntryInodeType, internalErr = inodeVolumeHandle.GetType(dirEntryInodeNumber) - if nil != internalErr { - logger.Fatalf("resolvePath(): failed obtaining InodeType for some part of path \"%s\"", path) - } - if dirEntryInodeType == inode.SymlinkType { - if pathSplitPartIndex < (len(pathSplit) - 1) { - followSymlink = resolvePathOptionsCheck(options, resolvePathFollowDirSymlinks) - } else { // pathSplitPartIndex == (len(pathSplit) - 1) - followSymlink = resolvePathOptionsCheck(options, resolvePathFollowDirEntrySymlinks) - } - - if followSymlink { - symlinkTarget, internalErr = inodeVolumeHandle.GetSymlink(dirEntryInodeNumber) - if nil != internalErr { - logger.Fatalf("resolvePath(): failure from inode.GetSymlink() on a known SymlinkInode: %v", err) - } - - // Free locks not recorded in heldLocks (if any) - - if !dirEntryInodeLockAlreadyHeld { - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - } - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - // Enforce SymlinkMax setting - - if 0 != globals.symlinkMax { - symlinkCount++ - - if symlinkCount > globals.symlinkMax { - err = blunder.NewError(blunder.TooManySymlinksError, "resolvePath(): exceeded SymlinkMax") - return - } - } - - // Apply symlinkTarget to path, reCanonicalize resultant path, and restart - - pathSplit, internalErr = reCanonicalizePathForSymlink(pathSplit, pathSplitPartIndex, symlinkTarget) - goto RestartAfterFollowingSymlink - } else { - // Not following SymlinkInode... so its a failure if not last pathSplitPart - - if pathSplitPartIndex < (len(pathSplit) - 1) { - // But first, free locks not recorded in heldLocks (if any) - - if !dirEntryInodeLockAlreadyHeld { - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - } - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,\"%s\",,) invalid", path) - return - } else { - // Being the last pathSplitPart, ensure caller didn't require it to be a DirInode or FileInode - - if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeDirectory) { - if !dirEntryInodeLockAlreadyHeld { - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - } - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - err = blunder.NewError(blunder.InvalidArgError, - "resolvePath(,\"%s\",,) '%s' is not a directory "+ - "stack:\n%s", - path, pathSplitPart, utils.MyStackTrace()) - - return - } else if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeFile) { - if !dirEntryInodeLockAlreadyHeld { - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - } - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,\"%s\",,) did not find FileInode", path) - return - } else { - // SymlinkInode is ok - } - } - } - } else { - // Not a SymlinkInode... check if it's last pathSplitPart and, if so, of correct InodeType - - if pathSplitPartIndex == (len(pathSplit) - 1) { - if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeDirectory) { - if dirEntryInodeType != inode.DirType { - if !dirEntryInodeLockAlreadyHeld { - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - } - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - err = blunder.NewError(blunder.InvalidArgError, - "resolvePath(,\"%s\",,) '%s' is not a directory "+ - "stack:\n%s", - path, pathSplitPart, utils.MyStackTrace()) - return - } - } else if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeFile) { - if dirEntryInodeType != inode.FileType { - if !dirEntryInodeLockAlreadyHeld { - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - } - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,\"%s\",,) did not find FileInode", path) - return - } - } else if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeSymlink) { - if dirEntryInodeType != inode.SymlinkType { - if !dirEntryInodeLockAlreadyHeld { - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - } - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,\"%s\",,) did not find SymlinkInode", path) - return - } - } else { - // Any InodeType is ok - } - } - } - } else { - // Lookup() failed... is resolvePath() asked to create missing Inode? - - if resolvePathOptionsCheck(options, resolvePathCreateMissingPathElements) { - // Cannot implicitly create a SymlinkInode for last pathSplitPart - - if (pathSplitPartIndex == (len(pathSplit) - 1)) && resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeSymlink) { - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(,\"%s\",,) did not find SymlinkInode", path) - return - } - - // Must hold exclusive lock to create missing {Dir|File}Inode - - if !dirInodeLockAlreadyExclusive { - if dirInodeLockAlreadyShared { - // Promote heldLocks.shared InodeLock to .exclusive - - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - - delete(heldLocks.shared, dirInodeNumber) - dirInodeLockAlreadyHeld = false - dirInodeLockAlreadyShared = false - - dirInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - - retryRequired = true - return - } - - heldLocks.exclusive[dirInodeNumber] = dirInodeLock - dirInodeLockAlreadyExclusive = true - dirInodeLockAlreadyHeld = true - } else { - // Promote temporary ReadLock to heldLocks.exclusive - - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - - dirInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - - retryRequired = true - return - } - - heldLocks.exclusive[dirInodeNumber] = dirInodeLock - dirInodeLockAlreadyExclusive = true - dirInodeLockAlreadyHeld = true - } - } - - // Create missing {Dir|File}Inode (cannot implicitly create a SymlinkInode) - - if (pathSplitPartIndex < (len(pathSplit) - 1)) || resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeDirectory) { - // Create a DirInode to be inserted - - dirEntryInodeType = inode.DirType - - dirEntryInodeNumber, internalErr = inodeVolumeHandle.CreateDir(inode.InodeMode(0777), inode.InodeRootUserID, inode.InodeGroupID(0)) - if nil != internalErr { - err = blunder.NewError(blunder.PermDeniedError, "resolvePath(): failed to create a DirInode: %v", err) - return - } - } else { // (pathSplitPartIndex == (len(pathSplit) - 1)) && !resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeDirectory) - if resolvePathOptionsCheck(options, resolvePathDirEntryInodeMustBeSymlink) { - // Cannot implicitly create a SymlinkInode - - err = blunder.NewError(blunder.InvalidArgError, "resolvePath(): cannot create a missing SymlinkInode") - return - } else { - // Create a FileInode to be inserted - - dirEntryInodeType = inode.FileType - - dirEntryInodeNumber, internalErr = inodeVolumeHandle.CreateFile(inode.InodeMode(0666), inode.InodeRootUserID, inode.InodeGroupID(0)) - if nil != internalErr { - err = blunder.NewError(blunder.PermDeniedError, "resolvePath(): failed to create a FileInode: %v", err) - return - } - } - } - - // Obtain and record an exclusive lock on just created {Dir|File}Inode - - dirEntryInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirEntryInodeNumber, dlmCallerID) - if nil != internalErr { - logger.Fatalf("resolvePath(): failed to exclusively lock just-created Inode 0x%016X: %v", dirEntryInodeNumber, internalErr) - } - - heldLocks.exclusive[dirEntryInodeNumber] = dirEntryInodeLock - dirEntryInodeLockAlreadyExclusive = true - dirEntryInodeLockAlreadyHeld = true - dirEntryInodeLockAlreadyShared = false - - // Now insert created {Dir|File}Inode - - internalErr = inodeVolumeHandle.Link(dirInodeNumber, pathSplitPart, dirEntryInodeNumber, false) - if nil != internalErr { - err = blunder.NewError(blunder.PermDeniedError, "resolvePath(): failed to Link created {Dir|File}Inode into path %s: %v", path, internalErr) - internalErr = inodeVolumeHandle.Destroy(dirEntryInodeNumber) - if nil != internalErr { - logger.Errorf("resolvePath(): failed to Destroy() created {Dir|File}Inode 0x%016X: %v", dirEntryInodeNumber, internalErr) - } - return - } - } else { - // Don't create missing Inode... so its a failure - // But first, free locks not recorded in heldLocks (if any) - - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - err = blunder.NewError(blunder.NotFoundError, "resolvePath(,\"%s\",,) invalid", path) - return - } - } - } - - if resolvePathOptionsCheck(options, resolvePathRequireExclusiveLockOnDirEntryInode) { - if !dirEntryInodeLockAlreadyExclusive { - if dirEntryInodeLockAlreadyShared { - // Promote heldLocks.shared dirEntryInodeLock to .exclusive - - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - - delete(heldLocks.shared, dirEntryInodeNumber) - dirEntryInodeLockAlreadyHeld = false - dirEntryInodeLockAlreadyShared = false - - dirEntryInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirEntryInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - retryRequired = true - return - } - - heldLocks.exclusive[dirEntryInodeNumber] = dirEntryInodeLock - dirEntryInodeLockAlreadyExclusive = true - dirEntryInodeLockAlreadyHeld = true - } else { - // Promote temporary ReadLock dirEntryInodeLock to .exclusive - - internalErr = dirEntryInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirEntryInodeLock.LockID, internalErr) - } - - dirEntryInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirEntryInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - - if !dirInodeLockAlreadyHeld { - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - - retryRequired = true - return - } - - heldLocks.exclusive[dirEntryInodeNumber] = dirEntryInodeLock - dirEntryInodeLockAlreadyExclusive = true - dirEntryInodeLockAlreadyHeld = true - } - } - } else { - if !dirEntryInodeLockAlreadyHeld { - // Promote temporary ReadLock dirEntryInodeLock to heldLocks.shared - - heldLocks.shared[dirEntryInodeNumber] = dirEntryInodeLock - dirEntryInodeLockAlreadyHeld = true - dirEntryInodeLockAlreadyShared = true - } - } - - if resolvePathOptionsCheck(options, resolvePathRequireExclusiveLockOnDirInode) { - if !dirInodeLockAlreadyExclusive { - if dirInodeLockAlreadyShared { - // Promote heldLocks.shared dirInodeLock to .exclusive - - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - - delete(heldLocks.shared, dirInodeNumber) - dirInodeLockAlreadyHeld = false - dirInodeLockAlreadyShared = false - - dirInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - - retryRequired = true - return - } - - heldLocks.exclusive[dirInodeNumber] = dirInodeLock - dirInodeLockAlreadyExclusive = true - dirInodeLockAlreadyHeld = true - } else { - // Promote temporary ReadLock dirInodeLock to .exclusive - - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - - dirInodeLock, internalErr = inodeVolumeHandle.AttemptWriteLock(dirInodeNumber, dlmCallerID) - if nil != internalErr { - // Caller must call heldLocks.free(), "backoff", and retry - - retryRequired = true - return - } - - heldLocks.exclusive[dirInodeNumber] = dirInodeLock - dirInodeLockAlreadyExclusive = true - dirInodeLockAlreadyHeld = true - } - } - } else if resolvePathOptionsCheck(options, resolvePathRequireSharedLockOnDirInode) { - if !dirInodeLockAlreadyHeld { - // Promote temporary ReadLock dirInodeLock to .shared - - heldLocks.shared[dirInodeNumber] = dirInodeLock - dirInodeLockAlreadyHeld = true - dirInodeLockAlreadyShared = true - } - } else { - if !dirInodeLockAlreadyHeld { - // If only temporary ReadLock dirInodeLock is held, release it - - internalErr = dirInodeLock.Unlock() - if nil != internalErr { - logger.Fatalf("resolvePath(): failed unlocking held LockID %s: %v", dirInodeLock.LockID, internalErr) - } - } - } - - return -} - -func canonicalizePath(path string) (canonicalizedPathSplit []string, err error) { - var ( - canonicalizedPathSplitLen int - pathSplit []string - pathSplitElement string - ) - - pathSplit = strings.Split(path, "/") - - canonicalizedPathSplit = make([]string, 0, len(pathSplit)) - - for _, pathSplitElement = range pathSplit { - switch pathSplitElement { - case "": - // drop pathSplitElement - case ".": - // drop pathSplitElement - case "..": - // backup one pathSplitElement - canonicalizedPathSplitLen = len(canonicalizedPathSplit) - if 0 == canonicalizedPathSplitLen { - err = fmt.Errorf("\"..\" in path stepped beyond start of path") - return - } - canonicalizedPathSplit = canonicalizedPathSplit[:canonicalizedPathSplitLen-1] - default: - // append pathSplitElement - canonicalizedPathSplit = append(canonicalizedPathSplit, pathSplitElement) - } - } - - err = nil - return -} - -func reCanonicalizePathForSymlink(canonicalizedPathSplit []string, symlinkIndex int, symlinkTarget string) (reCanonicalizedPathSplit []string, err error) { - var ( - updatedPath string - updatedPathAfterSymlink string - updatedPathBeforeSymlink string - ) - - if (0 == symlinkIndex) || strings.HasPrefix(symlinkTarget, "/") { - updatedPathBeforeSymlink = "" - } else { - updatedPathBeforeSymlink = strings.Join(canonicalizedPathSplit[:symlinkIndex], "/") - } - - if len(canonicalizedPathSplit) == (symlinkIndex - 1) { - updatedPathAfterSymlink = "" - } else { - updatedPathAfterSymlink = strings.Join(canonicalizedPathSplit[symlinkIndex+1:], "/") - } - - updatedPath = updatedPathBeforeSymlink + "/" + symlinkTarget + "/" + updatedPathAfterSymlink - - reCanonicalizedPathSplit, err = canonicalizePath(updatedPath) - - return -} - -// canonicalizePathAndLocateLeafDirInode performs what canonicalizePath() does above but, in addition, -// locates the index in the resultant canonicalizedPathSplit where an existing directory resides. Note -// that no DLM Locks should be held at invocation as this func will be performing any necessary retries -// and would have no way of backing out of a caller's existing DLM locks. -// -// Returns: -// If canonicalPath is empty or invalid, err will be non-nil -// If path-located Inode exists, -// If path-located Inode is a DirInode, dirInodeIndex == len(canonicalizedPathSplit) - 1 -// If path-located Inode is not a DirInode, dirInodeIndex == len(canonicalizedPathSplit) - 2 -// If path-located Inode does not exist, dirInodeIndex == len(canonicalizedPathSplit) - 2 -// -// Note 1: A dirInodeIndex == -1 is certainly possible and is, indeed, the expected result when -// parsing a path that directory refers to a Swift Container (i.e. root-level subdirectory). -// -// Note 2: No symbolic links will be followed. This is not a problem given that all we are really -// trying to indicate is if we know the path would resolve to a directory or not. It is -// certainly possible that the leaf element of the supplied path is a SymlinkInode pointing -// at a DirInode, but in that case it's still ok to begin searching from the DirInode -// containing the SymlinkInode (as in the case of MiddlewareGetContainer()'s use case). -// -// Note that a dirInodeIndex == -1 is possible -// -func (vS *volumeStruct) canonicalizePathAndLocateLeafDirInode(path string) (canonicalizedPathSplit []string, dirInodeIndex int, err error) { - var ( - dirEntryInodeType inode.InodeType - heldLocks *heldLocksStruct - retryRequired bool - tryLockBackoffContext *tryLockBackoffContextStruct - ) - - canonicalizedPathSplit, err = canonicalizePath(path) - if nil != err { - return - } - if 0 == len(canonicalizedPathSplit) { - err = fmt.Errorf("Canonically empty path \"%s\" not allowed", path) - return - } - - tryLockBackoffContext = &tryLockBackoffContextStruct{} - -Restart: - - tryLockBackoffContext.backoff() - - heldLocks = newHeldLocks() - - _, _, _, dirEntryInodeType, retryRequired, err = - vS.resolvePath( - inode.RootDirInodeNumber, - strings.Join(canonicalizedPathSplit, "/"), - heldLocks, - 0) - if nil != err { - heldLocks.free() - dirInodeIndex = len(canonicalizedPathSplit) - 2 - err = nil - return - } - if retryRequired { - heldLocks.free() - goto Restart - } - - heldLocks.free() - - if inode.DirType == dirEntryInodeType { - dirInodeIndex = len(canonicalizedPathSplit) - 1 - } else { - dirInodeIndex = len(canonicalizedPathSplit) - 2 - } - - err = nil - return -} diff --git a/fs/resolve_path_test.go b/fs/resolve_path_test.go deleted file mode 100644 index 5a6a521d..00000000 --- a/fs/resolve_path_test.go +++ /dev/null @@ -1,424 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fs - -import ( - "testing" - - "github.com/NVIDIA/proxyfs/inode" -) - -func TestResolvePath(t *testing.T) { - var ( - canonicalizedPathSplit []string - dirA inode.InodeNumber - dirEntryBasename string - dirEntryInodeNumber inode.InodeNumber - dirEntryInodeType inode.InodeType - dirInodeNumber inode.InodeNumber - err error - fileA inode.InodeNumber - fileB inode.InodeNumber - heldLocks *heldLocksStruct - foundInMap bool - reCanonicalizedPathSplit []string - retryRequired bool - symlinkA inode.InodeNumber - testContainer inode.InodeNumber - ) - - // Validate canonicalizePath() & reCanonicalizePathForSymlink() - - canonicalizedPathSplit, err = canonicalizePath("//.//a/b/c/../d") - if nil != err { - t.Fatalf("canonicalizePath(\"//.//a/b/c/../d\") failed: %v", err) - } - if (3 != len(canonicalizedPathSplit)) || - ("a" != canonicalizedPathSplit[0]) || - ("b" != canonicalizedPathSplit[1]) || - ("d" != canonicalizedPathSplit[2]) { - t.Fatalf("canonicalizedPathSplit(\"//.//a/b/c/../d\") returned unexpected results: %v", canonicalizedPathSplit) - } - - reCanonicalizedPathSplit, err = reCanonicalizePathForSymlink(canonicalizedPathSplit, 1, "e/f") - if nil != err { - t.Fatalf("reCanonicalizePathForSymlink(canonicalizedPathSplit, 1, \"e/f\") failed: %v", err) - } - if (4 != len(reCanonicalizedPathSplit)) || - ("a" != reCanonicalizedPathSplit[0]) || - ("e" != reCanonicalizedPathSplit[1]) || - ("f" != reCanonicalizedPathSplit[2]) || - ("d" != reCanonicalizedPathSplit[3]) { - t.Fatalf("reCanonicalizePathForSymlink(canonicalizedPathSplit, 1, \"e/f\") returned unexpected results: %v", reCanonicalizedPathSplit) - } - - // Build out directory hierarchy underneath "/TestResolvePathContainer/": - // - // / - // TestResolvePathContainer/ - // FileA - // SimlinkA --> FileA - // SimlinkB --> ../DirA - // - // During testing, a PUT of /TestResolvePathContainer/SimlinkB/FileB - // will implicitly create /TestResolvePathContainer/DirA/FileB: - // - // / - // TestResolvePathContainer/ - // FileA - // SimlinkA --> FileA - // SimlinkB --> ../DirA - // DirA\ - // FileB - - testSetup(t, false) - - testContainer, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, "TestResolvePathContainer", inode.PosixModePerm) - if nil != err { - t.Fatalf("Mkdir(,,,,\"TestResolvePathContainer\",) failed: %v", err) - } - fileA, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testContainer, "FileA", inode.PosixModePerm) - if nil != err { - t.Fatalf("Create(,,,,\"FileA\",) failed: %v", err) - } - symlinkA, err = testVolumeStruct.Symlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testContainer, "SymlinkA", "FileA") - if nil != err { - t.Fatalf("Symlink(,,,,\"SymlinkA\",) failed: %v", err) - } - - // GET or HEAD on "/TestResolvePathContainer/SymlinkA" should find FileA - - heldLocks = newHeldLocks() - - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, dirEntryInodeType, retryRequired, err = - testVolumeStruct.resolvePath( - inode.RootDirInodeNumber, - "/TestResolvePathContainer/SymlinkA", - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks) - - if nil != err { - t.Fatalf("resolvePath() for GET or HEAD failed: %v", err) - } - if (testContainer != dirInodeNumber) || - (fileA != dirEntryInodeNumber) || - ("FileA" != dirEntryBasename) || - (inode.FileType != dirEntryInodeType) || - retryRequired || - (0 != len(heldLocks.exclusive)) || - (1 != len(heldLocks.shared)) { - t.Fatalf("resolvePath() for GET or HEAD returned unexpected results") - } - - _, foundInMap = heldLocks.shared[fileA] - if !foundInMap { - t.Fatalf("resolvePath() for GET or HEAD returned heldLocks.shared missing fileA") - } - - heldLocks.free() - - // POST or COALESCE (destPath) on "/TestResolvePathContainer/SymlinkA" should find FileA - - heldLocks = newHeldLocks() - - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, dirEntryInodeType, retryRequired, err = - testVolumeStruct.resolvePath( - inode.RootDirInodeNumber, - "/TestResolvePathContainer/SymlinkA", - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks| - resolvePathCreateMissingPathElements| - resolvePathRequireExclusiveLockOnDirEntryInode) - - if nil != err { - t.Fatalf("resolvePath() for POST or COALESCE (destPath) failed: %v", err) - } - if (testContainer != dirInodeNumber) || - (fileA != dirEntryInodeNumber) || - ("FileA" != dirEntryBasename) || - (inode.FileType != dirEntryInodeType) || - retryRequired || - (1 != len(heldLocks.exclusive)) || - (0 != len(heldLocks.shared)) { - t.Fatalf("resolvePath() for POST or COALESCE (destPath) returned unexpected results") - } - - _, foundInMap = heldLocks.exclusive[fileA] - if !foundInMap { - t.Fatalf("resolvePath() for POST or COALESCE (destPath) returned heldLocks.exclusive missing fileA") - } - - heldLocks.free() - - // DELETE on "/TestResolvePathContainer/SymlinkA" should find SymlinkA - - heldLocks = newHeldLocks() - - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, dirEntryInodeType, retryRequired, err = - testVolumeStruct.resolvePath( - inode.RootDirInodeNumber, - "/TestResolvePathContainer/SymlinkA", - heldLocks, - resolvePathFollowDirSymlinks| - resolvePathRequireExclusiveLockOnDirEntryInode| - resolvePathRequireExclusiveLockOnDirInode) - - if nil != err { - t.Fatalf("resolvePath() for DELETE failed: %v", err) - } - if (testContainer != dirInodeNumber) || - (symlinkA != dirEntryInodeNumber) || - ("SymlinkA" != dirEntryBasename) || - (inode.SymlinkType != dirEntryInodeType) || - retryRequired || - (2 != len(heldLocks.exclusive)) || - (0 != len(heldLocks.shared)) { - t.Fatalf("resolvePath() for DELETE returned unexpected results") - } - - _, foundInMap = heldLocks.exclusive[symlinkA] - if !foundInMap { - t.Fatalf("resolvePath() for DELETE returned heldLocks.shared missing symlinkA") - } - _, foundInMap = heldLocks.exclusive[testContainer] - if !foundInMap { - t.Fatalf("resolvePath() for DELETE returned heldLocks.shared missing symlinkA") - } - - heldLocks.free() - - // PUT on /TestResolvePathContainer/SimlinkB/FileB should create /TestResolvePathContainer/DirA/FileB - - heldLocks = newHeldLocks() - - dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, dirEntryInodeType, retryRequired, err = - testVolumeStruct.resolvePath( - inode.RootDirInodeNumber, - "/TestResolvePathContainer/DirA/FileB", - heldLocks, - resolvePathFollowDirEntrySymlinks| - resolvePathFollowDirSymlinks| - resolvePathCreateMissingPathElements| - resolvePathRequireExclusiveLockOnDirEntryInode) - - if nil != err { - t.Fatalf("resolvePath() for PUT (thru symlink & missing dir) failed: %v", err) - } - if ("FileB" != dirEntryBasename) || - (inode.FileType != dirEntryInodeType) || - retryRequired || - (3 != len(heldLocks.exclusive)) || - (0 != len(heldLocks.shared)) { - t.Fatalf("resolvePath() for PUT (thru symlink & missing dir) returned unexpected results") - } - - dirA = dirInodeNumber - fileB = dirEntryInodeNumber - - _, foundInMap = heldLocks.exclusive[fileB] - if !foundInMap { - t.Fatalf("resolvePath() for PUT (thru symlink & missing dir) returned heldLocks.exclusive missing fileB") - } - _, foundInMap = heldLocks.exclusive[dirA] - if !foundInMap { - t.Fatalf("resolvePath() for PUT (thru symlink & missing dir) returned heldLocks.exclusive missing dirA") - } - _, foundInMap = heldLocks.exclusive[testContainer] - if !foundInMap { - t.Fatalf("resolvePath() for PUT (thru symlink & missing dir) returned heldLocks.exclusive missing testContainer") - } - - heldLocks.free() - - dirEntryInodeNumber, err = testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testContainer, "DirA") - if nil != err { - t.Fatalf("Lookup(,,,,\"DirA\") failed: %v", err) - } - if dirEntryInodeNumber != dirA { - t.Fatalf("Lookup(,,,,\"DirA\") returned 0x%016X... expected 0x%016X", dirEntryInodeNumber, dirA) - } - - dirEntryInodeNumber, err = testVolumeStruct.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirA, "FileB") - if nil != err { - t.Fatalf("Lookup(,,,,\"FileB\") failed: %v", err) - } - if dirEntryInodeNumber != fileB { - t.Fatalf("Lookup(,,,,\"FileB\") returned 0x%016X... expected 0x%016X", dirEntryInodeNumber, fileB) - } - - // Destroy directory hierachy underneath "/TestResolvePathContainer/" - - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirA, "FileB") - if nil != err { - t.Fatalf("Unlink(,,,,\"FileB\") failed: %v", err) - } - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testContainer, "DirA") - if nil != err { - t.Fatalf("Rmdir(,,,,\"DirA\") failed: %v", err) - } - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testContainer, "SymlinkA") - if nil != err { - t.Fatalf("Unlink(,,,,\"SymlinkA\") failed: %v", err) - } - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testContainer, "FileA") - if nil != err { - t.Fatalf("Unlink(,,,,\"FileA\") failed: %v", err) - } - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, "TestResolvePathContainer") - if nil != err { - t.Fatalf("Rmdir(,,,,\"TestResolvePathContainer\") failed: %v", err) - } - - testTeardown(t) -} - -type testCanonicalizePathItemStruct struct { - path string - shouldSucceed bool - canonicalizedPathSplit []string - dirInodeIndex int -} - -func TestCanonicalizePath(t *testing.T) { - var ( - canonicalizedPathSplit []string - containerInodeNumber inode.InodeNumber - dirInodeIndex int - directoryInodeNumber inode.InodeNumber - err error - pathSplitIndex int - testCanonicalizePathItem *testCanonicalizePathItemStruct - testCanonicalizePathList []*testCanonicalizePathItemStruct - ) - - testSetup(t, false) - - _, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, "RootFileName", inode.PosixModePerm) - if nil != err { - t.Fatal(err) - } - containerInodeNumber, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, "ContainerName", inode.PosixModePerm) - if nil != err { - t.Fatal(err) - } - _, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerInodeNumber, "ContainerFileName", inode.PosixModePerm) - if nil != err { - t.Fatal(err) - } - directoryInodeNumber, err = testVolumeStruct.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerInodeNumber, "DirectoryName", inode.PosixModePerm) - if nil != err { - t.Fatal(err) - } - _, err = testVolumeStruct.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, directoryInodeNumber, "DirectoryFileName", inode.PosixModePerm) - if nil != err { - t.Fatal(err) - } - - testCanonicalizePathList = []*testCanonicalizePathItemStruct{ - &testCanonicalizePathItemStruct{ - path: "", - shouldSucceed: false, - canonicalizedPathSplit: []string{}, // Not actually returned - dirInodeIndex: 0, // Not actually returned - }, - &testCanonicalizePathItemStruct{ - path: "MissingNameInRoot", - shouldSucceed: true, - canonicalizedPathSplit: []string{"MissingNameInRoot"}, - dirInodeIndex: -1, - }, - &testCanonicalizePathItemStruct{ - path: "RootFileName", - shouldSucceed: true, - canonicalizedPathSplit: []string{"RootFileName"}, - dirInodeIndex: -1, - }, - &testCanonicalizePathItemStruct{ - path: "ContainerName", - shouldSucceed: true, - canonicalizedPathSplit: []string{"ContainerName"}, - dirInodeIndex: 0, - }, - &testCanonicalizePathItemStruct{ - path: "ContainerName/MissingNameInContainer", - shouldSucceed: true, - canonicalizedPathSplit: []string{"ContainerName", "MissingNameInContainer"}, - dirInodeIndex: 0, - }, - &testCanonicalizePathItemStruct{ - path: "ContainerName/DirectoryName", - shouldSucceed: true, - canonicalizedPathSplit: []string{"ContainerName", "DirectoryName"}, - dirInodeIndex: 1, - }, - &testCanonicalizePathItemStruct{ - path: "ContainerName/DirectoryName/MissingNameInDirectory", - shouldSucceed: true, - canonicalizedPathSplit: []string{"ContainerName", "DirectoryName", "MissingNameInDirectory"}, - dirInodeIndex: 1, - }, - &testCanonicalizePathItemStruct{ - path: "ContainerName/DirectoryName/DirectoryFileName", - shouldSucceed: true, - canonicalizedPathSplit: []string{"ContainerName", "DirectoryName", "DirectoryFileName"}, - dirInodeIndex: 1, - }, - &testCanonicalizePathItemStruct{ - path: "ContainerName/DirectoryName/MissingNameInDirectory/MissingSubDirectoryName", - shouldSucceed: true, - canonicalizedPathSplit: []string{"ContainerName", "DirectoryName", "MissingNameInDirectory", "MissingSubDirectoryName"}, - dirInodeIndex: 2, - }, - } - - for _, testCanonicalizePathItem = range testCanonicalizePathList { - canonicalizedPathSplit, dirInodeIndex, err = testVolumeStruct.canonicalizePathAndLocateLeafDirInode(testCanonicalizePathItem.path) - if testCanonicalizePathItem.shouldSucceed { - if nil == err { - if len(canonicalizedPathSplit) != len(testCanonicalizePathItem.canonicalizedPathSplit) { - t.Fatalf("canonicalizePathAndLocateLeafDirInode(\"%s\") received unexpected canonicalizedPathSplit: %v", testCanonicalizePathItem.path, canonicalizedPathSplit) - } - for pathSplitIndex = range canonicalizedPathSplit { - if canonicalizedPathSplit[pathSplitIndex] != testCanonicalizePathItem.canonicalizedPathSplit[pathSplitIndex] { - t.Fatalf("canonicalizePathAndLocateLeafDirInode(\"%s\") received unexpected canonicalizedPathSplit: %v", testCanonicalizePathItem.path, canonicalizedPathSplit) - } - } - if dirInodeIndex != testCanonicalizePathItem.dirInodeIndex { - t.Fatalf("canonicalizePathAndLocateLeafDirInode(\"%s\") received unexpected canonicalizedPathSplit: %v", testCanonicalizePathItem.path, dirInodeIndex) - } - } else { - t.Fatalf("canonicalizePathAndLocateLeafDirInode(\"%s\") unexpectadly failed: %v", testCanonicalizePathItem.path, err) - } - } else { // !testCanonicalizePathItem.shouldSucceed - if nil == err { - t.Fatalf("canonicalizePathAndLocateLeafDirInode(\"%s\") unexpectadly succeeded", testCanonicalizePathItem.path) - } - } - } - - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, directoryInodeNumber, "DirectoryFileName") - if nil != err { - t.Fatal(err) - } - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerInodeNumber, "DirectoryName") - if nil != err { - t.Fatal(err) - } - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerInodeNumber, "ContainerFileName") - if nil != err { - t.Fatal(err) - } - err = testVolumeStruct.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, "ContainerName") - if nil != err { - t.Fatal(err) - } - err = testVolumeStruct.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, "RootFileName") - if nil != err { - t.Fatal(err) - } - - testTeardown(t) -} diff --git a/fs/setup_teardown_test.go b/fs/setup_teardown_test.go deleted file mode 100644 index 4ef4d751..00000000 --- a/fs/setup_teardown_test.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fs - -import ( - "io/ioutil" - "os" - "runtime" - "sync" - "syscall" - "testing" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/ramswift" - "github.com/NVIDIA/proxyfs/transitions" -) - -var ( - testConfMap conf.ConfMap - testRamswiftDoneChan chan bool // our test chan used during testTeardown() to know ramswift is, indeed, down - testVolumeStruct *volumeStruct // our global volumeStruct to be used in tests -) - -func testSetup(t *testing.T, starvationMode bool) { - var ( - err error - ok bool - signalHandlerIsArmedWG sync.WaitGroup - testConfMapStrings []string - testConfUpdateStrings []string - testDir string - testVolumeHandle VolumeHandle - ) - - testDir, err = ioutil.TempDir(os.TempDir(), "ProxyFS_test_fs_") - if nil != err { - t.Fatalf("ioutil.TempDir() failed: %v", err) - } - - err = os.Chdir(testDir) - if nil != err { - t.Fatalf("os.Chdir() failed: %v", err) - } - - err = os.Mkdir("TestVolume", os.ModePerm) - - testConfMapStrings = []string{ - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=35262", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=3", - "SwiftClient.RetryLimitObject=3", - "SwiftClient.RetryDelay=10ms", - "SwiftClient.RetryDelayObject=10ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000", - "Peer:Peer0.PublicIPAddr=127.0.0.1", - "Peer:Peer0.PrivateIPAddr=127.0.0.1", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - "Cluster.Peers=Peer0", - "Cluster.WhoAmI=Peer0", - "Volume:TestVolume.FSID=1", - "Volume:TestVolume.PrimaryPeer=Peer0", - "Volume:TestVolume.AccountName=AUTH_test", - "Volume:TestVolume.AutoFormat=true", - "Volume:TestVolume.CheckpointContainerName=.__checkpoint__", - "Volume:TestVolume.CheckpointContainerStoragePolicy=gold", - "Volume:TestVolume.CheckpointInterval=10s", - "Volume:TestVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way", - "Volume:TestVolume.MaxFlushSize=10485760", - "Volume:TestVolume.MaxFlushTime=10s", - "Volume:TestVolume.FileDefragmentChunkSize=10485760", - "Volume:TestVolume.FileDefragmentChunkDelay=10ms", - "Volume:TestVolume.NonceValuesToReserve=100", - "Volume:TestVolume.MaxEntriesPerDirNode=32", - "Volume:TestVolume.MaxExtentsPerFileNode=32", - "Volume:TestVolume.MaxInodesPerMetadataNode=32", - "Volume:TestVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:TestVolume.MaxDirFileNodesPerMetadataNode=16", - "Volume:TestVolume.MaxBytesInodeCache=100000", - "Volume:TestVolume.InodeCacheEvictInterval=1s", - "Volume:TestVolume.ActiveLeaseEvictLowLimit=5000", - "Volume:TestVolume.ActiveLeaseEvictHighLimit=5010", - "VolumeGroup:TestVolumeGroup.VolumeList=TestVolume", - "VolumeGroup:TestVolumeGroup.VirtualIPAddr=", - "VolumeGroup:TestVolumeGroup.PrimaryPeer=Peer0", - "VolumeGroup:TestVolumeGroup.ReadCacheLineSize=1000000", - "VolumeGroup:TestVolumeGroup.ReadCacheWeight=100", - "FSGlobals.VolumeGroupList=TestVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.DirEntryCacheEvictLowLimit=10000", - "FSGlobals.DirEntryCacheEvictHighLimit=10010", - "FSGlobals.FileExtentMapEvictLowLimit=10000", - "FSGlobals.FileExtentMapEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - if starvationMode { - testConfUpdateStrings = []string{ - "SwiftClient.ChunkedConnectionPoolSize=1", - "SwiftClient.NonChunkedConnectionPoolSize=1", - } - } else { - testConfUpdateStrings = []string{ - "SwiftClient.ChunkedConnectionPoolSize=256", - "SwiftClient.NonChunkedConnectionPoolSize=64", - } - } - - err = testConfMap.UpdateFromStrings(testConfUpdateStrings) - if nil != err { - t.Fatalf("testConfMap.UpdateFromStrings(testConfUpdateStrings) failed: %v", err) - } - - signalHandlerIsArmedWG.Add(1) - testRamswiftDoneChan = make(chan bool, 1) - go ramswift.Daemon("/dev/null", testConfMapStrings, &signalHandlerIsArmedWG, testRamswiftDoneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - err = transitions.Up(testConfMap) - if nil != err { - t.Fatalf("transitions.Up() failed: %v", err) - } - - testVolumeHandle, err = FetchVolumeHandleByVolumeName("TestVolume") - if nil != err { - t.Fatalf("fs.FetchVolumeHandleByVolumeName() failed: %v", err) - } - testVolumeStruct, ok = testVolumeHandle.(*volumeStruct) - if !ok { - t.Fatalf("fs.Mount() returned !ok") - } -} - -func testTeardown(t *testing.T) { - var ( - err error - testDir string - ) - - err = transitions.Down(testConfMap) - if nil != err { - t.Fatalf("transitions.Down() failed: %v", err) - } - - _ = syscall.Kill(syscall.Getpid(), unix.SIGTERM) - _ = <-testRamswiftDoneChan - - // Run GC to reclaim memory before we proceed to next test - runtime.GC() - - testDir, err = os.Getwd() - if nil != err { - t.Fatalf("os.Getwd() failed: %v", err) - } - - err = os.Chdir("..") - if nil != err { - t.Fatalf("os.Chdir() failed: %v", err) - } - - err = os.RemoveAll(testDir) - if nil != err { - t.Fatalf("os.RemoveAll() failed: %v", err) - } -} diff --git a/fs/treewalk.go b/fs/treewalk.go deleted file mode 100644 index 1faf29e4..00000000 --- a/fs/treewalk.go +++ /dev/null @@ -1,1448 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fs - -import ( - "encoding/binary" - "fmt" - "io" - "io/ioutil" - "os" - "regexp" - "strconv" - "sync" - "time" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/dlm" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/swiftclient" -) - -const ( - jobBPTreeMaxKeysPerNode = uint64(100) - jobBPTreeEvictLowLimit = uint64(90) - jobBPTreeEvictHighLimit = uint64(100) - - validateVolumeInodeParallelism = uint64(100) - validateVolumeCalculateLinkCountParallelism = uint64(50) - validateVolumeFixLinkCountParallelism = uint64(50) - validateVolumeIncrementByteCountsParallelism = uint64(100) - - scrubVolumeInodeParallelism = uint64(100) - - invalidLinkCount = uint64(0xFFFFFFFFFFFFFFFF) // Indicates Inode to be removed - - lostAndFoundDirName = ".Lost+Found" -) - -type jobStruct struct { - sync.Mutex - globalWaitGroup sync.WaitGroup - jobType string - volumeName string - active bool - stopFlag bool - err []string - info []string - volume *volumeStruct - bpTreeCache sortedmap.BPlusTreeCache - bpTreeFile *os.File - bpTreeFileNextOffset uint64 - parallelismChan chan struct{} - parallelismChanSize uint64 - inodeVolumeHandle inode.VolumeHandle - headhunterVolumeHandle headhunter.VolumeHandle - inodeBPTree sortedmap.BPlusTree // Maps Inode# to LinkCount - childrenWaitGroup sync.WaitGroup -} - -type validateVolumeStruct struct { - jobStruct - objectBPTree sortedmap.BPlusTree // Maps Object# to ByteCount - lostAndFoundDirInodeNumber inode.InodeNumber - accountName string - checkpointContainerName string -} - -type scrubVolumeStruct struct { - jobStruct -} - -func (jS *jobStruct) Active() (active bool) { - active = jS.active - return -} - -func (jS *jobStruct) Wait() { - jS.globalWaitGroup.Wait() -} - -func (jS *jobStruct) Cancel() { - jS.stopFlag = true - jS.Wait() -} - -func (jS *jobStruct) Error() (err []string) { - var errString string - - jS.Lock() - - err = make([]string, 0, len(jS.err)) - for _, errString = range jS.err { - err = append(err, errString) - } - - jS.Unlock() - - return -} - -func (jS *jobStruct) Info() (info []string) { - var infoString string - - jS.Lock() - - info = make([]string, 0, len(jS.info)) - for _, infoString = range jS.info { - info = append(info, infoString) - } - - jS.Unlock() - - return -} - -func (jS *jobStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsString = fmt.Sprintf("0x%016X", key.(uint64)) - err = nil - return -} - -func (jS *jobStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - valueAsString = fmt.Sprintf("0x%016X", value.(uint64)) - err = nil - return -} - -func (jS *jobStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { - jS.Lock() - defer jS.Unlock() - nodeByteSlice = make([]byte, 0, objectLength) - _, err = jS.bpTreeFile.Seek(io.SeekStart, int(objectOffset)) - if nil != err { - return - } - _, err = io.ReadFull(jS.bpTreeFile, nodeByteSlice) - return -} - -func (jS *jobStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { - jS.Lock() - defer jS.Unlock() - _, err = jS.bpTreeFile.Seek(io.SeekEnd, int(0)) - if nil != err { - return - } - _, err = jS.bpTreeFile.WriteAt(nodeByteSlice, int64(jS.bpTreeFileNextOffset)) - if nil != err { - return - } - objectNumber = 0 - objectOffset = jS.bpTreeFileNextOffset - jS.bpTreeFileNextOffset += uint64(len(nodeByteSlice)) - return -} - -func (jS *jobStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { - // Short-lived vVS.bpTreeFile... no compaction/garbage-collection needed - err = nil - return -} - -func (jS *jobStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { - var ( - u64 uint64 - ) - - u64 = key.(uint64) - - packedKey = make([]byte, 8) - - binary.LittleEndian.PutUint64(packedKey, u64) - - err = nil - return -} - -func (jS *jobStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { - if 8 > len(payloadData) { - err = fmt.Errorf("fs.validateVolumeStruct.UnpackKey() called with insufficiently sized payloadData (%v) - should be at least 8", len(payloadData)) - return - } - - key = binary.LittleEndian.Uint64(payloadData[:8]) - bytesConsumed = 8 - - err = nil - return -} - -func (jS *jobStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { - var ( - u64 uint64 - ) - - u64 = value.(uint64) - - packedValue = make([]byte, 8) - - binary.LittleEndian.PutUint64(packedValue, u64) - - err = nil - return -} - -func (jS *jobStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { - if 8 > len(payloadData) { - err = fmt.Errorf("fs.validateVolumeStruct.UnpackValue() called with insufficiently sized payloadData (%v) - should be at least 8", len(payloadData)) - return - } - - value = binary.LittleEndian.Uint64(payloadData[:8]) - bytesConsumed = 8 - - err = nil - return -} - -func (jS *jobStruct) jobStartParallelism(parallelismChanSize uint64) { - var ( - i uint64 - ) - - jS.parallelismChan = make(chan struct{}, parallelismChanSize) - jS.parallelismChanSize = parallelismChanSize - - for i = uint64(0); i < parallelismChanSize; i++ { - jS.jobReleaseParallelism() - } -} - -func (jS *jobStruct) jobGrabParallelism() { - _ = <-jS.parallelismChan -} - -func (jS *jobStruct) jobReleaseParallelism() { - jS.parallelismChan <- struct{}{} -} - -func (jS *jobStruct) jobEndParallelism() { - var ( - i uint64 - ) - - for i = uint64(0); i < jS.parallelismChanSize; i++ { - jS.jobGrabParallelism() - } -} - -func (jS *jobStruct) jobLogErr(formatString string, args ...interface{}) { - jS.Lock() - jS.jobLogErrWhileLocked(formatString, args...) - jS.Unlock() -} - -func (jS *jobStruct) jobLogErrWhileLocked(formatString string, args ...interface{}) { - var ( - suffix string - ) - - suffix = fmt.Sprintf(formatString, args...) - - logger.Errorf("%v[%v] %v", jS.jobType, jS.volumeName, suffix) - - jS.err = append(jS.err, fmt.Sprintf("%v %v", time.Now().Format(time.RFC3339), suffix)) -} - -func (jS *jobStruct) jobLogInfo(formatString string, args ...interface{}) { - jS.Lock() - jS.jobLogInfoWhileLocked(formatString, args...) - jS.Unlock() -} - -func (jS *jobStruct) jobLogInfoWhileLocked(formatString string, args ...interface{}) { - var ( - suffix string - ) - - suffix = fmt.Sprintf(formatString, args...) - - logger.Infof("%v[%v] %v", jS.jobType, jS.volumeName, suffix) - - jS.info = append(jS.info, fmt.Sprintf("%v %v", time.Now().Format(time.RFC3339), suffix)) -} - -func (vVS *validateVolumeStruct) validateVolumeInode(inodeNumber uint64) { - var ( - err error - ) - - defer vVS.childrenWaitGroup.Done() - - defer vVS.jobReleaseParallelism() - - err = vVS.inodeVolumeHandle.Validate(inode.InodeNumber(inodeNumber), false) - if nil != err { - vVS.jobLogInfo("Got inode.Validate(0x%016X) failure: %v", inodeNumber, err) - - // Ultimately we will to delete or fix corrupt inode, but only after - // fixing validation errors. - // - // vVS.Lock() - // defer vVS.Unlock() - // - // vVS.jobLogInfoWhileLocked("Got inode.Validate(0x%016X) failure: %v", inodeNumber, err) - // ok, err = vVS.inodeBPTree.Put(inodeNumber, invalidLinkCount) - // if nil != err { - // vVS.jobLogErrWhileLocked("Got vVS.inodeBPTree.Put(0x%016X, invalidLinkCount) failure: %v", inodeNumber, err) - // return - // } - // if !ok { - // vVS.jobLogErrWhileLocked("Got vVS.inodeBPTree.Put(0x%016X, invalidLinkCount) !ok", inodeNumber) - // return - // } - } -} - -func (vVS *validateVolumeStruct) validateVolumeIncrementCalculatedLinkCountWhileLocked(inodeNumber uint64) (ok bool) { - var ( - err error - linkCount uint64 - value sortedmap.Value - ) - - value, ok, err = vVS.inodeBPTree.GetByKey(inodeNumber) - if nil != err { - vVS.jobLogErrWhileLocked("Got vVS.inodeBPTree.GetByKey(0x%016X) failure: %v", inodeNumber, err) - ok = false - return - } - if !ok { - vVS.jobLogErrWhileLocked("Got vVS.inodeBPTree.GetByKey(0x%016X) !ok", inodeNumber) - ok = false - return - } - linkCount = value.(uint64) + 1 - ok, err = vVS.inodeBPTree.PatchByKey(inodeNumber, linkCount) - if nil != err { - vVS.jobLogErrWhileLocked("Got vVS.inodeBPTree.PatchByKey(0x%016X,) failure: %v", inodeNumber, err) - ok = false - return - } - if !ok { - vVS.jobLogErrWhileLocked("Got vVS.inodeBPTree.PatchByKey(0x%016X,) !ok", inodeNumber) - ok = false - return - } - - ok = true - - return -} - -func (vVS *validateVolumeStruct) validateVolumeCalculateLinkCount(parentDirInodeNumber uint64, dirInodeNumber uint64) { - var ( - dirEntrySlice []inode.DirEntry - dotDotWasSeen bool - dotWasSeen bool - err error - inodeNumber uint64 - inodeType inode.InodeType - moreEntries bool - ok bool - prevReturnedAsString string - ) - - defer vVS.childrenWaitGroup.Done() - - defer vVS.jobReleaseParallelism() - - prevReturnedAsString = "" - - dotWasSeen = false - dotDotWasSeen = false - -forLabel: - for { - if vVS.stopFlag { - return - } - - dirEntrySlice, moreEntries, err = vVS.inodeVolumeHandle.ReadDir(inode.InodeNumber(dirInodeNumber), 1, 0, prevReturnedAsString) - if nil != err { - vVS.jobLogErr("Got inode.ReadDir(0x%016X,1,0,\"%v\") failure: %v", dirInodeNumber, prevReturnedAsString, err) - return - } - - if 0 == len(dirEntrySlice) { - break forLabel - } else if 1 < len(dirEntrySlice) { - vVS.jobLogErr("Got too many DirEntry's from inode.ReadDir(0x%016X,1,0,\"%v\")", dirInodeNumber, prevReturnedAsString) - return - } - - // Increment LinkCount for dirEntrySlice[0]'s InodeNumber - - inodeNumber = uint64(dirEntrySlice[0].InodeNumber) - prevReturnedAsString = dirEntrySlice[0].Basename - - vVS.Lock() - - if ("." == prevReturnedAsString) && (inodeNumber != dirInodeNumber) { - _, err = vVS.inodeVolumeHandle.Unlink(inode.InodeNumber(dirInodeNumber), ".", true) - if nil != err { - vVS.jobLogErrWhileLocked("Got inode.Unlink(0x%016X,\".\",true) failure: %v", dirInodeNumber, err) - vVS.Unlock() - return - } - err = vVS.inodeVolumeHandle.Link(inode.InodeNumber(dirInodeNumber), ".", inode.InodeNumber(dirInodeNumber), true) - if nil != err { - vVS.jobLogErrWhileLocked("Got inode.Link(0x%016X,\".\",0x%016X,true) failure: %v", dirInodeNumber, dirInodeNumber, err) - vVS.Unlock() - return - } - - vVS.jobLogInfoWhileLocked("\".\" dirEntry in dirInodeNumber 0x%016X was 0x%016X...fixed to point to dirInodeNumber", dirInodeNumber, inodeNumber) - - inodeNumber = dirInodeNumber - } else if (".." == prevReturnedAsString) && (inodeNumber != parentDirInodeNumber) { - _, err = vVS.inodeVolumeHandle.Unlink(inode.InodeNumber(dirInodeNumber), "..", true) - if nil != err { - vVS.jobLogErrWhileLocked("Got inode.Unlink(0x%016X,\"..\",true) failure: %v", dirInodeNumber, err) - vVS.Unlock() - return - } - err = vVS.inodeVolumeHandle.Link(inode.InodeNumber(dirInodeNumber), "..", inode.InodeNumber(parentDirInodeNumber), true) - if nil != err { - vVS.jobLogErrWhileLocked("Got inode.Link(0x%016X,\"..\",0x%016X,true) failure: %v", dirInodeNumber, parentDirInodeNumber, err) - vVS.Unlock() - return - } - - vVS.jobLogInfoWhileLocked("\"..\" dirEntry in dirInodeNumber 0x%016X was 0x%016X...fixed to point to parentDirInodeNumber (0x%016X)", dirInodeNumber, inodeNumber, parentDirInodeNumber) - - inodeNumber = parentDirInodeNumber - } - - ok = vVS.validateVolumeIncrementCalculatedLinkCountWhileLocked(inodeNumber) - if !ok { - vVS.Unlock() - return - } - - vVS.Unlock() - - // Recurse into sub-directories - - if "." == prevReturnedAsString { - dotWasSeen = true - } else if ".." == prevReturnedAsString { - dotDotWasSeen = true - } else { - inodeType, err = vVS.inodeVolumeHandle.GetType(inode.InodeNumber(inodeNumber)) - if nil != err { - vVS.jobLogErr("Got inode.GetType(0x%016X) failure: %v", inodeNumber, err) - return - } - - if inode.DirType == inodeType { - vVS.jobGrabParallelism() - vVS.childrenWaitGroup.Add(1) - go vVS.validateVolumeCalculateLinkCount(dirInodeNumber, inodeNumber) - } - } - - if !moreEntries { - break forLabel - } - } - - if !dotWasSeen { - vVS.Lock() - err = vVS.inodeVolumeHandle.Link(inode.InodeNumber(dirInodeNumber), ".", inode.InodeNumber(dirInodeNumber), true) - if nil != err { - vVS.jobLogErrWhileLocked("Got inode.Link(0x%016X,\".\",0x%016X,true) failure: %v", dirInodeNumber, dirInodeNumber, err) - vVS.Unlock() - return - } - ok = vVS.validateVolumeIncrementCalculatedLinkCountWhileLocked(dirInodeNumber) - if !ok { - vVS.Unlock() - return - } - vVS.jobLogInfoWhileLocked("\"..\" dirEntry in dirInodeNumber 0x%016X was missing...fixed to point to dirInodeNumber", dirInodeNumber) - vVS.Unlock() - return - } - - if !dotDotWasSeen { - vVS.Lock() - err = vVS.inodeVolumeHandle.Link(inode.InodeNumber(dirInodeNumber), "..", inode.InodeNumber(parentDirInodeNumber), true) - if nil != err { - vVS.jobLogErrWhileLocked("Got inode.Link(0x%016X,\"..\",0x%016X,true) failure: %v", dirInodeNumber, parentDirInodeNumber, err) - vVS.Unlock() - return - } - ok = vVS.validateVolumeIncrementCalculatedLinkCountWhileLocked(parentDirInodeNumber) - if !ok { - vVS.Unlock() - return - } - vVS.jobLogInfoWhileLocked("\"..\" dirEntry in dirInodeNumber 0x%016X was missing...fixed to point to parentDirInodeNumber (0x%016X)", dirInodeNumber, parentDirInodeNumber) - vVS.Unlock() - return - } -} - -func (vVS *validateVolumeStruct) validateVolumeFixLinkCount(inodeNumber uint64, linkCountComputed uint64) { - var ( - err error - linkCountInInode uint64 - ) - - defer vVS.childrenWaitGroup.Done() - - defer vVS.jobReleaseParallelism() - - linkCountInInode, err = vVS.inodeVolumeHandle.GetLinkCount(inode.InodeNumber(inodeNumber)) - if nil != err { - vVS.jobLogErr("Got inode.GetLinkCount(0x%016X) failure: %v", inodeNumber, err) - return - } - - if linkCountComputed != linkCountInInode { - err = vVS.inodeVolumeHandle.SetLinkCount(inode.InodeNumber(inodeNumber), linkCountComputed) - if nil == err { - vVS.jobLogInfo("Corrected LinkCount in Inode# 0x%016X from 0x%016X to 0x%016X", inodeNumber, linkCountInInode, linkCountComputed) - } else { - vVS.jobLogErr("Got inode.SetLinkCount(0x%016X,) failure: %v", inodeNumber, err) - return - } - } -} - -func (vVS *validateVolumeStruct) validateVolumeIncrementByteCounts(inodeNumber uint64) { - var ( - err error - layoutReport sortedmap.LayoutReport - objectBytes uint64 - objectNumber uint64 - ok bool - value sortedmap.Value - ) - - defer vVS.childrenWaitGroup.Done() - - defer vVS.jobReleaseParallelism() - - if vVS.stopFlag { - return - } - - layoutReport, err = vVS.inodeVolumeHandle.FetchLayoutReport(inode.InodeNumber(inodeNumber)) - if nil != err { - vVS.jobLogErr("Got vVS.inodeVolumeHandle.FetchLayoutReport(0x%016X) failure: %v", inodeNumber, err) - return - } - - vVS.Lock() - defer vVS.Unlock() - - for objectNumber, objectBytes = range layoutReport { - value, ok, err = vVS.objectBPTree.GetByKey(objectNumber) - if nil != err { - vVS.jobLogErrWhileLocked("Got vVS.objectBPTree.GetByKey() failure: %v", err) - return - } - - if ok { - objectBytes += value.(uint64) - - ok, err = vVS.objectBPTree.PatchByKey(objectNumber, objectBytes) - if nil != err { - vVS.jobLogErrWhileLocked("Got vVS.objectBPTree.PatchByKey() failure: %v", err) - return - } - if !ok { - vVS.jobLogErrWhileLocked("Got vVS.objectBPTree.PatchByKey(0) !ok") - return - } - } else { - ok, err = vVS.objectBPTree.Put(objectNumber, objectBytes) - if nil != err { - vVS.jobLogErrWhileLocked("Got vVS.objectBPTree.Put() failure: %v", err) - return - } - if !ok { - vVS.jobLogErrWhileLocked("Got vVS.objectBPTree.Put() !ok") - return - } - } - } -} - -func (vVS *validateVolumeStruct) validateVolume() { - var ( - checkpointContainerObjectList []string - checkpointContainerObjectName string - checkpointContainerObjectNumber uint64 - discrepencies uint64 - err error - headhunterLayoutReport sortedmap.LayoutReport - inodeCount int - inodeIndex uint64 - inodeNumber uint64 - inodeType inode.InodeType - key sortedmap.Key - linkCountComputed uint64 - moreEntries bool - objectIndex uint64 - objectNumber uint64 - ok bool - toDestroyInodeNumber inode.InodeNumber - validObjectNameRE = regexp.MustCompile("\\A[0-9a-fA-F]+\\z") - value sortedmap.Value - ) - - vVS.jobLogInfo("FSCK job initiated") - - defer func(vVS *validateVolumeStruct) { - if vVS.stopFlag { - vVS.jobLogInfo("FSCK job stopped") - } else if 0 == len(vVS.err) { - vVS.jobLogInfo("FSCK job completed without error") - } else if 1 == len(vVS.err) { - vVS.jobLogInfo("FSCK job exited with one error") - } else { - vVS.jobLogInfo("FSCK job exited with errors") - } - }(vVS) - - defer func(vVS *validateVolumeStruct) { - vVS.active = false - }(vVS) - - defer vVS.globalWaitGroup.Done() - - // Find specified volume - - globals.Lock() - - vVS.volume, ok = globals.volumeMap[vVS.volumeName] - if !ok { - globals.Unlock() - vVS.jobLogErr("Couldn't find fs.volumeStruct") - return - } - - globals.Unlock() - - vVS.inodeVolumeHandle, err = inode.FetchVolumeHandle(vVS.volumeName) - if nil != err { - vVS.jobLogErr("Couldn't find inode.VolumeHandle") - return - } - - vVS.headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(vVS.volumeName) - if nil != err { - vVS.jobLogErr("Couldn't find headhunter.VolumeHandle") - return - } - - vVS.volume.jobRWMutex.Lock() - defer vVS.volume.jobRWMutex.Unlock() - - // Flush all File Inodes currently in flight - - vVS.volume.untrackInFlightFileInodeDataAll() - - vVS.jobLogInfo("Completed flush of all inflight File Inode write traffic") - - // Do a checkpoint before actual FSCK work - - err = vVS.headhunterVolumeHandle.DoCheckpoint() - if nil != err { - vVS.jobLogErr("Got headhunter.DoCheckpoint failure: %v", err) - return - } - - vVS.jobLogInfo("Completed checkpoint prior to FSCK work") - - // Setup B+Tree to hold arbitrarily sized map[uint64]uint64 (i.e. beyond what will fit in memoory) - - vVS.bpTreeFile, err = ioutil.TempFile("", "ProxyFS_VaidateVolume_") - if nil != err { - vVS.jobLogErr("Got ioutil.TempFile() failure: %v", err) - return - } - defer func(vVS *validateVolumeStruct) { - var ( - bpTreeFileName string - ) - - bpTreeFileName = vVS.bpTreeFile.Name() - _ = vVS.bpTreeFile.Close() - _ = os.Remove(bpTreeFileName) - }(vVS) - - vVS.bpTreeFileNextOffset = 0 - - vVS.bpTreeCache = sortedmap.NewBPlusTreeCache(jobBPTreeEvictLowLimit, jobBPTreeEvictHighLimit) - - // Validate all Inodes in InodeRec table in headhunter - - vVS.inodeBPTree = sortedmap.NewBPlusTree(jobBPTreeMaxKeysPerNode, sortedmap.CompareUint64, vVS, vVS.bpTreeCache) - defer func(vVS *validateVolumeStruct) { - var err error - - err = vVS.inodeBPTree.Discard() - if nil != err { - vVS.jobLogErr("Got vVS.inodeBPTree.Discard() failure: %v", err) - } - }(vVS) - - vVS.jobLogInfo("Beginning enumeration of inodes to be validated") - - inodeIndex = 0 - - for { - if vVS.stopFlag { - return - } - - inodeNumber, ok, err = vVS.headhunterVolumeHandle.IndexedInodeNumber(inodeIndex) - if nil != err { - vVS.jobLogErr("Got headhunter.IndexedInodeNumber(0x%016X) failure: %v", inodeIndex, err) - return - } - if !ok { - break - } - - ok, err = vVS.inodeBPTree.Put(inodeNumber, uint64(0)) // Initial Linkount is 0 - if nil != err { - vVS.jobLogErr("Got vVS.inodeBPTree.Put(0x%016X, 0) failure: %v", inodeNumber, err) - return - } - if !ok { - vVS.jobLogErr("Got vVS.inodeBPTree.Put(0x%016X, 0) !ok", inodeNumber) - return - } - - inodeIndex++ - } - - if vVS.stopFlag || (0 < len(vVS.err)) { - return - } - - vVS.jobLogInfo("Completed enumeration of inodes to be validated") - - vVS.jobLogInfo("Beginning validation of inodes") - - inodeCount, err = vVS.inodeBPTree.Len() - if nil != err { - vVS.jobLogErr("Got vVS.inodeBPTree.Len() failure: %v", err) - return - } - - vVS.jobStartParallelism(validateVolumeInodeParallelism) - - for inodeIndex = uint64(0); inodeIndex < uint64(inodeCount); inodeIndex++ { - if vVS.stopFlag { - vVS.childrenWaitGroup.Wait() - vVS.jobEndParallelism() - return - } - - key, _, ok, err = vVS.inodeBPTree.GetByIndex(int(inodeIndex)) - if nil != err { - vVS.jobLogErr("Got vVS.inodeBPTree.GetByIndex(0x%016X) failure: %v", inodeIndex, err) - vVS.childrenWaitGroup.Wait() - vVS.jobEndParallelism() - return - } - if !ok { - vVS.jobLogErr("Got vVS.inodeBPTree.GetByIndex(0x%016X) !ok", inodeIndex) - vVS.childrenWaitGroup.Wait() - vVS.jobEndParallelism() - return - } - - inodeNumber = key.(uint64) - - vVS.jobGrabParallelism() - vVS.childrenWaitGroup.Add(1) - go vVS.validateVolumeInode(inodeNumber) - } - - vVS.childrenWaitGroup.Wait() - - vVS.jobEndParallelism() - - if vVS.stopFlag || (0 < len(vVS.err)) { - return - } - - // Scan inodeBPTree looking for invalidated Inodes - - inodeIndex = 0 - - for { - key, value, ok, err = vVS.inodeBPTree.GetByIndex(int(inodeIndex)) - if nil != err { - vVS.jobLogErr("Got inodeBPTree.GetByIndex(0x%016X) failure: %v", inodeIndex, err) - return - } - - if !ok { - break - } - - inodeNumber = key.(uint64) - linkCountComputed = value.(uint64) - - if invalidLinkCount == linkCountComputed { - err = vVS.headhunterVolumeHandle.DeleteInodeRec(inodeNumber) - if nil != err { - vVS.jobLogErr("Got headhunter.DeleteInodeRec(0x%016X) failure: %v", inodeNumber, err) - return - } - ok, err = vVS.inodeBPTree.DeleteByIndex(int(inodeIndex)) - if nil != err { - vVS.jobLogErr("Got inodeBPTree.DeleteByIndex(0x%016X) [inodeNumber == 0x%16X] failure: %v", inodeIndex, inodeNumber, err) - return - } - if !ok { - vVS.jobLogErr("Got inodeBPTree.DeleteByIndex(0x%016X) [inodeNumber == 0x%16X] !ok", inodeIndex, inodeNumber) - return - } - vVS.jobLogInfo("Removing inodeNumber 0x%016X due to validation failure", inodeNumber) - } else { - inodeIndex++ - } - } - - if vVS.stopFlag || (0 < len(vVS.err)) { - return - } - - vVS.jobLogInfo("Completed validation of inodes") - - // TreeWalk computing LinkCounts for all inodeNumbers - - _, ok, err = vVS.inodeBPTree.GetByKey(uint64(inode.RootDirInodeNumber)) - if nil != err { - vVS.jobLogErr("Got vVS.inodeBPTree.GetByKey(RootDirInodeNumber) failure: %v", err) - return - } - if !ok { - vVS.jobLogErr("Got vVS.inodeBPTree.GetByKey(RootDirInodeNumber) !ok") - return - } - - vVS.jobStartParallelism(validateVolumeCalculateLinkCountParallelism) - - vVS.jobGrabParallelism() - vVS.childrenWaitGroup.Add(1) - go vVS.validateVolumeCalculateLinkCount(uint64(inode.RootDirInodeNumber), uint64(inode.RootDirInodeNumber)) - - vVS.childrenWaitGroup.Wait() - - vVS.jobEndParallelism() - - if vVS.stopFlag || (0 < len(vVS.err)) { - return - } - - vVS.jobLogInfo("Completed treewalk before populating /%v/", lostAndFoundDirName) - - // Establish that lostAndFoundDirName exists - - vVS.lostAndFoundDirInodeNumber, err = vVS.inodeVolumeHandle.Lookup(inode.RootDirInodeNumber, lostAndFoundDirName) - if nil == err { - // Found it - make sure it is a directory - - inodeType, err = vVS.inodeVolumeHandle.GetType(vVS.lostAndFoundDirInodeNumber) - if nil != err { - vVS.jobLogErr("Got inode.GetType(vVS.lostAndFoundDirNumber==0x%016X) failure: %v", vVS.lostAndFoundDirInodeNumber, err) - return - } - if inode.DirType != inodeType { - vVS.jobLogErr("Got inode.GetType(vVS.lostAndFoundDirNumber==0x%016X) non-DirType", vVS.lostAndFoundDirInodeNumber) - return - } - - vVS.jobLogInfo("Found pre-existing /%v/", lostAndFoundDirName) - } else { - if blunder.Is(err, blunder.NotFoundError) { - // Create it - - vVS.lostAndFoundDirInodeNumber, err = vVS.inodeVolumeHandle.CreateDir(inode.PosixModePerm, 0, 0) - if nil != err { - vVS.jobLogErr("Got inode.CreateDir() failure: %v", err) - return - } - err = vVS.inodeVolumeHandle.Link(inode.RootDirInodeNumber, lostAndFoundDirName, vVS.lostAndFoundDirInodeNumber, false) - if nil != err { - vVS.jobLogErr("Got inode.Link(inode.RootDirInodeNumber, lostAndFoundDirName, vVS.lostAndFoundDirInodeNumber, false) failure: %v", err) - err = vVS.inodeVolumeHandle.Destroy(vVS.lostAndFoundDirInodeNumber) - if nil != err { - vVS.jobLogErr("Got inode.Destroy(vVS.lostAndFoundDirInodeNumber) failure: %v", err) - } - return - } - - ok, err = vVS.inodeBPTree.Put(uint64(vVS.lostAndFoundDirInodeNumber), uint64(2)) // / as well as //. - if nil != err { - vVS.jobLogErr("Got vVS.inodeBPTree.Put(vVS.lostAndFoundDirInodeNumber, 1) failure: %v", err) - vVS.Unlock() - return - } - if !ok { - vVS.jobLogErr("Got vVS.inodeBPTree.Put(vVS.lostAndFoundDirInodeNumber, 1) !ok") - vVS.Unlock() - return - } - - vVS.jobLogInfo("Created /%v/", lostAndFoundDirName) - } else { - vVS.jobLogErr("Got inode.Lookup(inode.RootDirInodeNumber, lostAndFoundDirName) failure: %v", err) - return - } - } - - // TODO: Scan B+Tree placing top-most orphan DirInodes as elements of vVS.lostAndFoundDirInodeNumber - - // Re-compute LinkCounts - - inodeCount, err = vVS.inodeBPTree.Len() - if nil != err { - vVS.jobLogErr("Got vVS.inodeBPTree.Len() failure: %v", err) - return - } - - for inodeIndex = uint64(0); inodeIndex < uint64(inodeCount); inodeIndex++ { - if vVS.stopFlag { - return - } - - ok, err = vVS.inodeBPTree.PatchByIndex(int(inodeIndex), uint64(0)) - if nil != err { - vVS.jobLogErr("Got vVS.inodeBPTree.PatchByIndex(0x%016X, 0) failure: %v", inodeIndex, err) - return - } - if !ok { - vVS.jobLogErr("Got vVS.inodeBPTree.PatchByIndex(0x%016X, 0) !ok", inodeIndex) - return - } - } - - vVS.jobStartParallelism(validateVolumeCalculateLinkCountParallelism) - - vVS.jobGrabParallelism() - vVS.childrenWaitGroup.Add(1) - go vVS.validateVolumeCalculateLinkCount(uint64(inode.RootDirInodeNumber), uint64(inode.RootDirInodeNumber)) - - vVS.childrenWaitGroup.Wait() - - vVS.jobEndParallelism() - - if vVS.stopFlag || (0 < len(vVS.err)) { - return - } - - vVS.jobLogInfo("Completed treewalk after populating /%v/", lostAndFoundDirName) - - // TODO: Scan B+Tree placing orphaned non-DirInodes as elements of vVS.lostAndFoundDirInodeNumber - - // Update incorrect LinkCounts - - vVS.jobStartParallelism(validateVolumeFixLinkCountParallelism) - - for inodeIndex = uint64(0); inodeIndex < uint64(inodeCount); inodeIndex++ { - if vVS.stopFlag { - vVS.childrenWaitGroup.Wait() - vVS.jobEndParallelism() - return - } - - key, value, ok, err = vVS.inodeBPTree.GetByIndex(int(inodeIndex)) - if nil != err { - vVS.childrenWaitGroup.Wait() - vVS.jobEndParallelism() - vVS.jobLogErr("Got vVS.inodeBPTree.GetByIndex(0x%016X) failure: %v", inodeIndex, err) - return - } - if !ok { - vVS.childrenWaitGroup.Wait() - vVS.jobEndParallelism() - vVS.jobLogErr("Got vVS.inodeBPTree.GetByIndex(0x%016X) !ok", inodeIndex) - return - } - - inodeNumber = key.(uint64) - linkCountComputed = value.(uint64) - - vVS.jobGrabParallelism() - vVS.childrenWaitGroup.Add(1) - go vVS.validateVolumeFixLinkCount(inodeNumber, linkCountComputed) - } - - vVS.childrenWaitGroup.Wait() - - vVS.jobEndParallelism() - - if vVS.stopFlag || (0 < len(vVS.err)) { - return - } - - vVS.jobLogInfo("Completed LinkCount fix-up of all Inode's") - - // If vVS.lostAndFoundDirInodeNumber is empty, remove it - - _, moreEntries, err = vVS.inodeVolumeHandle.ReadDir(vVS.lostAndFoundDirInodeNumber, 2, 0) - if nil != err { - vVS.jobLogErr("Got ReadDir(vVS.lostAndFoundDirInodeNumber, 2, 0) failure: %v", err) - return - } - - if moreEntries { - vVS.jobLogInfo("Preserving non-empty /%v/", lostAndFoundDirName) - } else { - toDestroyInodeNumber, err = vVS.inodeVolumeHandle.Unlink(inode.RootDirInodeNumber, lostAndFoundDirName, false) - if nil != err { - vVS.jobLogErr("Got inode.Unlink(inode.RootDirInodeNumber, lostAndFoundDirName, false) failure: %v", err) - return - } - - if inode.InodeNumber(0) != toDestroyInodeNumber { - err = vVS.inodeVolumeHandle.Destroy(toDestroyInodeNumber) - if nil != err { - vVS.jobLogErr("Got inode.Destroy(toDestroyInodeNumber) failure: %v", err) - return - } - } - - vVS.jobLogInfo("Removed empty /%v/", lostAndFoundDirName) - } - - // Clean out unreferenced headhunter B+Tree "Objects") - - vVS.objectBPTree = sortedmap.NewBPlusTree(jobBPTreeMaxKeysPerNode, sortedmap.CompareUint64, vVS, vVS.bpTreeCache) - defer func(vVS *validateVolumeStruct) { - var err error - - err = vVS.objectBPTree.Discard() - if nil != err { - vVS.jobLogErr("Got vVS.objectBPTree.Discard() failure: %v", err) - } - }(vVS) - - vVS.jobStartParallelism(validateVolumeIncrementByteCountsParallelism) - - inodeIndex = 0 - - for { - if vVS.stopFlag { - vVS.childrenWaitGroup.Wait() - vVS.jobEndParallelism() - return - } - - inodeNumber, ok, err = vVS.headhunterVolumeHandle.IndexedInodeNumber(inodeIndex) - if nil != err { - vVS.jobLogErr("Got headhunter.IndexedInodeNumber(0x%016X) failure: %v", inodeIndex, err) - vVS.childrenWaitGroup.Wait() - vVS.jobEndParallelism() - return - } - if !ok { - break - } - - vVS.jobGrabParallelism() - vVS.childrenWaitGroup.Add(1) - go vVS.validateVolumeIncrementByteCounts(inodeNumber) - - inodeIndex++ - } - - vVS.childrenWaitGroup.Wait() - - vVS.jobEndParallelism() - - if vVS.stopFlag || (0 < len(vVS.err)) { - return - } - - objectIndex = 0 - - for { - if vVS.stopFlag { - return - } - - objectNumber, ok, err = vVS.headhunterVolumeHandle.IndexedBPlusTreeObjectNumber(objectIndex) - if nil != err { - vVS.jobLogErr("Got headhunter.IndexedBPlusTreeObjectNumber(0x%016X) failure: %v", objectIndex, err) - return - } - - if !ok { - break - } - - _, ok, err = vVS.objectBPTree.GetByKey(objectNumber) - if nil != err { - vVS.jobLogErr("Got vVS.objectBPTree.GetByKey(0x%016X) failure: %v", objectNumber, err) - return - } - - if ok { - objectIndex++ - } else { - err = vVS.headhunterVolumeHandle.DeleteBPlusTreeObject(objectNumber) - if nil == err { - vVS.jobLogInfo("Removed unreferenced headhunter B+Tree \"Object\" 0x%016X", objectNumber) - } else { - vVS.jobLogErr("Got headhunter.DeleteBPlusTreeObject(0x%016X) failure: %v", objectNumber, err) - return - } - } - } - - if vVS.stopFlag || (0 < len(vVS.err)) { - return - } - - vVS.jobLogInfo("Completed clean out unreferenced headhunter B+Tree \"Objects\"") - - // TODO: Walk all FileInodes tracking referenced LogSegments - // TODO: Remove non-Checkpoint Objects not in headhunter's LogSegment B+Tree - // TODO: Delete unreferenced LogSegments (both headhunter records & Objects) - - // Do a final checkpoint - - err = vVS.headhunterVolumeHandle.DoCheckpoint() - if nil != err { - vVS.jobLogErr("Got headhunter.DoCheckpoint failure: %v", err) - return - } - - vVS.jobLogInfo("Completed checkpoint after FSCK work") - - // Validate headhunter checkpoint container contents - - headhunterLayoutReport, discrepencies, err = vVS.headhunterVolumeHandle.FetchLayoutReport(headhunter.MergedBPlusTree, true) - if nil != err { - vVS.jobLogErr("Got headhunter.FetchLayoutReport() failure: %v", err) - return - } - if 0 != discrepencies { - vVS.jobLogErr("Found %v headhunter.FetchLayoutReport() discrepencies", discrepencies) - return - } - - vVS.accountName, vVS.checkpointContainerName = vVS.headhunterVolumeHandle.FetchAccountAndCheckpointContainerNames() - - _, checkpointContainerObjectList, err = swiftclient.ContainerGet(vVS.accountName, vVS.checkpointContainerName) - if nil != err { - vVS.jobLogErr("Got swiftclient.ContainerGet(\"%v\",\"%v\") failure: %v", vVS.accountName, vVS.checkpointContainerName, err) - return - } - - for _, checkpointContainerObjectName = range checkpointContainerObjectList { - checkpointContainerObjectNumber = uint64(0) // If remains 0 or results in returning to 0, - // checkpointContainerObjectName should be deleted - - if 16 == len(checkpointContainerObjectName) { - if validObjectNameRE.MatchString(checkpointContainerObjectName) { - checkpointContainerObjectNumber, err = strconv.ParseUint(checkpointContainerObjectName, 16, 64) - if nil != err { - vVS.jobLogErr("Got strconv.ParseUint(\"%v\",16,64) failure: %v", checkpointContainerObjectName, err) - return // Note: Already scheduled async Object DELETEs will continue in the background - } - - _, ok = headhunterLayoutReport[checkpointContainerObjectNumber] - if !ok { - checkpointContainerObjectNumber = uint64(0) - } - } - } - - if uint64(0) == checkpointContainerObjectNumber { - vVS.jobLogInfo("Removing unreferenced checkpointContainerObject %v", checkpointContainerObjectName) - swiftclient.ObjectDelete(vVS.accountName, vVS.checkpointContainerName, checkpointContainerObjectName, swiftclient.SkipRetry) - } - - if vVS.stopFlag { - return // Note: Already scheduled async Object DELETEs will continue in the background - } - } -} - -func (sVS *scrubVolumeStruct) Active() (active bool) { - active = sVS.active - return -} - -func (sVS *scrubVolumeStruct) Wait() { - sVS.globalWaitGroup.Wait() -} - -func (sVS *scrubVolumeStruct) Cancel() { - sVS.stopFlag = true - sVS.Wait() -} - -func (sVS *scrubVolumeStruct) Error() (err []string) { - var errString string - - sVS.Lock() - - err = make([]string, 0, len(sVS.err)) - for _, errString = range sVS.err { - err = append(err, errString) - } - - sVS.Unlock() - - return -} - -func (sVS *scrubVolumeStruct) Info() (info []string) { - var infoString string - - sVS.Lock() - - info = make([]string, 0, len(sVS.info)) - for _, infoString = range sVS.info { - info = append(info, infoString) - } - - sVS.Unlock() - - return -} - -func (sVS *scrubVolumeStruct) scrubVolumeInode(inodeNumber uint64) { - var ( - err error - inodeLock *dlm.RWLockStruct - ) - - defer sVS.childrenWaitGroup.Done() - - defer sVS.jobReleaseParallelism() - - inodeLock, err = sVS.jobStruct.inodeVolumeHandle.InitInodeLock(inode.InodeNumber(inodeNumber), nil) - if nil != err { - sVS.jobLogErr("Got initInodeLock(0x%016X) failure: %v", inodeNumber, err) - return - } - err = inodeLock.WriteLock() - if nil != err { - sVS.jobLogErr("Got inodeLock.WriteLock() for Inode# 0x%016X failure: %v", inodeNumber, err) - return - } - defer inodeLock.Unlock() - - err = sVS.inodeVolumeHandle.Validate(inode.InodeNumber(inodeNumber), true) - if nil != err { - sVS.jobLogInfo("Got inode.Validate(0x%016X) failure: %v ... removing it", inodeNumber, err) - - err = sVS.headhunterVolumeHandle.DeleteInodeRec(inodeNumber) - if nil != err { - sVS.jobLogErr("Got headhunter.DeleteInodeRec(0x%016X) failure: %v", inodeNumber, err) - } - - // Note: We could identify removal of an inode as an error here, but we don't - // know if it is actually referenced. A subsequent validateVolume() call - // would catch that condition. - } -} - -func (sVS *scrubVolumeStruct) scrubVolume() { - var ( - err error - inodeCount int - inodeIndex uint64 - inodeNumber uint64 - key sortedmap.Key - ok bool - ) - - sVS.jobLogInfo("SCRUB job initiated") - - defer func(sVS *scrubVolumeStruct) { - if sVS.stopFlag { - sVS.jobLogInfo("SCRUB job stopped") - } else if 0 == len(sVS.err) { - sVS.jobLogInfo("SCRUB job completed without error") - } else if 1 == len(sVS.err) { - sVS.jobLogInfo("SCRUB job exited with one error") - } else { - sVS.jobLogInfo("SCRUB job exited with errors") - } - }(sVS) - - defer func(sVS *scrubVolumeStruct) { - sVS.active = false - }(sVS) - - defer sVS.globalWaitGroup.Done() - - // Find specified volume - - globals.Lock() - - sVS.volume, ok = globals.volumeMap[sVS.volumeName] - if !ok { - globals.Unlock() - sVS.jobLogErr("Couldn't find fs.volumeStruct") - return - } - - globals.Unlock() - - sVS.inodeVolumeHandle, err = inode.FetchVolumeHandle(sVS.volumeName) - if nil != err { - sVS.jobLogErr("Couldn't find inode.VolumeHandle") - return - } - - sVS.headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(sVS.volumeName) - if nil != err { - sVS.jobLogErr("Couldn't find headhunter.VolumeHandle") - return - } - - // Setup B+Tree to hold arbitrarily sized map[uint64]uint64 (i.e. beyond what will fit in memoory) - - sVS.bpTreeFile, err = ioutil.TempFile("", "ProxyFS_ScrubVolume_") - if nil != err { - sVS.jobLogErr("Got ioutil.TempFile() failure: %v", err) - return - } - defer func(sVS *scrubVolumeStruct) { - var ( - bpTreeFileName string - ) - - bpTreeFileName = sVS.bpTreeFile.Name() - _ = sVS.bpTreeFile.Close() - _ = os.Remove(bpTreeFileName) - }(sVS) - - sVS.bpTreeFileNextOffset = 0 - - sVS.bpTreeCache = sortedmap.NewBPlusTreeCache(jobBPTreeEvictLowLimit, jobBPTreeEvictHighLimit) - - sVS.inodeBPTree = sortedmap.NewBPlusTree(jobBPTreeMaxKeysPerNode, sortedmap.CompareUint64, sVS, sVS.bpTreeCache) - defer func(sVS *scrubVolumeStruct) { - var err error - - err = sVS.inodeBPTree.Discard() - if nil != err { - sVS.jobLogErr("Got sVS.inodeBPTree.Discard() failure: %v", err) - } - }(sVS) - - sVS.jobLogInfo("Beginning enumeration of inodes to be deeply validated") - - sVS.volume.jobRWMutex.Lock() - - inodeIndex = 0 - - for { - if sVS.stopFlag { - sVS.volume.jobRWMutex.Unlock() - return - } - - inodeNumber, ok, err = sVS.headhunterVolumeHandle.IndexedInodeNumber(inodeIndex) - if nil != err { - sVS.volume.jobRWMutex.Unlock() - sVS.jobLogErrWhileLocked("Got headhunter.IndexedInodeNumber(0x%016X) failure: %v", inodeIndex, err) - return - } - if !ok { - break - } - - ok, err = sVS.inodeBPTree.Put(inodeNumber, uint64(0)) // Unused LinkCount - just set it to 0 - if nil != err { - sVS.volume.jobRWMutex.Unlock() - sVS.jobLogErrWhileLocked("Got sVS.inodeBPTree.Put(0x%016X, 0) failure: %v", inodeNumber, err) - return - } - if !ok { - sVS.volume.jobRWMutex.Unlock() - sVS.jobLogErrWhileLocked("Got sVS.inodeBPTree.Put(0x%016X, 0) !ok", inodeNumber) - return - } - - inodeIndex++ - } - - sVS.volume.jobRWMutex.Unlock() - - sVS.jobLogInfo("Completed enumeration of inodes to be deeply validated") - - sVS.jobLogInfo("Beginning deep validation of inodes") - - inodeCount, err = sVS.inodeBPTree.Len() - if nil != err { - sVS.jobLogErr("Got sVS.inodeBPTree.Len() failure: %v", err) - return - } - - sVS.jobStartParallelism(scrubVolumeInodeParallelism) - - for inodeIndex = uint64(0); inodeIndex < uint64(inodeCount); inodeIndex++ { - if sVS.stopFlag { - sVS.childrenWaitGroup.Wait() - sVS.jobEndParallelism() - return - } - - key, _, ok, err = sVS.inodeBPTree.GetByIndex(int(inodeIndex)) - if nil != err { - sVS.jobLogErr("Got sVS.inodeBPTree.GetByIndex(0x%016X) failure: %v", inodeIndex, err) - return - } - if !ok { - sVS.jobLogErr("Got sVS.inodeBPTree.GetByIndex(0x%016X) !ok", inodeIndex) - return - } - - inodeNumber = key.(uint64) - - sVS.jobGrabParallelism() - sVS.childrenWaitGroup.Add(1) - go sVS.scrubVolumeInode(inodeNumber) - } - - sVS.childrenWaitGroup.Wait() - - sVS.jobEndParallelism() - - sVS.jobLogInfo("Completed deep validation of inodes") -} diff --git a/fs/utils.go b/fs/utils.go deleted file mode 100644 index 5df8460d..00000000 --- a/fs/utils.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fs - -import ( - "container/list" - "crypto/rand" - "fmt" - "math/big" - "time" - - "github.com/NVIDIA/proxyfs/stats" -) - -func (tryLockBackoffContext *tryLockBackoffContextStruct) backoff() { - var ( - serializedBackoffListElement *list.Element - ) - - stats.IncrementOperations(&stats.InodeTryLockBackoffOps) - - if 0 < tryLockBackoffContext.backoffsCompleted { - if tryLockBackoffContext.backoffsCompleted < globals.tryLockSerializationThreshhold { - // Just do normal delay backoff - - stats.IncrementOperations(&stats.InodeTryLockDelayedBackoffOps) - - backoffSleep() - } else { - stats.IncrementOperations(&stats.InodeTryLockSerializedBackoffOps) - - // Push us onto the serializedBackoffList - - globals.Lock() - - serializedBackoffListElement = globals.serializedBackoffList.PushBack(tryLockBackoffContext) - - if 1 == globals.serializedBackoffList.Len() { - // Nobody else is currently serialized... so just delay - - globals.Unlock() - - backoffSleep() - } else { - // We are not the only one on serializedBackoffList... so we must wait - - tryLockBackoffContext.Add(1) - - globals.Unlock() - - tryLockBackoffContext.Wait() - - // Now delay to get prior tryLockBackoffContext a chance to complete - - backoffSleep() - } - - // Now remove us from the front of serializedBackoffList - - globals.Lock() - - _ = globals.serializedBackoffList.Remove(serializedBackoffListElement) - - // Now check if there is a tryLockBackoffContext "behind" us - - serializedBackoffListElement = globals.serializedBackoffList.Front() - - globals.Unlock() - - if nil != serializedBackoffListElement { - // Awake the tryLockBackoffContext "behind" us - - serializedBackoffListElement.Value.(*tryLockBackoffContextStruct).Done() - } - } - } - - tryLockBackoffContext.backoffsCompleted++ -} - -func backoffSleep() { - var ( - err error - randomBigInt *big.Int - thisDelay time.Duration - ) - - randomBigInt, err = rand.Int(rand.Reader, big.NewInt(int64(globals.tryLockBackoffMax)-int64(globals.tryLockBackoffMin))) - if nil != err { - err = fmt.Errorf("fs.backoffSleep() failed in call to rand.Int(): %v", err) - panic(err) - } - - thisDelay = time.Duration(int64(globals.tryLockBackoffMin) + randomBigInt.Int64()) - - time.Sleep(thisDelay) -} diff --git a/fsworkout/Makefile b/fsworkout/Makefile deleted file mode 100644 index 29e82769..00000000 --- a/fsworkout/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/fsworkout - -include ../GoMakefile diff --git a/fsworkout/dummy_test.go b/fsworkout/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/fsworkout/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/fsworkout/main.go b/fsworkout/main.go deleted file mode 100644 index 7cf58d68..00000000 --- a/fsworkout/main.go +++ /dev/null @@ -1,352 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "os" - "runtime" - "strconv" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -const ( - dirInodeNamePrefix = "__fsworkout_dir_" - fileInodeNamePrefix = "__fsworkout_file_" -) - -var ( - doNextStepChan chan bool - inodesPerThread uint64 - measureCreate bool - measureDestroy bool - measureStat bool - perThreadDir bool - rootDirMutex trackedlock.Mutex - stepErrChan chan error - threads uint64 - volumeHandle fs.VolumeHandle - volumeName string -) - -func usage(file *os.File) { - fmt.Fprintf(file, "Usage:\n") - fmt.Fprintf(file, " %v [cCsSdD] threads inodes-per-thread conf-file [section.option=value]*\n", os.Args[0]) - fmt.Fprintf(file, " where:\n") - fmt.Fprintf(file, " c run create test in common root dir\n") - fmt.Fprintf(file, " C run create test in per thread dir\n") - fmt.Fprintf(file, " s run stat test in common root dir\n") - fmt.Fprintf(file, " S run stat test in per thread dir\n") - fmt.Fprintf(file, " d run destroy test in common root dir\n") - fmt.Fprintf(file, " D run destroy test in per thread dir\n") - fmt.Fprintf(file, " threads number of threads\n") - fmt.Fprintf(file, " inodes-per-thread number of inodes each thread will reference\n") - fmt.Fprintf(file, " conf-file input to conf.MakeConfMapFromFile()\n") - fmt.Fprintf(file, " [section.option=value]* optional input to conf.UpdateFromStrings()\n") - fmt.Fprintf(file, "\n") - fmt.Fprintf(file, "Note: Precisely one test selector must be specified\n") - fmt.Fprintf(file, " It is expected that c, s, then d are run in sequence\n") - fmt.Fprintf(file, " or that C, S, then D are run in sequence\n") - fmt.Fprintf(file, " It is expected that cleanproxyfs is run before & after the sequence\n") -} - -func main() { - var ( - confMap conf.ConfMap - durationOfMeasuredOperations time.Duration - err error - latencyPerOpInMilliSeconds float64 - opsPerSecond float64 - primaryPeer string - threadIndex uint64 - timeAfterMeasuredOperations time.Time - timeBeforeMeasuredOperations time.Time - volumeGroupToCheck string - volumeGroupToUse string - volumeGroupList []string - volumeList []string - whoAmI string - ) - - // Parse arguments - - if 5 > len(os.Args) { - usage(os.Stderr) - os.Exit(1) - } - - switch os.Args[1] { - case "c": - measureCreate = true - case "C": - measureCreate = true - perThreadDir = true - case "s": - measureStat = true - case "S": - measureStat = true - perThreadDir = true - case "d": - measureDestroy = true - case "D": - measureDestroy = true - perThreadDir = true - default: - fmt.Fprintf(os.Stderr, "os.Args[1] ('%v') must be one of 'c', 'C', 'r', 'R', 'd', or 'D'\n", os.Args[1]) - os.Exit(1) - } - - threads, err = strconv.ParseUint(os.Args[2], 10, 64) - if nil != err { - fmt.Fprintf(os.Stderr, "strconv.ParseUint(\"%v\", 10, 64) of threads failed: %v\n", os.Args[2], err) - os.Exit(1) - } - if 0 == threads { - fmt.Fprintf(os.Stderr, "threads must be a positive number\n") - os.Exit(1) - } - - inodesPerThread, err = strconv.ParseUint(os.Args[3], 10, 64) - if nil != err { - fmt.Fprintf(os.Stderr, "strconv.ParseUint(\"%v\", 10, 64) of inodes-per-thread failed: %v\n", os.Args[3], err) - os.Exit(1) - } - if 0 == inodesPerThread { - fmt.Fprintf(os.Stderr, "inodes-per-thread must be a positive number\n") - os.Exit(1) - } - - confMap, err = conf.MakeConfMapFromFile(os.Args[4]) - if nil != err { - fmt.Fprintf(os.Stderr, "conf.MakeConfMapFromFile(\"%v\") failed: %v\n", os.Args[4], err) - os.Exit(1) - } - - if 5 < len(os.Args) { - err = confMap.UpdateFromStrings(os.Args[5:]) - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.UpdateFromStrings(%#v) failed: %v\n", os.Args[5:], err) - os.Exit(1) - } - } - - // Upgrade confMap if necessary - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "Failed to upgrade config: %v", err) - os.Exit(1) - } - - // Start up needed ProxyFS components - - err = transitions.Up(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "transitions.Up() failed: %v\n", err) - os.Exit(1) - } - - // Select first Volume of the first "active" VolumeGroup in [FSGlobals]VolumeGroupList - whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueString(\"Cluster\", \"WhoAmI\") failed: %v\n", err) - os.Exit(1) - } - - volumeGroupList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"FSGlobals\", \"VolumeGroupList\") failed: %v\n", err) - os.Exit(1) - } - - volumeGroupToUse = "" - - for _, volumeGroupToCheck = range volumeGroupList { - primaryPeer, err = confMap.FetchOptionValueString("VolumeGroup:"+volumeGroupToCheck, "PrimaryPeer") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueString(\"VolumeGroup:%s\", \"PrimaryPeer\") failed: %v\n", volumeGroupToCheck, err) - os.Exit(1) - } - if whoAmI == primaryPeer { - volumeGroupToUse = volumeGroupToCheck - break - } - } - - if "" == volumeGroupToUse { - fmt.Fprintf(os.Stderr, "confMap didn't contain an \"active\" VolumeGroup") - os.Exit(1) - } - - volumeList, err = confMap.FetchOptionValueStringSlice("VolumeGroup:"+volumeGroupToUse, "VolumeList") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"VolumeGroup:%s\", \"PrimaryPeer\") failed: %v\n", volumeGroupToUse, err) - os.Exit(1) - } - if 1 > len(volumeList) { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"VolumeGroup:%s\", \"VolumeList\") returned empty volumeList", volumeGroupToUse) - os.Exit(1) - } - - volumeName = volumeList[0] - - volumeHandle, err = fs.FetchVolumeHandleByVolumeName(volumeName) - if nil != err { - fmt.Fprintf(os.Stderr, "fs.FetchVolumeHandleByVolumeName(\"%value\",) failed: %v\n", volumeName, err) - os.Exit(1) - } - - // Perform tests - - stepErrChan = make(chan error, 0) //threads) - doNextStepChan = make(chan bool, 0) //threads) - - // Do initialization step - for threadIndex = uint64(0); threadIndex < threads; threadIndex++ { - go fsWorkout(threadIndex) - } - for threadIndex = uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fsWorkout() initialization step returned: %v\n", err) - os.Exit(1) - } - } - - // Do measured operations step - timeBeforeMeasuredOperations = time.Now() - for threadIndex = uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex = uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fsWorkout() measured operations step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterMeasuredOperations = time.Now() - - // Do shutdown step - for threadIndex = uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex = uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fsWorkout() shutdown step returned: %v\n", err) - os.Exit(1) - } - } - - // Stop ProxyFS components launched above - - err = transitions.Down(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "transitions.Down() failed: %v\n", err) - os.Exit(1) - } - - // Report results - - durationOfMeasuredOperations = timeAfterMeasuredOperations.Sub(timeBeforeMeasuredOperations) - - opsPerSecond = float64(threads*inodesPerThread*1000*1000*1000) / float64(durationOfMeasuredOperations.Nanoseconds()) - latencyPerOpInMilliSeconds = float64(durationOfMeasuredOperations.Nanoseconds()) / float64(inodesPerThread*1000*1000) - - fmt.Printf("opsPerSecond = %10.2f\n", opsPerSecond) - fmt.Printf("latencyPerOp = %10.2f ms\n", latencyPerOpInMilliSeconds) -} - -func fsWorkout(threadIndex uint64) { - var ( - dirInodeName string - dirInodeNumber inode.InodeNumber - err error - fileInodeName []string - fileInodeNumber inode.InodeNumber - i uint64 - ) - - // Do initialization step - if perThreadDir { - dirInodeName = fmt.Sprintf("%s%016X", dirInodeNamePrefix, threadIndex) - if measureCreate { - dirInodeNumber, err = volumeHandle.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, dirInodeName, inode.PosixModePerm) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - } else { // measureStat || measureDestroy - dirInodeNumber, err = volumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, dirInodeName) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - } - } else { // !perThreadDir - dirInodeNumber = inode.RootDirInodeNumber - } - fileInodeName = make([]string, inodesPerThread) - for i = 0; i < inodesPerThread; i++ { - fileInodeName[i] = fmt.Sprintf("%s%016X_%016X", fileInodeNamePrefix, threadIndex, i) - } - - // Indicate initialization step is done - stepErrChan <- nil - - // Await signal to proceed with measured operations step - _ = <-doNextStepChan - - // Do measured operations - for i = 0; i < inodesPerThread; i++ { - if measureCreate { - fileInodeNumber, err = volumeHandle.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirInodeNumber, fileInodeName[i], inode.PosixModePerm) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - } else if measureStat { - fileInodeNumber, err = volumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirInodeNumber, fileInodeName[i]) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - _, err = volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInodeNumber) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - } else { // measureDestroy - err = volumeHandle.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirInodeNumber, fileInodeName[i]) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - } - } - - // Indicate measured operations step is done - stepErrChan <- nil - - // Await signal to proceed with shutdown step - _ = <-doNextStepChan - - // Do shutdown step - if perThreadDir && measureDestroy { - err = volumeHandle.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, dirInodeName) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - } - - // Indicate shutdown step is done - stepErrChan <- nil -} diff --git a/fuse/Makefile b/fuse/Makefile deleted file mode 100644 index 38e27e07..00000000 --- a/fuse/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/fuse - -include ../GoMakefile diff --git a/fuse/config.go b/fuse/config.go deleted file mode 100644 index 5d9d84af..00000000 --- a/fuse/config.go +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package fuse is a FUSE filesystem for ProxyFS (an alternative to the Samba-VFS frontend). -package fuse - -import ( - "fmt" - "os" - "os/exec" - "path" - "syscall" - "time" - - fuselib "bazil.org/fuse" - fusefslib "bazil.org/fuse/fs" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -const ( - maxRetryCount uint32 = 100 - retryGap = 100 * time.Millisecond -) - -type volumeStruct struct { - volumeName string - mountPointName string - mounted bool -} - -type globalsStruct struct { - gate trackedlock.RWMutex // API Requests RLock()/RUnlock() - // confMap changes Lock()/Unlock() - // Note: fuselib.Unmount() results in an Fsync() call on RootDir - // Hence, no current confMap changes currently call Lock() - volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName - mountPointMap map[string]*volumeStruct // key == volumeStruct.mountPointName -} - -var globals globalsStruct - -func init() { - transitions.Register("fuse", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - globals.volumeMap = make(map[string]*volumeStruct) - globals.mountPointMap = make(map[string]*volumeStruct) - - closeGate() // Ensure gate starts out in the Exclusively Locked state - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} - -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - volume *volumeStruct - ) - - volume = &volumeStruct{ - volumeName: volumeName, - mounted: false, - } - - volume.mountPointName, err = confMap.FetchOptionValueString("Volume:"+volumeName, "FUSEMountPointName") - if nil != err { - return - } - - globals.volumeMap[volume.volumeName] = volume - globals.mountPointMap[volume.mountPointName] = volume - - err = performMount(volume) - - return // return err from performMount() sufficient -} - -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - lazyUnmountCmd *exec.Cmd - ok bool - volume *volumeStruct - ) - - volume, ok = globals.volumeMap[volumeName] - - if ok { - if volume.mounted { - err = fuselib.Unmount(volume.mountPointName) - if nil == err { - logger.Infof("Unmounted %v", volume.mountPointName) - } else { - lazyUnmountCmd = exec.Command("fusermount", "-uz", volume.mountPointName) - err = lazyUnmountCmd.Run() - if nil == err { - logger.Infof("Lazily unmounted %v", volume.mountPointName) - } else { - logger.Infof("Unable to lazily unmount %v - got err == %v", volume.mountPointName, err) - } - } - } - - delete(globals.volumeMap, volume.volumeName) - delete(globals.mountPointMap, volume.mountPointName) - } - - err = nil - return -} - -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - // TODO: Might want to actually FUSE Unmount right here - - err = nil - return -} - -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - closeGate() - - err = nil - return -} - -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - openGate() - - err = nil - return -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - if 0 != len(globals.volumeMap) { - err = fmt.Errorf("fuse.Down() called with 0 != len(globals.volumeMap") - return - } - if 0 != len(globals.mountPointMap) { - err = fmt.Errorf("fuse.Down() called with 0 != len(globals.mountPointMap") - return - } - - openGate() // In case we are restarted... Up() expects Gate to initially be open - - err = nil - return -} - -func fetchInodeDevice(path string) (missing bool, inodeDevice int64, err error) { - fi, err := os.Stat(path) - if nil != err { - if os.IsNotExist(err) { - missing = true - err = nil - } else { - err = fmt.Errorf("fetchInodeDevice(%v): os.Stat() failed: %v", path, err) - } - return - } - if nil == fi.Sys() { - err = fmt.Errorf("fetchInodeDevice(%v): fi.Sys() was nil", path) - return - } - stat, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - err = fmt.Errorf("fetchInodeDevice(%v): fi.Sys().(*syscall.Stat_t) returned ok == false", path) - return - } - missing = false - inodeDevice = int64(stat.Dev) - return -} - -func performMount(volume *volumeStruct) (err error) { - var ( - conn *fuselib.Conn - curRetryCount uint32 - lazyUnmountCmd *exec.Cmd - missing bool - mountPointContainingDirDevice int64 - mountPointDevice int64 - mountPointNameBase string - volumeHandle fs.VolumeHandle - ) - - volume.mounted = false - - missing, mountPointContainingDirDevice, err = fetchInodeDevice(path.Dir(volume.mountPointName)) - if nil != err { - return - } - if missing { - logger.Infof("Unable to serve %s.FUSEMountPoint == %s (mount point dir's parent does not exist)", volume.volumeName, volume.mountPointName) - return - } - - missing, mountPointDevice, err = fetchInodeDevice(volume.mountPointName) - if nil == err { - if missing { - logger.Infof("Unable to serve %s.FUSEMountPoint == %s (mount point dir does not exist)", volume.volumeName, volume.mountPointName) - return - } - } - - if (nil != err) || (mountPointDevice != mountPointContainingDirDevice) { - // Presumably, the mount point is (still) currently mounted, so attempt to unmount it first - - lazyUnmountCmd = exec.Command("fusermount", "-uz", volume.mountPointName) - err = lazyUnmountCmd.Run() - if nil != err { - return - } - - curRetryCount = 0 - - for { - time.Sleep(retryGap) // Try again in a bit - missing, mountPointDevice, err = fetchInodeDevice(volume.mountPointName) - if nil == err { - if missing { - err = fmt.Errorf("Race condition: %s.FUSEMountPoint == %s disappeared [case 1]", volume.volumeName, volume.mountPointName) - return - } - if mountPointDevice == mountPointContainingDirDevice { - break - } - } - curRetryCount++ - if curRetryCount >= maxRetryCount { - err = fmt.Errorf("MaxRetryCount exceeded for %s.FUSEMountPoint == %s [case 1]", volume.volumeName, volume.mountPointName) - return - } - } - } - - mountPointNameBase = path.Base(volume.mountPointName) - - conn, err = fuselib.Mount( - volume.mountPointName, - fuselib.FSName(mountPointNameBase), - fuselib.AllowOther(), - // OS X specific— - fuselib.LocalVolume(), - fuselib.VolumeName(mountPointNameBase), - fuselib.NoAppleDouble(), - fuselib.NoAppleXattr(), - ) - - if nil != err { - logger.WarnfWithError(err, "Couldn't mount %s.FUSEMountPoint == %s", volume.volumeName, volume.mountPointName) - err = nil - return - } - - volumeHandle, err = fs.FetchVolumeHandleByVolumeName(volume.volumeName) - if nil != err { - return - } - - fs := &ProxyFUSE{volumeHandle: volumeHandle} - - // We synchronize the mounting of the mount point to make sure our FUSE goroutine - // has reached the point that it can service requests. - // - // Otherwise, if proxyfsd is killed after we block on a FUSE request but before our - // FUSE goroutine has had a chance to run we end up with an unkillable proxyfsd process. - // - // This would result in a "proxyfsd " process that is only cleared by rebooting - // the system. - fs.wg.Add(1) - - go func(mountPointName string, conn *fuselib.Conn) { - defer conn.Close() - fusefslib.Serve(conn, fs) - }(volume.mountPointName, conn) - - // Wait for FUSE to mount the file system. The "fs.wg.Done()" is in the - // Root() routine. - fs.wg.Wait() - - // If we made it to here, all was ok - - logger.Infof("Now serving %s.FUSEMountPoint == %s", volume.volumeName, volume.mountPointName) - - volume.mounted = true - - err = nil - return -} - -func openGate() { - globals.gate.Unlock() -} - -func closeGate() { - globals.gate.Lock() -} - -// Note: The following func's do nothing today. Thus, no "gate" is enforced in this package. -// The reason is that as part of the fuselib.Unmount() in UnserveVolume(), a call to -// Fsync() will be made. If the closeGate() were honored, the call to Fsync() would -// indefinitely block. - -func enterGate() { - // globals.gate.RLock() -} - -func leaveGate() { - // globals.gate.RUnlock() -} diff --git a/fuse/dir.go b/fuse/dir.go deleted file mode 100644 index 687e9ba3..00000000 --- a/fuse/dir.go +++ /dev/null @@ -1,357 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fuse - -import ( - "fmt" - "os" - "time" - - fuselib "bazil.org/fuse" - fusefslib "bazil.org/fuse/fs" - "golang.org/x/net/context" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" -) - -type Dir struct { - volumeHandle fs.VolumeHandle - inodeNumber inode.InodeNumber -} - -func (d Dir) Access(ctx context.Context, req *fuselib.AccessRequest) error { - enterGate() - defer leaveGate() - - if d.volumeHandle.Access(inode.InodeUserID(req.Uid), inode.InodeGroupID(req.Gid), nil, d.inodeNumber, inode.InodeMode(req.Mask)) { - return nil - } else { - return newFuseError(blunder.NewError(blunder.PermDeniedError, "EACCES")) - } -} - -func (d Dir) Attr(ctx context.Context, attr *fuselib.Attr) (err error) { - var ( - stat fs.Stat - ) - - enterGate() - defer leaveGate() - - stat, err = d.volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, d.inodeNumber) - if nil != err { - err = newFuseError(err) - return - } - if uint64(inode.DirType) != stat[fs.StatFType] { - err = fmt.Errorf("[fuse]Dir.Attr() called on non-Dir") - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return - } - - attr.Valid = time.Duration(time.Microsecond) // TODO: Make this settable if FUSE inside ProxyFS endures - attr.Inode = uint64(d.inodeNumber) // or stat[fs.StatINum] - attr.Size = stat[fs.StatSize] - attr.Atime = time.Unix(0, int64(stat[fs.StatATime])) - attr.Mtime = time.Unix(0, int64(stat[fs.StatMTime])) - attr.Ctime = time.Unix(0, int64(stat[fs.StatCTime])) - attr.Crtime = time.Unix(0, int64(stat[fs.StatCRTime])) - attr.Mode = os.ModeDir | os.FileMode(stat[fs.StatMode]&0777) - attr.Nlink = uint32(stat[fs.StatNLink]) - attr.Uid = uint32(stat[fs.StatUserID]) - attr.Gid = uint32(stat[fs.StatGroupID]) - - return -} - -func (d Dir) Setattr(ctx context.Context, req *fuselib.SetattrRequest, resp *fuselib.SetattrResponse) (err error) { - var ( - stat fs.Stat - statUpdates fs.Stat - ) - - enterGate() - defer leaveGate() - - stat, err = d.volumeHandle.Getstat(inode.InodeUserID(req.Uid), inode.InodeGroupID(req.Gid), nil, d.inodeNumber) - if nil != err { - err = newFuseError(err) - return - } - if uint64(inode.DirType) != stat[fs.StatFType] { - err = fmt.Errorf("[fuse]Dir.Attr() called on non-Dir") - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return - } - - statUpdates = make(fs.Stat) - - if 0 != (fuselib.SetattrMode & req.Valid) { - statUpdates[fs.StatMode] = uint64(req.Mode & 0777) - } - if 0 != (fuselib.SetattrUid & req.Valid) { - statUpdates[fs.StatUserID] = uint64(req.Uid) - } - if 0 != (fuselib.SetattrGid & req.Valid) { - statUpdates[fs.StatGroupID] = uint64(req.Gid) - } - if 0 != (fuselib.SetattrAtime & req.Valid) { - statUpdates[fs.StatATime] = uint64(req.Atime.UnixNano()) - } - if 0 != (fuselib.SetattrMtime & req.Valid) { - statUpdates[fs.StatMTime] = uint64(req.Mtime.UnixNano()) - } - if 0 != (fuselib.SetattrAtimeNow & req.Valid) { - statUpdates[fs.StatATime] = uint64(time.Now().UnixNano()) - } - if 0 != (fuselib.SetattrMtimeNow & req.Valid) { - statUpdates[fs.StatMTime] = uint64(time.Now().UnixNano()) - } - if 0 != (fuselib.SetattrCrtime & req.Valid) { - statUpdates[fs.StatCRTime] = uint64(req.Crtime.UnixNano()) - } - - err = d.volumeHandle.Setstat(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, statUpdates) - if nil != err { - err = newFuseError(err) - } - - return -} - -func (d Dir) Lookup(ctx context.Context, name string) (fusefslib.Node, error) { - enterGate() - defer leaveGate() - - childInodeNumber, err := d.volumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, d.inodeNumber, name) - if err != nil { - return nil, fuselib.ENOENT - } - - isDir, err := d.volumeHandle.IsDir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, childInodeNumber) - if isDir { - return Dir{volumeHandle: d.volumeHandle, inodeNumber: childInodeNumber}, nil - } else if err != nil { - err = newFuseError(err) - return nil, err - } - - isFile, err := d.volumeHandle.IsFile(inode.InodeRootUserID, inode.InodeGroupID(0), nil, childInodeNumber) - if isFile { - return File{volumeHandle: d.volumeHandle, inodeNumber: childInodeNumber}, nil - } else if err != nil { - err = newFuseError(err) - return nil, err - } - - isSymlink, err := d.volumeHandle.IsSymlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, childInodeNumber) - if isSymlink { - return Symlink{volumeHandle: d.volumeHandle, inodeNumber: childInodeNumber}, nil - } else if err != nil { - err = newFuseError(err) - return nil, err - } - - actualType, err := d.volumeHandle.GetType(inode.InodeRootUserID, inode.InodeGroupID(0), nil, childInodeNumber) - if err != nil { - err = newFuseError(err) - return nil, err - } else { - err = fmt.Errorf("Unrecognized inode type %v", actualType) - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return nil, err - } -} - -func inodeTypeToDirentType(inodeType inode.InodeType) fuselib.DirentType { - switch inodeType { - case inode.FileType: - return fuselib.DT_File - case inode.DirType: - return fuselib.DT_Dir - case inode.SymlinkType: - return fuselib.DT_Link - default: - return fuselib.DT_Unknown - } -} - -func (d Dir) ReadDirAll(ctx context.Context) ([]fuselib.Dirent, error) { - enterGate() - defer leaveGate() - - entries := make([]inode.DirEntry, 0) - entryCount := uint64(0) - lastEntryName := "" - - more := true - for more { - var readEntries []inode.DirEntry - var readCount uint64 - var err error - - readEntries, readCount, more, err = d.volumeHandle.Readdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, d.inodeNumber, 1024, lastEntryName) - if err != nil { - logger.ErrorfWithError(err, "Error in ReadDirAll") - return nil, fuselib.EIO - } - entries = append(entries, readEntries...) - entryCount += readCount - lastEntryName = readEntries[len(readEntries)-1].Basename - } - - fuseEntries := make([]fuselib.Dirent, entryCount) - - for i, entry := range entries { - - fuseEntries[i] = fuselib.Dirent{ - Inode: uint64(entry.InodeNumber), - Type: inodeTypeToDirentType(entry.Type), - Name: entry.Basename, - } - } - return fuseEntries, nil -} - -func (d Dir) Remove(ctx context.Context, req *fuselib.RemoveRequest) (err error) { - enterGate() - defer leaveGate() - - if req.Dir { - err = d.volumeHandle.Rmdir(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, req.Name) - } else { - err = d.volumeHandle.Unlink(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, req.Name) - } - if nil != err { - err = newFuseError(err) - } - return -} - -func (d Dir) Mknod(ctx context.Context, req *fuselib.MknodRequest) (fusefslib.Node, error) { - enterGate() - defer leaveGate() - - // Note: NFSd apparently prefers to use Mknod() instead of Create() when creating normal files... - if 0 != (inode.InodeMode(req.Mode) & ^inode.PosixModePerm) { - err := fmt.Errorf("Invalid Mode... only normal file creations supported") - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return nil, err - } - inodeNumber, err := d.volumeHandle.Create(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, req.Name, inode.InodeMode(req.Mode)) - if err != nil { - err = newFuseError(err) - return nil, err - } - file := File{volumeHandle: d.volumeHandle, inodeNumber: inodeNumber} - return file, nil -} - -func (d Dir) Create(ctx context.Context, req *fuselib.CreateRequest, resp *fuselib.CreateResponse) (fusefslib.Node, fusefslib.Handle, error) { - enterGate() - defer leaveGate() - - if 0 != (inode.InodeMode(req.Mode) & ^inode.PosixModePerm) { - err := fmt.Errorf("Invalid Mode... only normal file creations supported") - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return nil, nil, err - } - inodeNumber, err := d.volumeHandle.Create(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, req.Name, inode.InodeMode(req.Mode)) - if err != nil { - err = newFuseError(err) - return nil, nil, err - } - file := File{volumeHandle: d.volumeHandle, inodeNumber: inodeNumber} - return file, file, nil -} - -func (d Dir) Flush(ctx context.Context, req *fuselib.FlushRequest) error { - enterGate() - defer leaveGate() - - err := d.volumeHandle.Flush(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber) - if err != nil { - err = newFuseError(err) - } - return err -} - -func (d Dir) Fsync(ctx context.Context, req *fuselib.FsyncRequest) error { - enterGate() - defer leaveGate() - - err := d.volumeHandle.Flush(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, - d.inodeNumber) - if err != nil { - err = newFuseError(err) - } - return err -} - -func (d Dir) Mkdir(ctx context.Context, req *fuselib.MkdirRequest) (fusefslib.Node, error) { - enterGate() - defer leaveGate() - - trimmedMode := inode.InodeMode(req.Mode) & inode.PosixModePerm - newDirInodeNumber, err := d.volumeHandle.Mkdir(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, req.Name, trimmedMode) - if err != nil { - err = newFuseError(err) - return nil, err - } - return Dir{volumeHandle: d.volumeHandle, inodeNumber: newDirInodeNumber}, nil -} - -func (d Dir) Rename(ctx context.Context, req *fuselib.RenameRequest, newDir fusefslib.Node) error { - enterGate() - defer leaveGate() - - dstDir, ok := newDir.(Dir) - if !ok { - return fuselib.EIO - } - err := d.volumeHandle.Rename(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, req.OldName, dstDir.inodeNumber, req.NewName) - if err != nil { - err = newFuseError(err) - } - return err -} - -func (d Dir) Symlink(ctx context.Context, req *fuselib.SymlinkRequest) (fusefslib.Node, error) { - enterGate() - defer leaveGate() - - symlinkInodeNumber, err := d.volumeHandle.Symlink(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, req.NewName, req.Target) - if err != nil { - err = newFuseError(err) - return nil, err - } - - return Symlink{volumeHandle: d.volumeHandle, inodeNumber: symlinkInodeNumber}, nil -} - -func (d Dir) Link(ctx context.Context, req *fuselib.LinkRequest, old fusefslib.Node) (fusefslib.Node, error) { - enterGate() - defer leaveGate() - - oldFile, ok := old.(File) - if !ok { - err := fmt.Errorf("old.(File) failed") - return nil, err - } - - err := d.volumeHandle.Link(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, d.inodeNumber, req.NewName, oldFile.inodeNumber) - if err != nil { - err = newFuseError(err) - } - - return old, err -} diff --git a/fuse/dummy_test.go b/fuse/dummy_test.go deleted file mode 100644 index 1bc8b3ae..00000000 --- a/fuse/dummy_test.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fuse - -import "testing" - -func TestDUMMY(t *testing.T) { -} diff --git a/fuse/file.go b/fuse/file.go deleted file mode 100644 index 39ea8e79..00000000 --- a/fuse/file.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fuse - -import ( - "fmt" - "io" - "os" - "time" - - fuselib "bazil.org/fuse" - "golang.org/x/net/context" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" -) - -type File struct { - volumeHandle fs.VolumeHandle - inodeNumber inode.InodeNumber -} - -func (f File) Access(ctx context.Context, req *fuselib.AccessRequest) (err error) { - enterGate() - defer leaveGate() - - if f.volumeHandle.Access(inode.InodeUserID(req.Uid), inode.InodeGroupID(req.Gid), nil, f.inodeNumber, inode.InodeMode(req.Mask)) { - err = nil - } else { - err = newFuseError(blunder.NewError(blunder.PermDeniedError, "EACCES")) - } - - return -} - -func (f File) Attr(ctx context.Context, attr *fuselib.Attr) (err error) { - var ( - stat fs.Stat - ) - - enterGate() - defer leaveGate() - - stat, err = f.volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, f.inodeNumber) - if nil != err { - err = newFuseError(err) - return - } - if uint64(inode.FileType) != stat[fs.StatFType] { - err = fmt.Errorf("[fuse]Dir.Attr() called on non-File") - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return - } - - attr.Valid = time.Duration(time.Microsecond) // TODO: Make this settable if FUSE inside ProxyFS endures - attr.Inode = uint64(f.inodeNumber) // or stat[fs.StatINum] - attr.Size = stat[fs.StatSize] - attr.Blocks = (stat[fs.StatSize] + 511) / 512 - attr.Atime = time.Unix(0, int64(stat[fs.StatATime])) - attr.Mtime = time.Unix(0, int64(stat[fs.StatMTime])) - attr.Ctime = time.Unix(0, int64(stat[fs.StatCTime])) - attr.Crtime = time.Unix(0, int64(stat[fs.StatCRTime])) - attr.Mode = os.FileMode(stat[fs.StatMode] & 0777) - attr.Nlink = uint32(stat[fs.StatNLink]) - attr.Uid = uint32(stat[fs.StatUserID]) - attr.Gid = uint32(stat[fs.StatGroupID]) - attr.BlockSize = 4096 // Just a guess at a reasonable block size - - return -} - -func (f File) Setattr(ctx context.Context, req *fuselib.SetattrRequest, resp *fuselib.SetattrResponse) (err error) { - var ( - stat fs.Stat - statUpdates fs.Stat - ) - - enterGate() - defer leaveGate() - - stat, err = f.volumeHandle.Getstat(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, f.inodeNumber) - if nil != err { - err = newFuseError(err) - return - } - if uint64(inode.FileType) != stat[fs.StatFType] { - err = fmt.Errorf("[fuse]Dir.Attr() called on non-File") - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return - } - - statUpdates = make(fs.Stat) - - if 0 != (fuselib.SetattrMode & req.Valid) { - statUpdates[fs.StatMode] = uint64(req.Mode & 0777) - } - if 0 != (fuselib.SetattrUid & req.Valid) { - statUpdates[fs.StatUserID] = uint64(req.Uid) - } - if 0 != (fuselib.SetattrGid & req.Valid) { - statUpdates[fs.StatGroupID] = uint64(req.Gid) - } - if 0 != (fuselib.SetattrAtime & req.Valid) { - statUpdates[fs.StatATime] = uint64(req.Atime.UnixNano()) - } - if 0 != (fuselib.SetattrMtime & req.Valid) { - statUpdates[fs.StatMTime] = uint64(req.Mtime.UnixNano()) - } - if 0 != (fuselib.SetattrAtimeNow & req.Valid) { - statUpdates[fs.StatATime] = uint64(time.Now().UnixNano()) - } - if 0 != (fuselib.SetattrMtimeNow & req.Valid) { - statUpdates[fs.StatMTime] = uint64(time.Now().UnixNano()) - } - if 0 != (fuselib.SetattrCrtime & req.Valid) { - statUpdates[fs.StatCRTime] = uint64(req.Crtime.UnixNano()) - } - - err = f.volumeHandle.Setstat(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, f.inodeNumber, statUpdates) - if nil != err { - err = newFuseError(err) - return - } - - if 0 != (fuselib.SetattrSize & req.Valid) { - err = f.volumeHandle.Resize(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, f.inodeNumber, req.Size) - if nil != err { - err = newFuseError(err) - return - } - } - - return -} - -// Flush is called by Fuse when the VFS layer calls the fuse drivers flush() -// routine. According to some documentation, this is called as a result of -// a close() on a file descriptor: -// https://dri.freedesktop.org/docs/drm/filesystems/vfs.html#id2 -// -func (f File) Flush(ctx context.Context, req *fuselib.FlushRequest) (err error) { - enterGate() - defer leaveGate() - - // there's no flushing necessary for a close() - return -} - -func (f File) Fsync(ctx context.Context, req *fuselib.FsyncRequest) (err error) { - enterGate() - defer leaveGate() - - err = f.volumeHandle.Flush(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, f.inodeNumber) - if nil != err { - err = newFuseError(err) - } - return -} - -func (f File) Read(ctx context.Context, req *fuselib.ReadRequest, resp *fuselib.ReadResponse) (err error) { - enterGate() - defer leaveGate() - - buf, err := f.volumeHandle.Read(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, f.inodeNumber, uint64(req.Offset), uint64(req.Size), nil) - if err != nil && err != io.EOF { - err = newFuseError(err) - return - } - resp.Data = buf - err = nil - return -} - -func (f File) Write(ctx context.Context, req *fuselib.WriteRequest, resp *fuselib.WriteResponse) (err error) { - enterGate() - defer leaveGate() - - // We need to buffer contents of req.Data because fs.Write() will likely retain a reference to it - // (down in the Chunked PUT retry buffer) and Bazil FUSE will be reusing this WriteRequest (including - // its .Data buf) for the next request. - - bufferedData := make([]byte, len(req.Data), len(req.Data)) - copy(bufferedData, req.Data) - - size, err := f.volumeHandle.Write(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, f.inodeNumber, uint64(req.Offset), bufferedData, nil) - if nil == err { - resp.Size = int(size) - } else { - err = newFuseError(err) - } - return -} diff --git a/fuse/fuse.go b/fuse/fuse.go deleted file mode 100644 index a2293eb5..00000000 --- a/fuse/fuse.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fuse - -import ( - "fmt" - "sync" - - fuselib "bazil.org/fuse" - fusefslib "bazil.org/fuse/fs" - "golang.org/x/net/context" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" -) - -type ProxyFUSE struct { - volumeHandle fs.VolumeHandle - wg sync.WaitGroup // Used to synchronize mount -} - -func (pfs *ProxyFUSE) Root() (fusefslib.Node, error) { - root := Dir{volumeHandle: pfs.volumeHandle, inodeNumber: inode.RootDirInodeNumber} - - // Signal any waiters that we have completed mounting the volume. - // We know this because this call is only made after the user level FUSE - // library and the FUSE driver have agreed on the FUSE prototocol level. - pfs.wg.Done() - return root, nil -} - -func (pfs *ProxyFUSE) Statfs(ctx context.Context, req *fuselib.StatfsRequest, resp *fuselib.StatfsResponse) error { - enterGate() - defer leaveGate() - - statvfs, err := pfs.volumeHandle.StatVfs() - if err != nil { - return newFuseError(err) - } - resp.Blocks = statvfs[fs.StatVFSTotalBlocks] - resp.Bfree = statvfs[fs.StatVFSFreeBlocks] - resp.Bavail = statvfs[fs.StatVFSAvailBlocks] - resp.Files = statvfs[fs.StatVFSTotalInodes] - resp.Ffree = statvfs[fs.StatVFSFreeInodes] - resp.Bsize = uint32(statvfs[fs.StatVFSBlockSize]) - resp.Namelen = uint32(statvfs[fs.StatVFSMaxFilenameLen]) - resp.Frsize = uint32(statvfs[fs.StatVFSFragmentSize]) - return nil -} - -type fuseError struct { - str string - errno fuselib.Errno -} - -func (fE *fuseError) Errno() fuselib.Errno { - return fE.errno -} - -func (fE *fuseError) Error() string { - return fE.str -} - -func newFuseError(err error) *fuseError { - return &fuseError{ - str: fmt.Sprintf("%v", err), - errno: fuselib.Errno(blunder.Errno(err)), - } -} diff --git a/fuse/symlink.go b/fuse/symlink.go deleted file mode 100644 index cec57f44..00000000 --- a/fuse/symlink.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package fuse - -import ( - "fmt" - "os" - "time" - - fuselib "bazil.org/fuse" - "golang.org/x/net/context" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" -) - -type Symlink struct { - volumeHandle fs.VolumeHandle - inodeNumber inode.InodeNumber -} - -func (s Symlink) Attr(ctx context.Context, attr *fuselib.Attr) (err error) { - var ( - stat fs.Stat - ) - - enterGate() - defer leaveGate() - - stat, err = s.volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, s.inodeNumber) - if nil != err { - err = newFuseError(err) - return - } - if uint64(inode.SymlinkType) != stat[fs.StatFType] { - err = fmt.Errorf("[fuse]Dir.Attr() called on non-Symlink") - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return - } - - attr.Valid = time.Duration(time.Microsecond) // TODO: Make this settable if FUSE inside ProxyFS endures - attr.Inode = uint64(s.inodeNumber) // or stat[fs.StatINum] - attr.Size = stat[fs.StatSize] - attr.Atime = time.Unix(0, int64(stat[fs.StatATime])) - attr.Mtime = time.Unix(0, int64(stat[fs.StatMTime])) - attr.Ctime = time.Unix(0, int64(stat[fs.StatCTime])) - attr.Crtime = time.Unix(0, int64(stat[fs.StatCRTime])) - attr.Mode = os.ModeSymlink | os.FileMode(stat[fs.StatMode]&0777) - attr.Nlink = uint32(stat[fs.StatNLink]) - attr.Uid = uint32(stat[fs.StatUserID]) - attr.Gid = uint32(stat[fs.StatGroupID]) - - return -} - -func (s Symlink) Setattr(ctx context.Context, req *fuselib.SetattrRequest, resp *fuselib.SetattrResponse) (err error) { - var ( - stat fs.Stat - statUpdates fs.Stat - ) - - enterGate() - defer leaveGate() - - stat, err = s.volumeHandle.Getstat(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, s.inodeNumber) - if nil != err { - err = newFuseError(err) - return - } - if uint64(inode.SymlinkType) != stat[fs.StatFType] { - err = fmt.Errorf("[fuse]Dir.Attr() called on non-Symlink") - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - err = newFuseError(err) - return - } - - statUpdates = make(fs.Stat) - - if 0 != (fuselib.SetattrMode & req.Valid) { - statUpdates[fs.StatMode] = uint64(req.Mode & 0777) - } - if 0 != (fuselib.SetattrUid & req.Valid) { - statUpdates[fs.StatUserID] = uint64(req.Uid) - } - if 0 != (fuselib.SetattrGid & req.Valid) { - statUpdates[fs.StatGroupID] = uint64(req.Gid) - } - if 0 != (fuselib.SetattrAtime & req.Valid) { - statUpdates[fs.StatATime] = uint64(req.Atime.UnixNano()) - } - if 0 != (fuselib.SetattrMtime & req.Valid) { - statUpdates[fs.StatMTime] = uint64(req.Mtime.UnixNano()) - } - if 0 != (fuselib.SetattrAtimeNow & req.Valid) { - statUpdates[fs.StatATime] = uint64(time.Now().UnixNano()) - } - if 0 != (fuselib.SetattrMtimeNow & req.Valid) { - statUpdates[fs.StatMTime] = uint64(time.Now().UnixNano()) - } - if 0 != (fuselib.SetattrCrtime & req.Valid) { - statUpdates[fs.StatCRTime] = uint64(req.Crtime.UnixNano()) - } - - err = s.volumeHandle.Setstat(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, s.inodeNumber, statUpdates) - if nil != err { - err = newFuseError(err) - } - - return -} - -func (s Symlink) Fsync(ctx context.Context, req *fuselib.FsyncRequest) error { - return fuselib.ENOSYS -} - -func (s Symlink) Readlink(ctx context.Context, req *fuselib.ReadlinkRequest) (string, error) { - target, err := s.volumeHandle.Readsymlink(inode.InodeUserID(req.Header.Uid), inode.InodeGroupID(req.Header.Gid), nil, s.inodeNumber) - if nil != err { - err = newFuseError(err) - } - return target, err -} diff --git a/go.mod b/go.mod index c22d15f1..97cb6ed3 100644 --- a/go.mod +++ b/go.mod @@ -2,63 +2,21 @@ module github.com/NVIDIA/proxyfs go 1.17 -replace github.com/coreos/bbolt => go.etcd.io/bbolt v1.3.6 - -replace go.etcd.io/bbolt v1.3.6 => github.com/coreos/bbolt v1.3.6 - -replace github.com/coreos/etcd => github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible - -replace google.golang.org/grpc => google.golang.org/grpc v1.27.0 - require ( - bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 - github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 github.com/ansel1/merry v1.6.1 - github.com/creachadair/cityhash v0.1.1 github.com/google/btree v1.0.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 - go.etcd.io/etcd v3.3.27+incompatible - golang.org/x/net v0.0.0-20211216030914-fe4d6282115f golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e ) require ( + github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 // indirect github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect - github.com/coreos/bbolt v0.0.0-00010101000000-000000000000 // indirect - github.com/coreos/etcd v0.0.0-00010101000000-000000000000 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.4.3 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/jonboulle/clockwork v0.2.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.11.0 // indirect - github.com/soheilhy/cmux v0.1.5 // indirect - github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.19.1 // indirect - golang.org/x/text v0.3.6 // indirect - golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 // indirect - google.golang.org/grpc v1.36.0 // indirect - google.golang.org/protobuf v1.26.0-rc.1 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 32166f89..57ceb021 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,11 @@ -bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= -bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b h1:Jhps18HvZUrJKJR2E57rF5Kp/7chK6NDo/wZnD0gtF0= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b/go.mod h1:9wVslsyxZaBvW/ollg7JLxJOxKb+Ik2KH1WVs1nicMA= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/ansel1/merry v1.5.0/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= github.com/ansel1/merry v1.5.1/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= github.com/ansel1/merry v1.6.1 h1:AuheQTqQ04yQdBMCDnneVPvpBdWTBBg3LKMvKGMyRxU= @@ -20,55 +13,22 @@ github.com/ansel1/merry v1.6.1/go.mod h1:ioJjPJ/IsjxH+cC0lpf5TmbKnbcGa9qTk0fDbeR github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZtKWYbsCus= github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/creachadair/cityhash v0.1.1 h1:lFsixblDY79QBwzGNQWRvMu4mOBIuV+g3sz/3Vy3pk0= -github.com/creachadair/cityhash v0.1.1/go.mod h1:QhWJSDPs/2AkcOUL8Ogz0ZzA5acyIihEzKBG/9NTUHs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -76,7 +36,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -85,214 +44,61 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible h1:CAG0PUvo1fen+ZEfxKJjFIc8GuuN5RuaBuCAuaP2Hno= -github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible/go.mod h1:iIubILNIN6Jq9h8uiSLrN9L1tuj3iSSFwz3R61skm/A= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0= -go.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= -golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20210224155714-063164c882e6 h1:bXUwz2WkXXrXgiLxww3vWmoSHLOGv4ipdPdTvKymcKw= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -303,26 +109,11 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/halter/Makefile b/halter/Makefile deleted file mode 100644 index 0f9999ec..00000000 --- a/halter/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/halter - -include ../GoMakefile diff --git a/halter/api.go b/halter/api.go deleted file mode 100644 index 8d64fcdc..00000000 --- a/halter/api.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package halter - -import ( - "fmt" - "os" - "syscall" - - "github.com/NVIDIA/proxyfs/evtlog" -) - -// Note 1: Following const block and HaltLabelStrings should be kept in sync -// Note 2: HaltLabelStrings should be easily parseable as URL components - -const ( - apiTestHaltLabel1 = iota - apiTestHaltLabel2 - InodeFlushInodesEntry - InodeFlushInodesExit -) - -var ( - HaltLabelStrings = []string{ - "halter.testHaltLabel1", - "halter.testHaltLabel2", - "inode.flushInodes_Entry", - "inode.flushInodes_Exit", - } -) - -// Arm sets up a HALT on the haltAfterCount'd call to Trigger() -func Arm(haltLabelString string, haltAfterCount uint32) { - globals.Lock() - haltLabel, ok := globals.triggerNamesToNumbers[haltLabelString] - if !ok { - err := fmt.Errorf("halter.Arm(haltLabelString='%v',) - label unknown", haltLabelString) - haltWithErr(err) - } - if 0 == haltAfterCount { - err := fmt.Errorf("halter.Arm(haltLabel==%v,) called with haltAfterCount==0", haltLabelString) - haltWithErr(err) - } - globals.armedTriggers[haltLabel] = haltAfterCount - globals.Unlock() - evtlog.Record(evtlog.FormatHalterArm, haltLabelString, haltAfterCount) -} - -// Disarm removes a previously armed trigger via a call to Arm() -func Disarm(haltLabelString string) { - globals.Lock() - haltLabel, ok := globals.triggerNamesToNumbers[haltLabelString] - if !ok { - err := fmt.Errorf("halter.Disarm(haltLabelString='%v') - label unknown", haltLabelString) - haltWithErr(err) - } - delete(globals.armedTriggers, haltLabel) - globals.Unlock() - evtlog.Record(evtlog.FormatHalterDisarm, haltLabelString) -} - -// Trigger decrements the haltAfterCount if armed and, should it reach 0, HALTs -func Trigger(haltLabel uint32) { - globals.Lock() - numTriggersRemaining, armed := globals.armedTriggers[haltLabel] - if !armed { - globals.Unlock() - return - } - numTriggersRemaining-- - if 0 == numTriggersRemaining { - haltLabelString := globals.triggerNumbersToNames[haltLabel] - evtlog.Record(evtlog.FormatHalterDisarm, haltLabelString) - err := fmt.Errorf("halter.TriggerArm(haltLabelString==%v) triggered HALT", haltLabelString) - haltWithErr(err) - } - globals.armedTriggers[haltLabel] = numTriggersRemaining - globals.Unlock() -} - -// Dump returns a map of currently armed triggers and their remaining trigger count -func Dump() (armedTriggers map[string]uint32) { - armedTriggers = make(map[string]uint32) - for k, v := range globals.armedTriggers { - armedTriggers[globals.triggerNumbersToNames[k]] = v - } - return -} - -// List returns a slice of available triggers -func List() (availableTriggers []string) { - availableTriggers = make([]string, 0, len(globals.triggerNumbersToNames)) - for k := range globals.triggerNamesToNumbers { - availableTriggers = append(availableTriggers, k) - } - return -} - -// Stat returns the remaining haltAfterCount for the specified haltLabelString -func Stat(haltLabelString string) (haltAfterCount uint32, err error) { - globals.Lock() - haltLabel, ok := globals.triggerNamesToNumbers[haltLabelString] - if !ok { - globals.Unlock() - err = fmt.Errorf("halter.Stat(haltLabelString='%v') - label unknown", haltLabelString) - return - } - haltAfterCount, ok = globals.armedTriggers[haltLabel] - if !ok { - haltAfterCount = 0 - } - globals.Unlock() - return -} - -func haltWithErr(err error) { - if nil == globals.testModeHaltCB { - fmt.Println(err) - os.Exit(int(syscall.SIGKILL)) - } else { - globals.testModeHaltCB(err) - } -} diff --git a/halter/api_test.go b/halter/api_test.go deleted file mode 100644 index 1be723f3..00000000 --- a/halter/api_test.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package halter - -import ( - "fmt" - "testing" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/transitions" -) - -var ( - testHaltErr error -) - -func TestAPI(t *testing.T) { - testConfMapStrings := []string{ - "Logging.LogFilePath=/dev/null", - "Cluster.WhoAmI=nobody", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - } - - testConfMap, err := conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatal(err) - } - - err = transitions.Up(testConfMap) - if nil != err { - t.Fatal(err) - } - - configureTestModeHaltCB(testHalt) - - availableTriggers := List() - apiTestHaltLabel1Found := false - apiTestHaltLabel2Found := false - for _, s := range availableTriggers { - if "halter.testHaltLabel1" == s { - apiTestHaltLabel1Found = true - } - if "halter.testHaltLabel2" == s { - apiTestHaltLabel2Found = true - } - } - if !apiTestHaltLabel1Found { - t.Fatalf("List() unexpectedly missing 'halter.testHaltLabel1'") - } - if !apiTestHaltLabel2Found { - t.Fatalf("List() unexpectedly missing 'halter.testHaltLabel2'") - } - - m1 := Dump() - if 0 != len(m1) { - t.Fatalf("Dump() unexpectedly returned length %v map at start-up", len(m1)) - } - - testHaltErr = nil - Arm("halter.testHaltLabel0", 1) - if nil == testHaltErr { - t.Fatalf("Arm(apiTestHaltLabel0,) unexpectedly left testHaltErr as nil") - } - if "halter.Arm(haltLabelString='halter.testHaltLabel0',) - label unknown" != testHaltErr.Error() { - t.Fatalf("Arm(apiTestHaltLabel0,) unexpectedly set testHaltErr to %v", testHaltErr) - } - _, err = Stat("halter.testHaltLabel0") - if nil == err { - t.Fatalf("Stat(\"halter.testHaltLabel0\") unexpectedly succeeded") - } - - testHaltErr = nil - Arm("halter.testHaltLabel1", 0) - if nil == testHaltErr { - t.Fatalf("Arm(apiTestHaltLabel1,0) unexpectedly left testHaltErr as nil") - } - if "halter.Arm(haltLabel==halter.testHaltLabel1,) called with haltAfterCount==0" != testHaltErr.Error() { - fmt.Println(testHaltErr.Error()) - t.Fatalf("Arm(apiTestHaltLabel0,) unexpectedly set testHaltErr to %v", testHaltErr) - } - v0, err := Stat("halter.testHaltLabel1") - if nil != err { - t.Fatalf("Stat(\"halter.testHaltLabel1\") unexpectedly failed: %v", err) - } - if 0 != v0 { - t.Fatalf("Stat(\"halter.testHaltLabel1\") unexpectedly returned haltAfterCount == %v (should have been 0)", v0) - } - - Arm("halter.testHaltLabel1", 1) - m2 := Dump() - if 1 != len(m2) { - t.Fatalf("Dump() unexpectedly returned length %v map after Arm(apiTestHaltLabel1,)", len(m2)) - } - m2v1, ok := m2["halter.testHaltLabel1"] - if !ok { - t.Fatalf("Dump() unexpectedly missing m2[apiTestHaltLabel1]") - } - if 1 != m2v1 { - t.Fatalf("Dump() unexpectedly returned %v for m2[apiTestHaltLabel1]", m2v1) - } - v1, err := Stat("halter.testHaltLabel1") - if nil != err { - t.Fatalf("Stat(\"halter.testHaltLabel1\") unexpectedly failed: %v", err) - } - if 1 != v1 { - t.Fatalf("Stat(\"halter.testHaltLabel1\") unexpectedly returned haltAfterCount == %v (should have been 1)", v1) - } - - Arm("halter.testHaltLabel2", 2) - m3 := Dump() - if 2 != len(m3) { - t.Fatalf("Dump() unexpectedly returned length %v map after Arm(apiTestHaltLabel2,)", len(m3)) - } - m3v1, ok := m3["halter.testHaltLabel1"] - if !ok { - t.Fatalf("Dump() unexpectedly missing m3[apiTestHaltLabel1]") - } - if 1 != m3v1 { - t.Fatalf("Dump() unexpectedly returned %v for m3[apiTestHaltLabel1]", m3v1) - } - m3v2, ok := m3["halter.testHaltLabel2"] - if !ok { - t.Fatalf("Dump() unexpectedly missing m3[apiTestHaltLabel2]") - } - if 2 != m3v2 { - t.Fatalf("Dump() unexpectedly returned %v for m3[apiTestHaltLabel1]", m3v2) - } - - testHaltErr = nil - Disarm("halter.testHaltLabel0") - if nil == testHaltErr { - t.Fatalf("Disarm(apiTestHaltLabel) unexpectedly left testHaltErr as nil") - } - if "halter.Disarm(haltLabelString='halter.testHaltLabel0') - label unknown" != testHaltErr.Error() { - t.Fatalf("Disarm(apiTestHaltLabel0) unexpectedly set testHaltErr to %v", testHaltErr) - } - - Disarm("halter.testHaltLabel1") - m4 := Dump() - if 1 != len(m4) { - t.Fatalf("Dump() unexpectedly returned length %v map after Disarm(apiTestHaltLabel1)", len(m4)) - } - m4v2, ok := m4["halter.testHaltLabel2"] - if !ok { - t.Fatalf("Dump() unexpectedly missing m4[apiTestHaltLabel2]") - } - if 2 != m4v2 { - t.Fatalf("Dump() unexpectedly returned %v for m4[apiTestHaltLabel2]", m4v2) - } - - testHaltErr = nil - Trigger(apiTestHaltLabel2) - if nil != testHaltErr { - t.Fatalf("Trigger(apiTestHaltLabel2) [case 1] unexpectedly set testHaltErr to %v", testHaltErr) - } - m5 := Dump() - if 1 != len(m5) { - t.Fatalf("Dump() unexpectedly returned length %v map after Trigger(apiTestHaltLabel2)", len(m5)) - } - m5v2, ok := m5["halter.testHaltLabel2"] - if !ok { - t.Fatalf("Dump() unexpectedly missing m5[apiTestHaltLabel2]") - } - if 2 != m4v2 { - t.Fatalf("Dump() unexpectedly returned %v for m5[apiTestHaltLabel2]", m5v2) - } - - Trigger(apiTestHaltLabel2) - if nil == testHaltErr { - t.Fatalf("Trigger(apiTestHaltLabel2) [case 2] unexpectedly left testHaltErr as nil") - } - if "halter.TriggerArm(haltLabelString==halter.testHaltLabel2) triggered HALT" != testHaltErr.Error() { - t.Fatalf("Trigger(apiTestHaltLabel2) [case 2] unexpectedly set testHaltErr to %v", testHaltErr) - } - - err = transitions.Down(testConfMap) - if nil != err { - t.Fatal(err) - } -} - -func testHalt(err error) { - testHaltErr = err -} diff --git a/halter/config.go b/halter/config.go deleted file mode 100644 index bf112b53..00000000 --- a/halter/config.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package halter - -import ( - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -type globalsStruct struct { - trackedlock.Mutex - armedTriggers map[uint32]uint32 // key: haltLabel; value: haltAfterCount (remaining) - triggerNamesToNumbers map[string]uint32 - triggerNumbersToNames map[uint32]string - testModeHaltCB func(err error) -} - -var globals globalsStruct - -func init() { - transitions.Register("halter", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - globals.armedTriggers = make(map[uint32]uint32) - globals.triggerNamesToNumbers = make(map[string]uint32) - globals.triggerNumbersToNames = make(map[uint32]string) - for i, s := range HaltLabelStrings { - globals.triggerNamesToNumbers[s] = uint32(i) - globals.triggerNumbersToNames[uint32(i)] = s - } - globals.testModeHaltCB = nil - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - return nil -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - err = nil - return -} - -func configureTestModeHaltCB(testHalt func(err error)) { - globals.Lock() - globals.testModeHaltCB = testHalt - globals.Unlock() -} diff --git a/headhunter/Makefile b/headhunter/Makefile deleted file mode 100644 index a5302e8b..00000000 --- a/headhunter/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/headhunter - -include ../GoMakefile diff --git a/headhunter/api.go b/headhunter/api.go deleted file mode 100644 index 8fecdebc..00000000 --- a/headhunter/api.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package headhunter manages the headhunter databases, which keep track of which log segments correspond to the current revision of a given inode. -package headhunter - -import ( - "fmt" - "time" - - "github.com/NVIDIA/sortedmap" -) - -type BPlusTreeType uint32 - -const ( - MergedBPlusTree BPlusTreeType = iota // Used only for FetchLayoutReport when a merged result is desired - InodeRecBPlusTree - LogSegmentRecBPlusTree - BPlusTreeObjectBPlusTree - CreatedObjectsBPlusTree - DeletedObjectsBPlusTree -) - -type SnapShotIDType uint8 - -const ( - SnapShotIDTypeLive SnapShotIDType = iota - SnapShotIDTypeSnapShot - SnapShotIDTypeDotSnapShot -) - -type SnapShotStruct struct { - ID uint64 - Time time.Time - Name string -} - -type VolumeEventListener interface { - CheckpointCompleted() -} - -// VolumeHandle is used to operate on a given volume's database -type VolumeHandle interface { - RegisterForEvents(listener VolumeEventListener) - UnregisterForEvents(listener VolumeEventListener) - FetchAccountAndCheckpointContainerNames() (accountName string, checkpointContainerName string) - FetchNonce() (nonce uint64) - GetInodeRec(inodeNumber uint64) (value []byte, ok bool, err error) - PutInodeRec(inodeNumber uint64, value []byte) (err error) - PutInodeRecs(inodeNumbers []uint64, values [][]byte) (err error) - DeleteInodeRec(inodeNumber uint64) (err error) - IndexedInodeNumber(index uint64) (inodeNumber uint64, ok bool, err error) - NextInodeNumber(lastInodeNumber uint64) (nextInodeNumber uint64, ok bool, err error) - GetLogSegmentRec(logSegmentNumber uint64) (value []byte, err error) - PutLogSegmentRec(logSegmentNumber uint64, value []byte) (err error) - DeleteLogSegmentRec(logSegmentNumber uint64) (err error) - IndexedLogSegmentNumber(index uint64) (logSegmentNumber uint64, ok bool, err error) - GetBPlusTreeObject(objectNumber uint64) (value []byte, err error) - PutBPlusTreeObject(objectNumber uint64, value []byte) (err error) - DeleteBPlusTreeObject(objectNumber uint64) (err error) - IndexedBPlusTreeObjectNumber(index uint64) (objectNumber uint64, ok bool, err error) - DoCheckpoint() (err error) - FetchLayoutReport(treeType BPlusTreeType, validate bool) (layoutReport sortedmap.LayoutReport, discrepencies uint64, err error) - DefragmentMetadata(treeType BPlusTreeType, thisStartPercentage float64, thisStopPercentage float64) (err error) - SnapShotCreateByInodeLayer(name string) (id uint64, err error) - SnapShotDeleteByInodeLayer(id uint64) (err error) - SnapShotCount() (snapShotCount uint64) - SnapShotLookupByName(name string) (snapShot SnapShotStruct, ok bool) - SnapShotListByID(reversed bool) (list []SnapShotStruct) - SnapShotListByTime(reversed bool) (list []SnapShotStruct) - SnapShotListByName(reversed bool) (list []SnapShotStruct) - SnapShotU64Decode(snapShotU64 uint64) (snapShotIDType SnapShotIDType, snapShotID uint64, nonce uint64) - SnapShotIDAndNonceEncode(snapShotID uint64, nonce uint64) (snapShotU64 uint64) - SnapShotTypeDotSnapShotAndNonceEncode(nonce uint64) (snapShotU64 uint64) -} - -// FetchVolumeHandle is used to fetch a VolumeHandle to use when operating on a given volume's database -func FetchVolumeHandle(volumeName string) (volumeHandle VolumeHandle, err error) { - volume, ok := globals.volumeMap[volumeName] - if !ok { - err = fmt.Errorf("FetchVolumeHandle(\"%v\") unable to find volume", volumeName) - return - } - - volumeHandle = volume - err = nil - - return -} - -// DisableObjectDeletions prevents objects from being deleted until EnableObjectDeletions() is called -func DisableObjectDeletions() { - globals.backgroundObjectDeleteRWMutex.Lock() - - if !globals.backgroundObjectDeleteEnabled { - // Already disabled... just exit - globals.backgroundObjectDeleteRWMutex.Unlock() - return - } - - globals.backgroundObjectDeleteEnabled = false - - globals.backgroundObjectDeleteEnabledWG.Add(1) - - globals.backgroundObjectDeleteRWMutex.Unlock() - - globals.backgroundObjectDeleteActiveWG.Wait() -} - -// EnableObjectDeletions resumes background object deletion blocked by a prior call to DisableObjectDeletions() -func EnableObjectDeletions() { - globals.backgroundObjectDeleteRWMutex.Lock() - - if globals.backgroundObjectDeleteEnabled { - // Already enabled... just exit - globals.backgroundObjectDeleteRWMutex.Unlock() - return - } - - globals.backgroundObjectDeleteEnabled = true - - globals.backgroundObjectDeleteEnabledWG.Done() - - globals.backgroundObjectDeleteRWMutex.Unlock() -} diff --git a/headhunter/api_internal.go b/headhunter/api_internal.go deleted file mode 100644 index eb3d3804..00000000 --- a/headhunter/api_internal.go +++ /dev/null @@ -1,1771 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package headhunter - -import ( - "container/list" - "fmt" - "math/big" - "time" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/swiftclient" -) - -func (volume *volumeStruct) RegisterForEvents(listener VolumeEventListener) { - var ( - ok bool - ) - - volume.Lock() - - _, ok = volume.eventListeners[listener] - if ok { - logger.Fatalf("headhunter.RegisterForEvents() called for volume %v listener %p already active", volume.volumeName, listener) - } - - volume.eventListeners[listener] = struct{}{} - - volume.Unlock() -} - -func (volume *volumeStruct) UnregisterForEvents(listener VolumeEventListener) { - var ( - ok bool - ) - - volume.Lock() - - _, ok = volume.eventListeners[listener] - if !ok { - // this should never happen, its an internal logic error, so log - // it as an error. it is not a panic or fatal because this - // internal error will probably not break proxyfs (and panic'ing - // is hard on the SMB clients) - logger.Errorf("headhunter.UnregisterForEvents() called for volume %v listener %p not active", volume.volumeName, listener) - } - - delete(volume.eventListeners, listener) - - volume.Unlock() -} - -func (volume *volumeStruct) FetchAccountAndCheckpointContainerNames() (accountName string, checkpointContainerName string) { - accountName = volume.accountName - checkpointContainerName = volume.checkpointContainerName - return -} - -func (volume *volumeStruct) fetchNonceWhileLocked() (nonce uint64) { - var ( - checkpointContainerHeaders map[string][]string - checkpointHeaderValue string - checkpointHeaderValues []string - err error - newCheckpointHeader *CheckpointHeaderStruct - ) - - if volume.nextNonce >= volume.maxNonce { - logger.Fatalf("Nonces have been exhausted !!!") - } - - if volume.nextNonce == volume.checkpointHeader.ReservedToNonce { - newCheckpointHeader = &CheckpointHeaderStruct{ - CheckpointVersion: volume.checkpointHeader.CheckpointVersion, - CheckpointObjectTrailerStructObjectNumber: volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber, - CheckpointObjectTrailerStructObjectLength: volume.checkpointHeader.CheckpointObjectTrailerStructObjectLength, - ReservedToNonce: volume.checkpointHeader.ReservedToNonce + uint64(volume.nonceValuesToReserve), - } - - volume.checkpointHeader = newCheckpointHeader - - // TODO: Move this inside recordTransaction() once it is a, uh, transaction :-) - evtlog.Record(evtlog.FormatHeadhunterRecordTransactionNonceRangeReserve, volume.volumeName, volume.nextNonce, volume.checkpointHeader.ReservedToNonce-1) - - if globals.etcdEnabled { - volume.checkpointHeaderEtcdRevision, err = volume.putEtcdCheckpointHeader(volume.checkpointHeader, volume.checkpointHeaderEtcdRevision) - if nil != err { - logger.Fatalf("Unable to persist checkpointHeader in etcd: %v", err) - } - } - - checkpointHeaderValue = fmt.Sprintf("%016X %016X %016X %016X", - volume.checkpointHeader.CheckpointVersion, - volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber, - volume.checkpointHeader.CheckpointObjectTrailerStructObjectLength, - volume.checkpointHeader.ReservedToNonce, - ) - - checkpointHeaderValues = []string{checkpointHeaderValue} - - checkpointContainerHeaders = make(map[string][]string) - - checkpointContainerHeaders[CheckpointHeaderName] = checkpointHeaderValues - - err = swiftclient.ContainerPost(volume.accountName, volume.checkpointContainerName, checkpointContainerHeaders) - if nil != err { - logger.Fatalf("Unable to persist checkpointHeader in Swift: %v", err) - } - if globals.logCheckpointHeaderPosts { - logger.Infof("POST'd checkpointHeaderValue %s for volume %s from fetchNonceWhileLocked()", checkpointHeaderValue, volume.volumeName) - } - } - - nonce = volume.nextNonce - volume.nextNonce++ - - return -} - -func (volume *volumeStruct) FetchNonce() (nonce uint64) { - startTime := time.Now() - defer func() { - globals.FetchNonceUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - volume.Lock() - nonce = volume.fetchNonceWhileLocked() - volume.Unlock() - - return -} - -func (volume *volumeStruct) findVolumeViewAndNonceWhileLocked(key uint64) (volumeView *volumeViewStruct, nonce uint64, ok bool, err error) { - var ( - snapShotID uint64 - snapShotIDType SnapShotIDType - value sortedmap.Value - ) - - snapShotIDType, snapShotID, nonce = volume.SnapShotU64Decode(key) - - if SnapShotIDTypeDotSnapShot == snapShotIDType { - err = fmt.Errorf("findVolumeViewAndNonceWhileLocked() called for SnapShotIDTypeDotSnapShot key") - return - } - - if SnapShotIDTypeLive == snapShotIDType { - volumeView = volume.liveView - ok = true - err = nil - return - } - - value, ok, err = volume.viewTreeByID.GetByKey(snapShotID) - if (nil != err) || !ok { - return - } - - volumeView, ok = value.(*volumeViewStruct) - - return -} - -func (volume *volumeStruct) GetInodeRec(inodeNumber uint64) (value []byte, ok bool, err error) { - - startTime := time.Now() - defer func() { - globals.GetInodeRecUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.GetInodeRecBytes.Add(uint64(len(value))) - if err != nil { - globals.GetInodeRecErrors.Add(1) - } - }() - - volume.Lock() - - volumeView, nonce, ok, err := volume.findVolumeViewAndNonceWhileLocked(inodeNumber) - if nil != err { - volume.Unlock() - return - } - if !ok { - volume.Unlock() - evtlog.Record(evtlog.FormatHeadhunterMissingInodeRec, volume.volumeName, inodeNumber) - err = fmt.Errorf("inodeNumber 0x%016X not found in volume \"%v\" inodeRecWrapper.bPlusTree", inodeNumber, volume.volumeName) - return - } - - valueAsValue, ok, err := volumeView.inodeRecWrapper.bPlusTree.GetByKey(nonce) - if nil != err { - volume.Unlock() - return - } - if !ok { - volume.Unlock() - evtlog.Record(evtlog.FormatHeadhunterMissingInodeRec, volume.volumeName, inodeNumber) - err = fmt.Errorf("inodeNumber 0x%016X not found in volume \"%v\" inodeRecWrapper.bPlusTree", inodeNumber, volume.volumeName) - return - } - valueFromTree := valueAsValue.([]byte) - value = make([]byte, len(valueFromTree)) - copy(value, valueFromTree) - - volume.Unlock() - - err = nil - return -} - -func (volume *volumeStruct) PutInodeRec(inodeNumber uint64, value []byte) (err error) { - - startTime := time.Now() - defer func() { - globals.PutInodeRecUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.PutInodeRecBytes.Add(uint64(len(value))) - if err != nil { - globals.PutInodeRecErrors.Add(1) - } - }() - - valueToTree := make([]byte, len(value)) - copy(valueToTree, value) - - volume.Lock() - - volume.checkpointTriggeringEvents++ - - ok, err := volume.liveView.inodeRecWrapper.bPlusTree.PatchByKey(inodeNumber, valueToTree) - if nil != err { - volume.Unlock() - return - } - if !ok { - _, err = volume.liveView.inodeRecWrapper.bPlusTree.Put(inodeNumber, valueToTree) - if nil != err { - volume.Unlock() - return - } - } - - volume.recordTransaction(transactionPutInodeRec, inodeNumber, value) - - volume.Unlock() - - err = nil - return -} - -func (volume *volumeStruct) PutInodeRecs(inodeNumbers []uint64, values [][]byte) (err error) { - - startTime := time.Now() - defer func() { - var totalBytes int - for _, inodeValue := range values { - totalBytes += len(inodeValue) - } - - globals.PutInodeRecsUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.PutInodeRecsBytes.Add(uint64(totalBytes)) - if err != nil { - globals.PutInodeRecsErrors.Add(1) - } - }() - - if len(inodeNumbers) != len(values) { - err = fmt.Errorf("InodeNumber and Values array don't match") - return - } - - valuesToTree := make([][]byte, len(inodeNumbers)) - - for i := range inodeNumbers { - valuesToTree[i] = make([]byte, len(values[i])) - copy(valuesToTree[i], values[i]) - } - - volume.Lock() - - volume.checkpointTriggeringEvents += uint64(len(inodeNumbers)) - - for i, inodeNumber := range inodeNumbers { - ok, nonShadowingErr := volume.liveView.inodeRecWrapper.bPlusTree.PatchByKey(inodeNumber, valuesToTree[i]) - if nil != nonShadowingErr { - volume.Unlock() - err = nonShadowingErr - return - } - if !ok { - _, err = volume.liveView.inodeRecWrapper.bPlusTree.Put(inodeNumber, valuesToTree[i]) - if nil != err { - volume.Unlock() - return - } - } - } - - volume.recordTransaction(transactionPutInodeRecs, inodeNumbers, values) - - volume.Unlock() - - err = nil - return -} - -func (volume *volumeStruct) DeleteInodeRec(inodeNumber uint64) (err error) { - - startTime := time.Now() - defer func() { - globals.DeleteInodeRecUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.DeleteInodeRecErrors.Add(1) - } - }() - - volume.Lock() - - volume.checkpointTriggeringEvents++ - - _, err = volume.liveView.inodeRecWrapper.bPlusTree.DeleteByKey(inodeNumber) - - volume.recordTransaction(transactionDeleteInodeRec, inodeNumber, nil) - - volume.Unlock() - - return -} - -func (volume *volumeStruct) IndexedInodeNumber(index uint64) (inodeNumber uint64, ok bool, err error) { - - startTime := time.Now() - defer func() { - globals.IndexedInodeNumberUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.IndexedInodeNumberErrors.Add(1) - } - }() - - volume.Lock() - key, _, ok, err := volume.liveView.inodeRecWrapper.bPlusTree.GetByIndex(int(index)) - if nil != err { - volume.Unlock() - return - } - volume.Unlock() - - if !ok { - return - } - - inodeNumber = key.(uint64) - - return -} - -func (volume *volumeStruct) NextInodeNumber(lastInodeNumber uint64) (nextInodeNumber uint64, ok bool, err error) { - - startTime := time.Now() - defer func() { - globals.NextInodeNumberUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.NextInodeNumberErrors.Add(1) - } - }() - - if uint64(0xFFFFFFFFFFFFFFFF) == lastInodeNumber { - ok = false - return - } - - volume.Lock() - index, _, err := volume.liveView.inodeRecWrapper.bPlusTree.BisectRight(lastInodeNumber + 1) - if nil != err { - volume.Unlock() - return - } - key, _, ok, err := volume.liveView.inodeRecWrapper.bPlusTree.GetByIndex(int(index)) - if nil != err { - volume.Unlock() - return - } - volume.Unlock() - - if ok { - nextInodeNumber = key.(uint64) - } - - return -} - -func (volume *volumeStruct) GetLogSegmentRec(logSegmentNumber uint64) (value []byte, err error) { - - startTime := time.Now() - defer func() { - globals.GetLogSegmentRecUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.GetLogSegmentRecErrors.Add(1) - } - }() - - volume.Lock() - - volumeView, nonce, ok, err := volume.findVolumeViewAndNonceWhileLocked(logSegmentNumber) - if (nil != err) || !ok { - volume.Unlock() - return - } - if !ok { - volume.Unlock() - evtlog.Record(evtlog.FormatHeadhunterMissingLogSegmentRec, volume.volumeName, logSegmentNumber) - err = fmt.Errorf("logSegmentNumber 0x%016X not found in volume \"%v\" logSegmentRecWrapper.bPlusTree", logSegmentNumber, volume.volumeName) - return - } - - valueAsValue, ok, err := volumeView.logSegmentRecWrapper.bPlusTree.GetByKey(nonce) - if nil != err { - volume.Unlock() - return - } - if !ok { - volume.Unlock() - evtlog.Record(evtlog.FormatHeadhunterMissingLogSegmentRec, volume.volumeName, logSegmentNumber) - err = fmt.Errorf("logSegmentNumber 0x%016X not found in volume \"%v\" logSegmentRecWrapper.bPlusTree", logSegmentNumber, volume.volumeName) - return - } - valueFromTree := valueAsValue.([]byte) - value = make([]byte, len(valueFromTree)) - copy(value, valueFromTree) - - volume.Unlock() - - err = nil - return -} - -func (volume *volumeStruct) PutLogSegmentRec(logSegmentNumber uint64, value []byte) (err error) { - - startTime := time.Now() - defer func() { - globals.PutLogSegmentRecUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.PutLogSegmentRecErrors.Add(1) - } - }() - - valueToTree := make([]byte, len(value)) - copy(valueToTree, value) - - volume.Lock() - - volume.checkpointTriggeringEvents++ - - ok, err := volume.liveView.logSegmentRecWrapper.bPlusTree.PatchByKey(logSegmentNumber, valueToTree) - if nil != err { - volume.Unlock() - return - } - if !ok { - _, err = volume.liveView.logSegmentRecWrapper.bPlusTree.Put(logSegmentNumber, valueToTree) - if nil != err { - volume.Unlock() - return - } - } - - if nil != volume.priorView { - _, err = volume.priorView.createdObjectsWrapper.bPlusTree.Put(logSegmentNumber, valueToTree) - if nil != err { - volume.Unlock() - return - } - } - - volume.recordTransaction(transactionPutLogSegmentRec, logSegmentNumber, value) - - volume.Unlock() - - err = nil - return -} - -func (volume *volumeStruct) DeleteLogSegmentRec(logSegmentNumber uint64) (err error) { - var ( - containerNameAsValue sortedmap.Value - ok bool - ) - - startTime := time.Now() - defer func() { - globals.DeleteLogSegmentRecUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.DeleteLogSegmentRecErrors.Add(1) - } - }() - - volume.Lock() - defer volume.Unlock() - - volume.checkpointTriggeringEvents++ - - containerNameAsValue, ok, err = volume.liveView.logSegmentRecWrapper.bPlusTree.GetByKey(logSegmentNumber) - if nil != err { - return - } - if !ok { - err = fmt.Errorf("Missing logSegmentNumber (0x%016X) in volume %v LogSegmentRec B+Tree", logSegmentNumber, volume.volumeName) - return - } - - _, err = volume.liveView.logSegmentRecWrapper.bPlusTree.DeleteByKey(logSegmentNumber) - if nil != err { - return - } - - if nil == volume.priorView { - _, err = volume.liveView.deletedObjectsWrapper.bPlusTree.Put(logSegmentNumber, containerNameAsValue) - if nil != err { - return - } - } else { - ok, err = volume.priorView.createdObjectsWrapper.bPlusTree.DeleteByKey(logSegmentNumber) - if nil != err { - return - } - if ok { - _, err = volume.liveView.deletedObjectsWrapper.bPlusTree.Put(logSegmentNumber, containerNameAsValue) - if nil != err { - return - } - } else { - _, err = volume.priorView.deletedObjectsWrapper.bPlusTree.Put(logSegmentNumber, containerNameAsValue) - if nil != err { - return - } - } - } - - volume.recordTransaction(transactionDeleteLogSegmentRec, logSegmentNumber, nil) - - return -} - -func (volume *volumeStruct) IndexedLogSegmentNumber(index uint64) (logSegmentNumber uint64, ok bool, err error) { - - startTime := time.Now() - defer func() { - globals.IndexedLogSegmentNumberUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.IndexedLogSegmentNumberErrors.Add(1) - } - }() - - volume.Lock() - key, _, ok, err := volume.liveView.logSegmentRecWrapper.bPlusTree.GetByIndex(int(index)) - if nil != err { - volume.Unlock() - return - } - volume.Unlock() - - if !ok { - return - } - - logSegmentNumber = key.(uint64) - - return -} - -func (volume *volumeStruct) GetBPlusTreeObject(objectNumber uint64) (value []byte, err error) { - - startTime := time.Now() - defer func() { - globals.GetBPlusTreeObjectUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.GetBPlusTreeObjectBytes.Add(uint64(len(value))) - if err != nil { - globals.GetBPlusTreeObjectErrors.Add(1) - } - }() - - volume.Lock() - - volumeView, nonce, ok, err := volume.findVolumeViewAndNonceWhileLocked(objectNumber) - if nil != err { - volume.Unlock() - return - } - if !ok { - volume.Unlock() - evtlog.Record(evtlog.FormatHeadhunterMissingBPlusTreeObject, volume.volumeName, objectNumber) - err = fmt.Errorf("objectNumber 0x%016X not found in volume \"%v\" bPlusTreeObjectWrapper.bPlusTree", objectNumber, volume.volumeName) - return - } - - valueAsValue, ok, err := volumeView.bPlusTreeObjectWrapper.bPlusTree.GetByKey(nonce) - if nil != err { - volume.Unlock() - return - } - if !ok { - volume.Unlock() - evtlog.Record(evtlog.FormatHeadhunterMissingBPlusTreeObject, volume.volumeName, objectNumber) - err = fmt.Errorf("objectNumber 0x%016X not found in volume \"%v\" bPlusTreeObjectWrapper.bPlusTree", objectNumber, volume.volumeName) - return - } - valueFromTree := valueAsValue.([]byte) - value = make([]byte, len(valueFromTree)) - copy(value, valueFromTree) - - volume.Unlock() - - err = nil - return -} - -func (volume *volumeStruct) PutBPlusTreeObject(objectNumber uint64, value []byte) (err error) { - - startTime := time.Now() - defer func() { - globals.PutBPlusTreeObjectUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - globals.PutBPlusTreeObjectBytes.Add(uint64(len(value))) - if err != nil { - globals.PutBPlusTreeObjectErrors.Add(1) - } - }() - - valueToTree := make([]byte, len(value)) - copy(valueToTree, value) - - volume.Lock() - - volume.checkpointTriggeringEvents++ - - ok, err := volume.liveView.bPlusTreeObjectWrapper.bPlusTree.PatchByKey(objectNumber, valueToTree) - if nil != err { - volume.Unlock() - return - } - if !ok { - _, err = volume.liveView.bPlusTreeObjectWrapper.bPlusTree.Put(objectNumber, valueToTree) - if nil != err { - volume.Unlock() - return - } - } - - volume.recordTransaction(transactionPutBPlusTreeObject, objectNumber, value) - - volume.Unlock() - - err = nil - return -} - -func (volume *volumeStruct) DeleteBPlusTreeObject(objectNumber uint64) (err error) { - - startTime := time.Now() - defer func() { - globals.DeleteBPlusTreeObjectUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.DeleteBPlusTreeObjectErrors.Add(1) - } - }() - - volume.Lock() - - volume.checkpointTriggeringEvents++ - - _, err = volume.liveView.bPlusTreeObjectWrapper.bPlusTree.DeleteByKey(objectNumber) - - volume.recordTransaction(transactionDeleteBPlusTreeObject, objectNumber, nil) - - volume.Unlock() - - return -} - -func (volume *volumeStruct) IndexedBPlusTreeObjectNumber(index uint64) (objectNumber uint64, ok bool, err error) { - - startTime := time.Now() - defer func() { - globals.IndexedBPlusTreeObjectNumberUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.IndexedBPlusTreeObjectNumberErrors.Add(1) - } - }() - - volume.Lock() - key, _, ok, err := volume.liveView.bPlusTreeObjectWrapper.bPlusTree.GetByIndex(int(index)) - if nil != err { - volume.Unlock() - return - } - volume.Unlock() - - if !ok { - return - } - - objectNumber = key.(uint64) - - return -} - -func (volume *volumeStruct) DoCheckpoint() (err error) { - var ( - checkpointRequest checkpointRequestStruct - ) - - startTime := time.Now() - defer func() { - globals.DoCheckpointUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.DoCheckpointErrors.Add(1) - } - }() - - checkpointRequest.exitOnCompletion = false - - checkpointRequest.waitGroup.Add(1) - volume.checkpointRequestChan <- &checkpointRequest - checkpointRequest.waitGroup.Wait() - - err = checkpointRequest.err - - return -} - -func (volume *volumeStruct) fetchLayoutReport(treeType BPlusTreeType, validate bool) (layoutReport sortedmap.LayoutReport, discrepencies uint64, err error) { - var ( - measuredLayoutReport sortedmap.LayoutReport - objectBytesMeasured uint64 - objectBytesTracked uint64 - objectNumber uint64 - ok bool - treeName string - treeWrapper *bPlusTreeWrapperStruct - ) - - switch treeType { - case InodeRecBPlusTree: - treeName = "InodeRec" - treeWrapper = volume.liveView.inodeRecWrapper - case LogSegmentRecBPlusTree: - treeName = "LogSegmentRec" - treeWrapper = volume.liveView.logSegmentRecWrapper - case BPlusTreeObjectBPlusTree: - treeName = "BPlusTreeObject" - treeWrapper = volume.liveView.bPlusTreeObjectWrapper - case CreatedObjectsBPlusTree: - treeName = "CreatedObjectObject" - treeWrapper = volume.liveView.createdObjectsWrapper - case DeletedObjectsBPlusTree: - treeName = "DeletedObjectObject" - treeWrapper = volume.liveView.deletedObjectsWrapper - default: - err = fmt.Errorf("fetchLayoutReport(treeType %d): bad tree type", treeType) - logger.ErrorfWithError(err, "volume '%s'", volume.volumeName) - return - } - - discrepencies = 0 - - if validate { - // Validate against a LayoutReport fetched from sortedmap (which "walks" the entire B+Tree) - - measuredLayoutReport, err = treeWrapper.bPlusTree.FetchLayoutReport() - if nil != err { - logger.ErrorfWithError(err, "FetchLayoutReport() volume '%s' tree '%s'", volume.volumeName, treeName) - return - } - - // Compare measuredLayoutReport & trackingLayoutReport computing discrepencies - - for objectNumber, objectBytesMeasured = range measuredLayoutReport { - objectBytesTracked, ok = treeWrapper.bPlusTreeTracker.bPlusTreeLayout[objectNumber] - if ok { - if objectBytesMeasured != objectBytesTracked { - discrepencies++ - logger.Errorf("headhunter.fetchLayoutReport(%v) for volume %v found objectBytes mismatch between measuredLayoutReport (0x%016X) & trackingLayoutReport (0x%016X) for objectNumber 0x%016X", treeName, volume.volumeName, objectBytesMeasured, objectBytesTracked, objectNumber) - } - } else { - if 0 != objectBytesMeasured { - discrepencies++ - logger.Errorf("headhunter.fetchLayoutReport(%v) for volume %v found objectBytes in measuredLayoutReport (0x%016X) but missing from trackingLayoutReport for objectNumber 0x%016X", treeName, volume.volumeName, objectBytesMeasured, objectNumber) - } - } - } - - for objectNumber, objectBytesTracked = range treeWrapper.bPlusTreeTracker.bPlusTreeLayout { - objectBytesMeasured, ok = measuredLayoutReport[objectNumber] - if ok { - // Already handled above - } else { - if 0 != objectBytesTracked { - discrepencies++ - logger.Errorf("headhunter.fetchLayoutReport(%v) for volume %v found objectBytes in trackingLayoutReport (0x%016X) but missing from measuredLayoutReport for objectNumber 0x%016X", treeName, volume.volumeName, objectBytesTracked, objectNumber) - } - } - } - - // In the case that they differ, return measuredLayoutReport rather than trackingLayoutReport - - layoutReport = measuredLayoutReport - } else { - // Not validating, so just clone treeWrapper.bPlusTreeTracker.bPlusTreeLayout - - layoutReport = make(sortedmap.LayoutReport) - - for objectNumber, objectBytesTracked = range treeWrapper.bPlusTreeTracker.bPlusTreeLayout { - layoutReport[objectNumber] = objectBytesTracked - } - - err = nil - } - - return -} - -// FetchLayoutReport returns the B+Tree sortedmap.LayoutReport for one or all -// of the HeadHunter tables. In the case of requesting "all", the checkpoint -// overhead will also be included. -// -// Note that only the "live" view information is reported. If prior SnapShot's -// exist, these will not be reported (even t for the "all" case). This is -// primarily due to the fact that SnapShot's inherently overlap in the objects -// they reference. In the case where one or more SnapShot's exist, the -// CreatedObjectsBPlusTree should be expected to be empty. -func (volume *volumeStruct) FetchLayoutReport(treeType BPlusTreeType, validate bool) (layoutReport sortedmap.LayoutReport, discrepencies uint64, err error) { - var ( - checkpointLayoutReport sortedmap.LayoutReport - checkpointObjectBytes uint64 - discrepenciesToAdd uint64 - objectBytes uint64 - objectNumber uint64 - ok bool - perTreeLayoutReport sortedmap.LayoutReport - perTreeObjectBytes uint64 - ) - - startTime := time.Now() - defer func() { - globals.FetchLayoutReportUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.FetchLayoutReportErrors.Add(1) - } - }() - - volume.Lock() - defer volume.Unlock() - - if MergedBPlusTree == treeType { - // First, accumulate the 5 B+Tree sortedmap.LayoutReport's - - layoutReport, discrepencies, err = volume.fetchLayoutReport(InodeRecBPlusTree, validate) - if nil != err { - return - } - perTreeLayoutReport, discrepenciesToAdd, err = volume.fetchLayoutReport(LogSegmentRecBPlusTree, validate) - if nil != err { - return - } - discrepencies += discrepenciesToAdd - for objectNumber, perTreeObjectBytes = range perTreeLayoutReport { - objectBytes, ok = layoutReport[objectNumber] - if ok { - layoutReport[objectNumber] = objectBytes + perTreeObjectBytes - } else { - layoutReport[objectNumber] = perTreeObjectBytes - } - } - perTreeLayoutReport, discrepenciesToAdd, err = volume.fetchLayoutReport(BPlusTreeObjectBPlusTree, validate) - if nil != err { - return - } - discrepencies += discrepenciesToAdd - for objectNumber, perTreeObjectBytes = range perTreeLayoutReport { - objectBytes, ok = layoutReport[objectNumber] - if ok { - layoutReport[objectNumber] = objectBytes + perTreeObjectBytes - } else { - layoutReport[objectNumber] = perTreeObjectBytes - } - } - perTreeLayoutReport, discrepenciesToAdd, err = volume.fetchLayoutReport(CreatedObjectsBPlusTree, validate) - if nil != err { - return - } - discrepencies += discrepenciesToAdd - for objectNumber, perTreeObjectBytes = range perTreeLayoutReport { - objectBytes, ok = layoutReport[objectNumber] - if ok { - layoutReport[objectNumber] = objectBytes + perTreeObjectBytes - } else { - layoutReport[objectNumber] = perTreeObjectBytes - } - } - perTreeLayoutReport, discrepenciesToAdd, err = volume.fetchLayoutReport(DeletedObjectsBPlusTree, validate) - if nil != err { - return - } - discrepencies += discrepenciesToAdd - for objectNumber, perTreeObjectBytes = range perTreeLayoutReport { - objectBytes, ok = layoutReport[objectNumber] - if ok { - layoutReport[objectNumber] = objectBytes + perTreeObjectBytes - } else { - layoutReport[objectNumber] = perTreeObjectBytes - } - } - - // Now, add in the checkpointLayoutReport - - checkpointLayoutReport, err = volume.fetchCheckpointLayoutReport() - if nil != err { - return - } - for objectNumber, checkpointObjectBytes = range checkpointLayoutReport { - objectBytes, ok = layoutReport[objectNumber] - if ok { - layoutReport[objectNumber] = objectBytes + checkpointObjectBytes - } else { - layoutReport[objectNumber] = perTreeObjectBytes - } - } - } else { - layoutReport, discrepencies, err = volume.fetchLayoutReport(treeType, validate) - } - - return -} - -func (volume *volumeStruct) DefragmentMetadata(treeType BPlusTreeType, thisStartPercentage float64, thisStopPercentage float64) (err error) { - var ( - bPlusTree sortedmap.BPlusTree - bPlusTreeLenAsInt int - bPlusTreeLenAsBigFloat *big.Float - nextItemIndexToTouch uint64 - oneHundredAsBigFloat *big.Float - startIndexAsBigFloat *big.Float - startIndexAsU64 uint64 - stopIndexAsBigFloat *big.Float - stopIndexAsU64 uint64 - thisItemIndexToTouch uint64 - treeWrapper *bPlusTreeWrapperStruct - ) - - if 0 > thisStartPercentage { - err = fmt.Errorf("thisStartPercentage (%v) must be >= 0", thisStartPercentage) - return - } - if 100 <= thisStartPercentage { - err = fmt.Errorf("thisStartPercentage (%v) must be < 100", thisStartPercentage) - return - } - if 100 < thisStopPercentage { - err = fmt.Errorf("thisStopPercentage (%v) must be <= 100", thisStopPercentage) - return - } - if thisStartPercentage >= thisStopPercentage { - err = fmt.Errorf("thisStartPercentage (%v) must be < thisStopPercentage (%v)", thisStartPercentage, thisStopPercentage) - } - - volume.Lock() - - volume.checkpointTriggeringEvents++ - - switch treeType { - case InodeRecBPlusTree: - treeWrapper = volume.liveView.inodeRecWrapper - case LogSegmentRecBPlusTree: - treeWrapper = volume.liveView.logSegmentRecWrapper - case BPlusTreeObjectBPlusTree: - treeWrapper = volume.liveView.bPlusTreeObjectWrapper - case CreatedObjectsBPlusTree: - treeWrapper = volume.liveView.createdObjectsWrapper - case DeletedObjectsBPlusTree: - treeWrapper = volume.liveView.deletedObjectsWrapper - default: - err = fmt.Errorf("DefragmentMetadata(treeType %d): bad tree type", treeType) - logger.ErrorfWithError(err, "volume '%s'", volume.volumeName) - return - } - - if nil == treeWrapper { - // Just return... apparently there is no B+Tree for this treeType [Case 1] - volume.Unlock() - err = nil - return - } - - bPlusTree = treeWrapper.bPlusTree - - if nil == bPlusTree { - // Just return... apparently there is no B+Tree for this treeType [Case 2] - volume.Unlock() - err = nil - return - } - - bPlusTreeLenAsInt, err = bPlusTree.Len() - if nil != err { - volume.Unlock() - return - } - - if 0 == bPlusTreeLenAsInt { - volume.Unlock() - err = nil - return - } - - bPlusTreeLenAsBigFloat = big.NewFloat(100) - _ = bPlusTreeLenAsBigFloat.SetInt64(int64(bPlusTreeLenAsInt)) - - oneHundredAsBigFloat = big.NewFloat(100) - - if 0 == thisStartPercentage { - startIndexAsU64 = 0 - } else { - startIndexAsBigFloat = big.NewFloat(thisStartPercentage) - _ = startIndexAsBigFloat.Mul(startIndexAsBigFloat, bPlusTreeLenAsBigFloat) - _ = startIndexAsBigFloat.Quo(startIndexAsBigFloat, oneHundredAsBigFloat) - startIndexAsU64, _ = startIndexAsBigFloat.Uint64() - } - - if 100 == thisStopPercentage { - stopIndexAsU64 = uint64(bPlusTreeLenAsInt) - } else { - stopIndexAsBigFloat = big.NewFloat(thisStopPercentage) - _ = stopIndexAsBigFloat.Mul(stopIndexAsBigFloat, bPlusTreeLenAsBigFloat) - _ = stopIndexAsBigFloat.Quo(stopIndexAsBigFloat, oneHundredAsBigFloat) - stopIndexAsU64, _ = stopIndexAsBigFloat.Uint64() - } - - thisItemIndexToTouch = startIndexAsU64 - - nextItemIndexToTouch, err = bPlusTree.TouchItem(thisItemIndexToTouch) - if nil != err { - volume.Unlock() - return - } - - for nextItemIndexToTouch < stopIndexAsU64 { - if 0 == nextItemIndexToTouch { - volume.Unlock() - err = nil - return - } - - thisItemIndexToTouch = nextItemIndexToTouch - - nextItemIndexToTouch, err = bPlusTree.TouchItem(thisItemIndexToTouch) - if nil != err { - volume.Unlock() - return - } - } - - volume.Unlock() - - err = nil - return -} - -func (volume *volumeStruct) SnapShotCreateByInodeLayer(name string) (id uint64, err error) { - var ( - availableSnapShotIDListElement *list.Element - bPlusTreeObjectBPlusTreeRootObjectLength uint64 - bPlusTreeObjectBPlusTreeRootObjectNumber uint64 - bPlusTreeObjectBPlusTreeRootObjectOffset uint64 - inodeRecBPlusTreeRootObjectLength uint64 - inodeRecBPlusTreeRootObjectNumber uint64 - inodeRecBPlusTreeRootObjectOffset uint64 - logSegmentRecBPlusTreeRootObjectLength uint64 - logSegmentRecBPlusTreeRootObjectNumber uint64 - logSegmentRecBPlusTreeRootObjectOffset uint64 - ok bool - snapShotNonce uint64 - snapShotTime time.Time - volumeView *volumeViewStruct - ) - - startTime := time.Now() - defer func() { - globals.SnapShotCreateByInodeLayerUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.SnapShotCreateByInodeLayerErrors.Add(1) - } - }() - - volume.Lock() - - volume.checkpointTriggeringEvents++ - - _, ok, err = volume.viewTreeByName.GetByKey(name) - if nil != err { - volume.Unlock() - return - } - if ok { - volume.Unlock() - err = fmt.Errorf("SnapShot Name \"%v\" already in use", name) - return - } - - ok = true - - for ok { - snapShotTime = time.Now() - _, ok, err = volume.viewTreeByTime.GetByKey(snapShotTime) - if nil != err { - volume.Unlock() - return - } - } - - if 0 == volume.availableSnapShotIDList.Len() { - volume.Unlock() - err = fmt.Errorf("No SnapShot IDs available") - return - } - - snapShotNonce = volume.fetchNonceWhileLocked() - - availableSnapShotIDListElement = volume.availableSnapShotIDList.Front() - id, ok = availableSnapShotIDListElement.Value.(uint64) - if !ok { - logger.Fatalf("Logic error - volume %v has non-uint64 element at Front() of availableSnapShotIDList", volume.volumeName) - } - _ = volume.availableSnapShotIDList.Remove(availableSnapShotIDListElement) - - evtlog.Record(evtlog.FormatHeadhunterCheckpointStart, volume.volumeName) - err = volume.putCheckpoint() - if nil != err { - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndFailure, volume.volumeName, err.Error()) - logger.FatalfWithError(err, "Shutting down to prevent subsequent checkpoints from corrupting Swift") - } - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndSuccess, volume.volumeName) - - volume.checkpointTriggeringEvents++ - - volumeView = &volumeViewStruct{ - volume: volume, - nonce: snapShotNonce, - snapShotID: id, - snapShotTime: snapShotTime, - snapShotName: name, - } - - volumeView.inodeRecWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: nil, - } - - inodeRecBPlusTreeRootObjectNumber, - inodeRecBPlusTreeRootObjectOffset, - inodeRecBPlusTreeRootObjectLength = volume.liveView.inodeRecWrapper.bPlusTree.FetchLocation() - - if 0 == inodeRecBPlusTreeRootObjectNumber { - volumeView.inodeRecWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volumeView.volume.maxInodesPerMetadataNode, - sortedmap.CompareUint64, - volumeView.inodeRecWrapper, - globals.inodeRecCache) - } else { - volumeView.inodeRecWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - inodeRecBPlusTreeRootObjectNumber, - inodeRecBPlusTreeRootObjectOffset, - inodeRecBPlusTreeRootObjectLength, - sortedmap.CompareUint64, - volumeView.inodeRecWrapper, - globals.inodeRecCache) - if nil != err { - logger.Fatalf("Logic error - sortedmap.OldBPlusTree() failed with error: %v", err) - } - } - - volumeView.logSegmentRecWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: nil, - } - - logSegmentRecBPlusTreeRootObjectNumber, - logSegmentRecBPlusTreeRootObjectOffset, - logSegmentRecBPlusTreeRootObjectLength = volume.liveView.logSegmentRecWrapper.bPlusTree.FetchLocation() - - if 0 == logSegmentRecBPlusTreeRootObjectNumber { - volumeView.logSegmentRecWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volumeView.volume.maxLogSegmentsPerMetadataNode, - sortedmap.CompareUint64, - volumeView.logSegmentRecWrapper, - globals.logSegmentRecCache) - } else { - volumeView.logSegmentRecWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - logSegmentRecBPlusTreeRootObjectNumber, - logSegmentRecBPlusTreeRootObjectOffset, - logSegmentRecBPlusTreeRootObjectLength, - sortedmap.CompareUint64, - volumeView.logSegmentRecWrapper, - globals.logSegmentRecCache) - if nil != err { - logger.Fatalf("Logic error - sortedmap.OldBPlusTree() failed with error: %v", err) - } - } - - volumeView.bPlusTreeObjectWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: nil, - } - - bPlusTreeObjectBPlusTreeRootObjectNumber, - bPlusTreeObjectBPlusTreeRootObjectOffset, - bPlusTreeObjectBPlusTreeRootObjectLength = volume.liveView.bPlusTreeObjectWrapper.bPlusTree.FetchLocation() - - if 0 == bPlusTreeObjectBPlusTreeRootObjectNumber { - volumeView.bPlusTreeObjectWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volumeView.volume.maxDirFileNodesPerMetadataNode, - sortedmap.CompareUint64, - volumeView.bPlusTreeObjectWrapper, - globals.bPlusTreeObjectCache) - } else { - volumeView.bPlusTreeObjectWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - bPlusTreeObjectBPlusTreeRootObjectNumber, - bPlusTreeObjectBPlusTreeRootObjectOffset, - bPlusTreeObjectBPlusTreeRootObjectLength, - sortedmap.CompareUint64, - volumeView.bPlusTreeObjectWrapper, - globals.bPlusTreeObjectCache) - if nil != err { - logger.Fatalf("Logic error - sortedmap.OldBPlusTree() failed with error: %v", err) - } - } - - volumeView.createdObjectsWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: volumeView.volume.liveView.createdObjectsWrapper.bPlusTreeTracker, - } - - volumeView.createdObjectsWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volumeView.volume.maxCreatedDeletedObjectsPerMetadataNode, - sortedmap.CompareUint64, - volumeView.volume.liveView.createdObjectsWrapper, - globals.createdDeletedObjectsCache) - - volumeView.deletedObjectsWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: volumeView.volume.liveView.deletedObjectsWrapper.bPlusTreeTracker, - } - - volumeView.deletedObjectsWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volumeView.volume.maxCreatedDeletedObjectsPerMetadataNode, - sortedmap.CompareUint64, - volumeView.volume.liveView.deletedObjectsWrapper, - globals.createdDeletedObjectsCache) - - ok, err = volume.viewTreeByNonce.Put(snapShotNonce, volumeView) - if nil != err { - logger.Fatalf("Logic error - viewTreeByNonce.Put() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByNonce.Put() returned ok == false") - } - - ok, err = volume.viewTreeByID.Put(id, volumeView) - if nil != err { - logger.Fatalf("Logic error - viewTreeByID.Put() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByID.Put() returned ok == false") - } - - ok, err = volume.viewTreeByTime.Put(snapShotTime, volumeView) - if nil != err { - logger.Fatalf("Logic error - viewTreeByTime.Put() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByTime.Put() returned ok == false") - } - - ok, err = volume.viewTreeByName.Put(name, volumeView) - if nil != err { - logger.Fatalf("Logic error - viewTreeByName.Put() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByName.Put() returned ok == false") - } - - volume.priorView = volumeView - - evtlog.Record(evtlog.FormatHeadhunterCheckpointStart, volume.volumeName) - err = volume.putCheckpoint() - if nil != err { - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndFailure, volume.volumeName, err.Error()) - logger.FatalfWithError(err, "Shutting down to prevent subsequent checkpoints from corrupting Swift") - } - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndSuccess, volume.volumeName) - - volume.Unlock() - - return -} - -func (volume *volumeStruct) SnapShotDeleteByInodeLayer(id uint64) (err error) { - var ( - deletedVolumeView *volumeViewStruct - deletedVolumeViewIndex int - found bool - key sortedmap.Key - newPriorVolumeView *volumeViewStruct - ok bool - predecessorToDeletedVolumeView *volumeViewStruct - remainingSnapShotCount int - value sortedmap.Value - ) - - startTime := time.Now() - defer func() { - globals.SnapShotDeleteByInodeLayerUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.SnapShotDeleteByInodeLayerErrors.Add(1) - } - }() - - volume.Lock() - - volume.checkpointTriggeringEvents++ - - value, ok, err = volume.viewTreeByID.GetByKey(id) - if nil != err { - volume.Unlock() - return - } - if !ok { - volume.Unlock() - err = fmt.Errorf("viewTreeByID.GetByKey(0x%016X) not found", id) - return - } - - deletedVolumeView, ok = value.(*volumeViewStruct) - if !ok { - logger.Fatalf("viewTreeByID.GetByKey(0x%016X) returned something other than a volumeView", id) - } - deletedVolumeViewIndex, found, err = volume.viewTreeByNonce.BisectLeft(deletedVolumeView.nonce) - if nil != err { - logger.Fatalf("Logic error - viewTreeByNonce.BisectLeft() failed with error: %v", err) - } - if !found { - logger.Fatalf("Logic error - viewTreeByNonce.BisectLeft() returned found == false") - } - - if 0 == deletedVolumeViewIndex { - ok = true - for ok { - ok, err = deletedVolumeView.createdObjectsWrapper.bPlusTree.DeleteByIndex(0) - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.createdObjectsWrapper.bPlusTree.DeleteByIndex(0) failed with error: %v", err) - } - } - ok = true - for ok { - key, value, ok, err = deletedVolumeView.deletedObjectsWrapper.bPlusTree.GetByIndex(0) - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.deletedObjectsWrapper.bPlusTree.GetByIndex(0) failed with error: %v", err) - } - if ok { - ok, err = deletedVolumeView.deletedObjectsWrapper.bPlusTree.DeleteByIndex(0) - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.deletedObjectsWrapper.bPlusTree.DeleteByIndex(0) failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - deletedVolumeView.deletedObjectsWrapper.bPlusTree.DeleteByIndex(0) returned ok == false") - } - ok, err = volume.liveView.deletedObjectsWrapper.bPlusTree.Put(key, value) - if nil != err { - logger.Fatalf("Logic error - liveView.deletedObjectsWrapper.bPlusTree.Put() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - liveView.deletedObjectsWrapper.bPlusTree.Put() returned ok == false") - } - } - } - } else { - _, value, ok, err = volume.viewTreeByNonce.GetByIndex(deletedVolumeViewIndex - 1) - if nil != err { - logger.Fatalf("Logic error - viewTreeByNonce.GetByIndex() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByNonce.GetByIndex() returned ok == false") - } - predecessorToDeletedVolumeView, ok = value.(*volumeViewStruct) - if !ok { - logger.Fatalf("Logic error - viewTreeByNonce.GetByIndex() returned something other than a volumeView") - } - - ok = true - for ok { - key, value, ok, err = deletedVolumeView.createdObjectsWrapper.bPlusTree.GetByIndex(0) - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.createdObjectsWrapper.bPlusTree.GetByIndex(0) failed with error: %v", err) - } - if ok { - ok, err = deletedVolumeView.createdObjectsWrapper.bPlusTree.DeleteByIndex(0) - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.createdObjectsWrapper.bPlusTree.DeleteByIndex(0) failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - deletedVolumeView.createdObjectsWrapper.bPlusTree.DeleteByIndex(0) returned ok == false") - } - ok, err = predecessorToDeletedVolumeView.createdObjectsWrapper.bPlusTree.Put(key, value) - if nil != err { - logger.Fatalf("Logic error - predecessorToDeletedVolumeView.createdObjectsWrapper.bPlusTree.Put() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - predecessorToDeletedVolumeView.createdObjectsWrapper.bPlusTree.Put() returned ok == false") - } - } - } - ok = true - for ok { - key, value, ok, err = deletedVolumeView.deletedObjectsWrapper.bPlusTree.GetByIndex(0) - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.deletedObjectsWrapper.bPlusTree.GetByIndex(0) failed with error: %v", err) - } - if ok { - ok, err = deletedVolumeView.deletedObjectsWrapper.bPlusTree.DeleteByIndex(0) - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.deletedObjectsWrapper.bPlusTree.DeleteByIndex(0) failed with error: %v", err) - } - _, found, err = predecessorToDeletedVolumeView.createdObjectsWrapper.bPlusTree.BisectLeft(key) - if nil != err { - logger.Fatalf("Logic error - predecessorToDeletedVolumeView.createdObjectsWrapper.bPlusTree.BisectLeft(key) failed with error: %v", err) - } - if found { - ok, err = predecessorToDeletedVolumeView.createdObjectsWrapper.bPlusTree.DeleteByKey(key) - if nil != err { - logger.Fatalf("Logic error - predecessorToDeletedVolumeView.createdObjectsWrapper.bPlusTree.DeleteByKey(key) failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - predecessorToDeletedVolumeView.createdObjectsWrapper.bPlusTree.DeleteByKey(key) returned ok == false") - } - ok, err = volume.liveView.deletedObjectsWrapper.bPlusTree.Put(key, value) - if nil != err { - logger.Fatalf("Logic error - liveView.deletedObjectsWrapper.bPlusTree.Put() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - liveView.deletedObjectsWrapper.bPlusTree.Put() returned ok == false") - } - } else { - ok, err = predecessorToDeletedVolumeView.deletedObjectsWrapper.bPlusTree.Put(key, value) - if nil != err { - logger.Fatalf("Logic error - predecessorToDeletedVolumeView.deletedObjectsWrapper.bPlusTree.Put() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - predecessorToDeletedVolumeView.deletedObjectsWrapper.bPlusTree.Put() returned ok == false") - } - } - } - } - } - - evtlog.Record(evtlog.FormatHeadhunterCheckpointStart, volume.volumeName) - err = volume.putCheckpoint() - if nil != err { - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndFailure, volume.volumeName, err.Error()) - logger.FatalfWithError(err, "Shutting down to prevent subsequent checkpoints from corrupting Swift") - } - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndSuccess, volume.volumeName) - - volume.checkpointTriggeringEvents++ - - ok, err = volume.viewTreeByNonce.DeleteByKey(deletedVolumeView.nonce) - if nil != err { - logger.Fatalf("Logic error - viewTreeByNonce.DeleteByKey() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByNonce.DeleteByKey() returned ok == false") - } - - ok, err = volume.viewTreeByID.DeleteByKey(id) - if nil != err { - logger.Fatalf("Logic error - viewTreeByID.DeleteByKey() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByID.DeleteByKey() returned ok == false") - } - - ok, err = volume.viewTreeByTime.DeleteByKey(deletedVolumeView.snapShotTime) - if nil != err { - logger.Fatalf("Logic error - viewTreeByTime.DeleteByKey() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByTime.DeleteByKey() returned ok == false") - } - - ok, err = volume.viewTreeByName.DeleteByKey(deletedVolumeView.snapShotName) - if nil != err { - logger.Fatalf("Logic error - viewTreeByName.DeleteByKey() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByName.DeleteByKey() returned ok == false") - } - - remainingSnapShotCount, err = volume.viewTreeByNonce.Len() - if nil != err { - logger.Fatalf("Logic error - viewTreeByNonce.Len() failed with error: %v", err) - } - - if 0 == remainingSnapShotCount { - volume.priorView = nil - } else { - _, value, ok, err = volume.viewTreeByNonce.GetByIndex(remainingSnapShotCount - 1) - if nil != err { - logger.Fatalf("Logic error - viewTreeByNonce.GetByIndex() failed with error: %v", err) - } - if !ok { - logger.Fatalf("Logic error - viewTreeByNonce.GetByIndex() returned ok == false") - } - newPriorVolumeView, ok = value.(*volumeViewStruct) - if !ok { - logger.Fatalf("Logic error - viewTreeByNonce.GetByIndex() returned something other than a volumeView") - } - volume.priorView = newPriorVolumeView - } - - volume.availableSnapShotIDList.PushBack(id) - - evtlog.Record(evtlog.FormatHeadhunterCheckpointStart, volume.volumeName) - err = volume.putCheckpoint() - if nil != err { - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndFailure, volume.volumeName, err.Error()) - logger.FatalfWithError(err, "Shutting down to prevent subsequent checkpoints from corrupting Swift") - } - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndSuccess, volume.volumeName) - - err = deletedVolumeView.inodeRecWrapper.bPlusTree.Prune() - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.inodeRecWrapper.bPlusTree.Prune() failed with error: %v", err) - } - - err = deletedVolumeView.logSegmentRecWrapper.bPlusTree.Prune() - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.logSegmentRecWrapper.bPlusTree.Prune() failed with error: %v", err) - } - - err = deletedVolumeView.bPlusTreeObjectWrapper.bPlusTree.Prune() - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.bPlusTreeObjectWrapper.bPlusTree.Prune() failed with error: %v", err) - } - - err = deletedVolumeView.createdObjectsWrapper.bPlusTree.Prune() - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.createdObjectsWrapper.bPlusTree.Prune() failed with error: %v", err) - } - - err = deletedVolumeView.deletedObjectsWrapper.bPlusTree.Prune() - if nil != err { - logger.Fatalf("Logic error - deletedVolumeView.deletedObjectsWrapper.bPlusTree.Prune() failed with error: %v", err) - } - - volume.Unlock() - - return -} - -func (volume *volumeStruct) SnapShotCount() (snapShotCount uint64) { - var ( - err error - len int - ) - - startTime := time.Now() - defer func() { - globals.SnapShotCountUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.SnapShotCountErrors.Add(1) - } - }() - - len, err = volume.viewTreeByID.Len() - if nil != err { - logger.Fatalf("headhunter.SnapShotCount() for volume %v hit viewTreeByID.Len() error: %v", volume.volumeName, err) - } - snapShotCount = uint64(len) - return -} - -func (volume *volumeStruct) SnapShotLookupByName(name string) (snapShot SnapShotStruct, ok bool) { - var ( - err error - value sortedmap.Value - volumeView *volumeViewStruct - ) - - startTime := time.Now() - defer func() { - globals.SnapShotLookupByNameUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.SnapShotLookupByNameErrors.Add(1) - } - }() - - value, ok, err = volume.viewTreeByName.GetByKey(name) - if nil != err { - logger.Fatalf("headhunter.SnapShotLookupByName() for volume %v hit viewTreeByName.GetByKey(\"%v\") error: %v", volume.volumeName, name, err) - } - if !ok { - return - } - - volumeView, ok = value.(*volumeViewStruct) - if !ok { - logger.Fatalf("headhunter.SnapShotLookupByName() for volume %v hit value.(*volumeViewStruct) !ok", volume.volumeName) - } - - snapShot = SnapShotStruct{ - ID: volumeView.snapShotID, - Time: volumeView.snapShotTime, - Name: volumeView.snapShotName, // == name - } - - return -} - -func (volume *volumeStruct) snapShotList(viewTree sortedmap.LLRBTree, reversed bool) (list []SnapShotStruct) { - var ( - err error - len int - listIndex int - ok bool - treeIndex int - value sortedmap.Value - volumeView *volumeViewStruct - ) - - len, err = viewTree.Len() - if nil != err { - logger.Fatalf("headhunter.snapShotList() for volume %v hit viewTree.Len() error: %v", volume.volumeName, err) - } - - list = make([]SnapShotStruct, len, len) - - for treeIndex = 0; treeIndex < len; treeIndex++ { - if reversed { - listIndex = len - 1 - treeIndex - } else { - listIndex = treeIndex - } - - _, value, ok, err = viewTree.GetByIndex(treeIndex) - if nil != err { - logger.Fatalf("headhunter.snapShotList() for volume %v hit viewTree.GetByIndex() error: %v", volume.volumeName, err) - } - if !ok { - logger.Fatalf("headhunter.snapShotList() for volume %v hit viewTree.GetByIndex() !ok", volume.volumeName) - } - - volumeView, ok = value.(*volumeViewStruct) - if !ok { - logger.Fatalf("headhunter.snapShotList() for volume %v hit value.(*volumeViewStruct) !ok", volume.volumeName) - } - - list[listIndex].ID = volumeView.snapShotID - list[listIndex].Time = volumeView.snapShotTime - list[listIndex].Name = volumeView.snapShotName - } - - return -} - -func (volume *volumeStruct) SnapShotListByID(reversed bool) (list []SnapShotStruct) { - - startTime := time.Now() - defer func() { - globals.SnapShotListByIDUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - volume.Lock() - list = volume.snapShotList(volume.viewTreeByID, reversed) - volume.Unlock() - - return -} - -func (volume *volumeStruct) SnapShotListByTime(reversed bool) (list []SnapShotStruct) { - - startTime := time.Now() - defer func() { - globals.SnapShotListByTimeUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - volume.Lock() - list = volume.snapShotList(volume.viewTreeByTime, reversed) - volume.Unlock() - - return -} - -func (volume *volumeStruct) SnapShotListByName(reversed bool) (list []SnapShotStruct) { - - startTime := time.Now() - defer func() { - globals.SnapShotListByNameUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - volume.Lock() - list = volume.snapShotList(volume.viewTreeByName, reversed) - volume.Unlock() - - return -} - -func (volume *volumeStruct) SnapShotU64Decode(snapShotU64 uint64) (snapShotIDType SnapShotIDType, snapShotID uint64, nonce uint64) { - - startTime := time.Now() - defer func() { - globals.SnapShotU64DecodeUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - snapShotID = snapShotU64 >> volume.snapShotIDShift - if 0 == snapShotID { - snapShotIDType = SnapShotIDTypeLive - } else if volume.dotSnapShotDirSnapShotID == snapShotID { - snapShotIDType = SnapShotIDTypeDotSnapShot - } else { - snapShotIDType = SnapShotIDTypeSnapShot - } - - nonce = snapShotU64 & volume.snapShotU64NonceMask - - return -} - -func (volume *volumeStruct) SnapShotIDAndNonceEncode(snapShotID uint64, nonce uint64) (snapShotU64 uint64) { - - startTime := time.Now() - defer func() { - globals.SnapShotIDAndNonceEncodeUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - snapShotU64 = snapShotID - snapShotU64 = snapShotU64 << volume.snapShotIDShift - snapShotU64 = snapShotU64 | nonce - - return -} - -func (volume *volumeStruct) SnapShotTypeDotSnapShotAndNonceEncode(nonce uint64) (snapShotU64 uint64) { - - startTime := time.Now() - defer func() { - globals.SnapShotTypeDotSnapShotAndNonceEncodeUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - snapShotU64 = volume.dotSnapShotDirSnapShotID - snapShotU64 = snapShotU64 << volume.snapShotIDShift - snapShotU64 = snapShotU64 | nonce - - return -} diff --git a/headhunter/api_test.go b/headhunter/api_test.go deleted file mode 100644 index 2557ef9a..00000000 --- a/headhunter/api_test.go +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package headhunter - -import ( - "bytes" - "sync" - "testing" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/ramswift" - "github.com/NVIDIA/proxyfs/transitions" -) - -func inodeRecPutGet(t *testing.T, volume VolumeHandle, key uint64, value []byte) { - err := volume.PutInodeRec(key, value) - if nil != err { - t.Fatalf("Failed to Put %d %s : %v", key, value, err) - } - - value1, ok, err := volume.GetInodeRec(key) - if nil != err || !ok { - t.Fatalf("Failed to Get %d %s : %v", key, value, err) - } - - if 0 != bytes.Compare(value, value1) { - t.Fatalf("Get Value does not match Inital Value: %d %v %v", key, value, err) - } -} - -func logsegmentRecPutGet(t *testing.T, volume VolumeHandle, key uint64, value []byte) { - err := volume.PutLogSegmentRec(key, value) - if nil != err { - t.Fatalf("Failed to Put %d %s : %v", key, value, err) - } - - value1, err := volume.GetLogSegmentRec(key) - if nil != err { - t.Fatalf("Failed to Get %d %s : %v", key, value, err) - } - - if 0 != bytes.Compare(value, value1) { - t.Fatalf("Get Value does not match Inital Value: %d %v %v", key, value, err) - } -} - -func putInodeRecsTest(t *testing.T, volume VolumeHandle) { - var keys []uint64 - var values [][]byte - - keys = make([]uint64, 10) - values = make([][]byte, 10) - - for i := 0; i < 10; i++ { - keys[i] = uint64(i) - } - - for i := 0; i < 10; i++ { - values[i] = make([]byte, 10) - values[i] = []byte("Value 1") - } - - err := volume.PutInodeRecs(keys, values) - if nil != err { - t.Fatalf("Failed to PutInodeRecs: %v", err) - } - - for i := 0; i < 10; i++ { - var value []byte - value, ok, err := volume.GetInodeRec(keys[i]) - if err != nil || !ok { - t.Fatalf("Unable to get inode %d", keys[i]) - } - if bytes.Compare(value, values[i]) != 0 { - t.Fatalf("Get Value does not match Initial Value: %v %v %v", keys[i], values[i], value[i]) - } - - err = volume.DeleteInodeRec(keys[i]) - if nil != err { - t.Fatalf("Failed to delete InodeRec key: %v %v", keys[i], err) - } - } -} - -func TestHeadHunterAPI(t *testing.T) { - var ( - confMap conf.ConfMap - confStrings []string - doneChan chan bool - err error - firstUpNonce uint64 - key uint64 - /* - // The following is now obsolete given the deprecation of ReplayLog in practice - - replayLogFile *os.File - replayLogFileName string - */ - secondUpNonce uint64 - signalHandlerIsArmedWG sync.WaitGroup - value []byte - volume VolumeHandle - ) - - confStrings = []string{ - "Logging.LogFilePath=/dev/null", - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - "SwiftClient.NoAuthIPAddr=127.0.0.1", - - "TrackedLock.LockHoldTimeLimit=0s", - "TrackedLock.LockCheckPeriod=0s", - - "SwiftClient.NoAuthTCPPort=9999", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=0", - "SwiftClient.RetryLimitObject=0", - "SwiftClient.RetryDelay=1s", - "SwiftClient.RetryDelayObject=1s", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=64", - "SwiftClient.NonChunkedConnectionPoolSize=32", - "Cluster.WhoAmI=Peer0", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - "Volume:TestVolume.PrimaryPeer=Peer0", - "Volume:TestVolume.AccountName=TestAccount", - "Volume:TestVolume.CheckpointContainerName=.__checkpoint__", - "Volume:TestVolume.CheckpointContainerStoragePolicy=gold", - "Volume:TestVolume.CheckpointInterval=10s", - "Volume:TestVolume.MaxFlushSize=10000000", - "Volume:TestVolume.NonceValuesToReserve=100", - "Volume:TestVolume.MaxInodesPerMetadataNode=32", - "Volume:TestVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:TestVolume.MaxDirFileNodesPerMetadataNode=16", - "VolumeGroup:TestVolumeGroup.VolumeList=TestVolume", - "VolumeGroup:TestVolumeGroup.VirtualIPAddr=", - "VolumeGroup:TestVolumeGroup.PrimaryPeer=Peer0", - "FSGlobals.VolumeGroupList=TestVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - } - - /* - // The following is now obsolete given the deprecation of ReplayLog in practice - - // Construct replayLogFileName to use as Volume:TestVolume.ReplayLogFileName - - replayLogFile, err = ioutil.TempFile("", "TestVolume_Replay_Log_") - if nil != err { - t.Fatalf("ioutil.TempFile() returned error: %v", err) - } - - replayLogFileName = replayLogFile.Name() - - err = replayLogFile.Close() - if nil != err { - t.Fatalf("replayLogFile.Close() returned error: %v", err) - } - - err = os.Remove(replayLogFileName) - if nil != err { - t.Fatalf("os.Remove(replayLogFileName) returned error: %v", err) - } - - confStrings = append(confStrings, "Volume:TestVolume.ReplayLogFileName="+replayLogFileName) - */ - - // Launch a ramswift instance - - signalHandlerIsArmedWG.Add(1) - doneChan = make(chan bool, 1) // Must be buffered to avoid race - - go ramswift.Daemon("/dev/null", confStrings, &signalHandlerIsArmedWG, doneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - confMap, err = conf.MakeConfMapFromStrings(confStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned error: %v", err) - } - - // Schedule a Format of TestVolume on first Up() - - err = confMap.UpdateFromString("Volume:TestVolume.AutoFormat=true") - if nil != err { - t.Fatalf("conf.UpdateFromString(\"Volume:TestVolume.AutoFormat=true\") returned error: %v", err) - } - - // Up packages (TestVolume will be formatted) - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() [case 1] returned error: %v", err) - } - - // Unset AutoFormat for all subsequent uses of ConfMap - - err = confMap.UpdateFromString("Volume:TestVolume.AutoFormat=false") - if nil != err { - t.Fatalf("conf.UpdateFromString(\"Volume:TestVolume.AutoFormat=false\") returned error: %v", err) - } - - // Perform test - - volume, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") [case 1] returned error: %v", err) - } - - volume.RegisterForEvents(nil) - volume.UnregisterForEvents(nil) - - firstUpNonce = volume.FetchNonce() - - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() [case 1] returned error: %v", err) - } - - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() [case 2] returned error: %v", err) - } - - volume, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") [case 2] returned error: %v", err) - } - - secondUpNonce = volume.FetchNonce() - if firstUpNonce >= secondUpNonce { - t.Fatalf("FetchNonce() [case 2] returned unexpected nonce: %v (should have been > %v)", secondUpNonce, firstUpNonce) - } - - key = 1234 - value = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} - - inodeRecPutGet(t, volume, key, value) - - value = []byte{11, 12, 13, 14, 15, 16, 17, 18, 19} - inodeRecPutGet(t, volume, key, value) - - err = volume.DeleteInodeRec(key) - if nil != err { - t.Fatalf("Delete of key %d failed: %v", key, err) - } - - putInodeRecsTest(t, volume) - - logsegmentRecPutGet(t, volume, key, value) - - value = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} - logsegmentRecPutGet(t, volume, key, value) - - err = volume.DeleteLogSegmentRec(key) - if nil != err { - t.Fatalf("Delete of key %d failed: %v", key, err) - } - - // Shutdown packages - - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() [case 2] returned error: %v", err) - } - - /* - // The following is now obsolete given the deprecation of ReplayLog in practice - - err = os.Remove(replayLogFileName) - if nil == err { - t.Fatal("os.Remove(replayLogFileName) should not have succeeded") - } else { - if os.IsNotExist(err) { - // This is what we expect - } else { - t.Fatalf("os.Remove(replayLogFileName) returned unexpected error: %v", err) - } - } - */ - - // Send ourself a SIGTERM to terminate ramswift.Daemon() - - unix.Kill(unix.Getpid(), unix.SIGTERM) - - _ = <-doneChan -} diff --git a/headhunter/checkpoint.go b/headhunter/checkpoint.go deleted file mode 100644 index d6ccbdcf..00000000 --- a/headhunter/checkpoint.go +++ /dev/null @@ -1,3119 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package headhunter - -import ( - "container/list" - "context" - "encoding/json" - "fmt" - "hash/crc64" - "io" - "os" - "strconv" - "strings" - "sync" - "time" - "unsafe" - - etcd "go.etcd.io/etcd/clientv3" - - "github.com/NVIDIA/cstruct" - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/platform" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/utils" -) - -var ( - LittleEndian = cstruct.LittleEndian // All data cstruct's to be serialized in LittleEndian form -) - -type uint64Struct struct { - U64 uint64 -} - -const ( - CheckpointVersion3 uint64 = iota + 3 - // uint64 in %016X indicating checkpointVersion2 or CheckpointVersion3 - // ' ' - // uint64 in %016X indicating objectNumber containing checkpoint record at tail of object - // ' ' - // uint64 in %016X indicating length of checkpoint record at tail of object - // ' ' - // uint64 in %016X indicating reservedToNonce -) - -type CheckpointHeaderStruct struct { - CheckpointVersion uint64 // either checkpointVersion2 or CheckpointVersion3 - CheckpointObjectTrailerStructObjectNumber uint64 // checkpointObjectTrailerV?Struct found at "tail" of object - CheckpointObjectTrailerStructObjectLength uint64 // this length includes appended non-fixed sized arrays - ReservedToNonce uint64 // highest nonce value reserved -} - -type CheckpointObjectTrailerV3Struct struct { - InodeRecBPlusTreeObjectNumber uint64 // if != 0, objectNumber-named Object in . where root of inodeRec B+Tree - InodeRecBPlusTreeObjectOffset uint64 // ...and offset into the Object where root starts - InodeRecBPlusTreeObjectLength uint64 // ...and length if that root node - InodeRecBPlusTreeLayoutNumElements uint64 // elements immediately follow CheckpointObjectTrailerV3Struct - LogSegmentRecBPlusTreeObjectNumber uint64 // if != 0, objectNumber-named Object in . where root of logSegment B+Tree - LogSegmentRecBPlusTreeObjectOffset uint64 // ...and offset into the Object where root starts - LogSegmentRecBPlusTreeObjectLength uint64 // ...and length if that root node - LogSegmentRecBPlusTreeLayoutNumElements uint64 // elements immediately follow inodeRecBPlusTreeLayout - BPlusTreeObjectBPlusTreeObjectNumber uint64 // if != 0, objectNumber-named Object in . where root of bPlusTreeObject B+Tree - BPlusTreeObjectBPlusTreeObjectOffset uint64 // ...and offset into the Object where root starts - BPlusTreeObjectBPlusTreeObjectLength uint64 // ...and length if that root node - BPlusTreeObjectBPlusTreeLayoutNumElements uint64 // elements immediately follow logSegmentRecBPlusTreeLayout - CreatedObjectsBPlusTreeLayoutNumElements uint64 // elements immediately follow bPlusTreeObjectBPlusTreeLayout - DeletedObjectsBPlusTreeLayoutNumElements uint64 // elements immediately follow createdObjectsBPlusTreeLayout - SnapShotIDNumBits uint64 // number of bits reserved to hold SnapShotIDs - SnapShotListNumElements uint64 // elements immediately follow deletedObjectsBPlusTreeLayout - SnapShotListTotalSize uint64 // size of entire SnapShotList - // InodeRecBPlusTreeLayout serialized as [InodeRecBPlusTreeLayoutNumElements ]ElementOfBPlusTreeLayoutStruct - // LogSegmentBPlusTreeLayout serialized as [LogSegmentRecBPlusTreeLayoutNumElements ]ElementOfBPlusTreeLayoutStruct - // BPlusTreeObjectBPlusTreeLayout serialized as [BPlusTreeObjectBPlusTreeLayoutNumElements]ElementOfBPlusTreeLayoutStruct - // CreatedObjectsBPlusTreeLayout serialized as [CreatedObjectsBPlusTreeLayoutNumElements ]ElementOfBPlusTreeLayoutStruct - // DeletedObjectsBPlusTreeLayout serialized as [DeletedObjectsBPlusTreeLayoutNumElements ]ElementOfBPlusTreeLayoutStruct - // SnapShotList serialized as [SnapShotListNumElements ]ElementOfSnapShotListStruct -} - -type ElementOfBPlusTreeLayoutStruct struct { - ObjectNumber uint64 - ObjectBytes uint64 -} - -type ElementOfSnapShotListStruct struct { // Note: for illustrative purposes... not marshalled with cstruct - Nonce uint64 // supplies strict time-ordering of SnapShots regardless of timebase resets - ID uint64 // in the range [1:2^SnapShotIDNumBits-2] - // ID == 0 reserved for the "live" view - // ID == 2^SnapShotIDNumBits-1 reserved for the .snapshot subdir of a dir - TimeStamp time.Time // serialized/deserialized as a uint64 length followed by a that sized []byte - // func (t time.Time) time.MarshalBinary() ([]byte, error) - // func (t *time.Time) time.UnmarshalBinary(data []byte) (error) - Name string // serialized/deserialized as a uint64 length followed by a that sized []byte - // func utils.ByteSliceToString(byteSlice []byte) (str string) - // func utils.StringToByteSlice(str string) (byteSlice []byte) - InodeRecBPlusTreeObjectNumber uint64 - InodeRecBPlusTreeObjectOffset uint64 - InodeRecBPlusTreeObjectLength uint64 - LogSegmentRecBPlusTreeObjectNumber uint64 - LogSegmentRecBPlusTreeObjectOffset uint64 - LogSegmentRecBPlusTreeObjectLength uint64 - BPlusTreeObjectBPlusTreeObjectNumber uint64 - BPlusTreeObjectBPlusTreeObjectOffset uint64 - BPlusTreeObjectBPlusTreeObjectLength uint64 - CreatedObjectsBPlusTreeObjectNumber uint64 - CreatedObjectsBPlusTreeObjectOffset uint64 - CreatedObjectsBPlusTreeObjectLength uint64 - DeletedObjectsBPlusTreeObjectNumber uint64 - DeletedObjectsBPlusTreeObjectOffset uint64 - DeletedObjectsBPlusTreeObjectLength uint64 -} - -type checkpointRequestStruct struct { - waitGroup sync.WaitGroup - err error - exitOnCompletion bool -} - -const ( - replayLogWriteBufferAlignment uintptr = 4096 - replayLogWriteBufferDefaultSize uint64 = 100 * uint64(replayLogWriteBufferAlignment) -) - -const ( - transactionPutInodeRec uint64 = iota - transactionPutInodeRecs - transactionDeleteInodeRec - transactionPutLogSegmentRec - transactionDeleteLogSegmentRec - transactionPutBPlusTreeObject - transactionDeleteBPlusTreeObject -) - -type replayLogTransactionFixedPartStruct struct { // transactions begin on a replayLogWriteBufferAlignment boundary - CRC64 uint64 // checksum of everything after this field - BytesFollowing uint64 // bytes following in this transaction - LastCheckpointObjectTrailerStructObjectNumber uint64 // last CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - TransactionType uint64 // transactionType from above const() block -} - -type delayedObjectDeleteStruct struct { - containerName string - objectNumber uint64 -} - -func constructReplayLogWriteBuffer(minBufferSize uint64) (alignedBuf []byte) { - var ( - alignedBufAddr uintptr - alignedBufOffset uintptr - alignedBufSize uintptr - allocSize uintptr - unalignedBuf []byte - unalignedBufAddr uintptr - ) - - alignedBufSize = (uintptr(minBufferSize) + replayLogWriteBufferAlignment - 1) & ^(replayLogWriteBufferAlignment - 1) - allocSize = alignedBufSize + replayLogWriteBufferAlignment - 1 - unalignedBuf = make([]byte, allocSize) - unalignedBufAddr = uintptr(unsafe.Pointer(&unalignedBuf[0])) - alignedBufAddr = (unalignedBufAddr + replayLogWriteBufferAlignment - 1) & ^(replayLogWriteBufferAlignment - 1) - alignedBufOffset = uintptr(alignedBufAddr) - unalignedBufAddr - alignedBuf = unalignedBuf[alignedBufOffset : alignedBufOffset+uintptr(alignedBufSize)] - - return -} - -func (volume *volumeStruct) minimizeReplayLogWriteBuffer(bytesNeeded uint64) (minimizedBuf []byte) { - var ( - truncatedDefaultReplayLogWriteBufferSize uintptr - ) - - truncatedDefaultReplayLogWriteBufferSize = (uintptr(bytesNeeded) + replayLogWriteBufferAlignment - 1) & ^(replayLogWriteBufferAlignment - 1) - - minimizedBuf = volume.defaultReplayLogWriteBuffer[:truncatedDefaultReplayLogWriteBufferSize] - - return -} - -func (volume *volumeStruct) recordTransaction(transactionType uint64, keys interface{}, values interface{}) { - var ( - bytesNeeded uint64 - err error - i int - multipleKeys []uint64 - multipleValues [][]byte - packedUint64 []byte - replayLogWriteBuffer []byte - replayLogWriteBufferPosition uint64 - singleKey uint64 - singleValue []byte - ) - - // TODO: Eventually embed this stuff in the case statement below - switch transactionType { - case transactionPutInodeRec: - evtlog.Record(evtlog.FormatHeadhunterRecordTransactionPutInodeRec, volume.volumeName, keys.(uint64)) - case transactionPutInodeRecs: - evtlog.Record(evtlog.FormatHeadhunterRecordTransactionPutInodeRecs, volume.volumeName, keys.([]uint64)) - case transactionDeleteInodeRec: - evtlog.Record(evtlog.FormatHeadhunterRecordTransactionDeleteInodeRec, volume.volumeName, keys.(uint64)) - case transactionPutLogSegmentRec: - evtlog.Record(evtlog.FormatHeadhunterRecordTransactionPutLogSegmentRec, volume.volumeName, keys.(uint64), string(values.([]byte)[:])) - case transactionDeleteLogSegmentRec: - evtlog.Record(evtlog.FormatHeadhunterRecordTransactionDeleteLogSegmentRec, volume.volumeName, keys.(uint64)) - case transactionPutBPlusTreeObject: - evtlog.Record(evtlog.FormatHeadhunterRecordTransactionPutBPlusTreeObject, volume.volumeName, keys.(uint64)) - case transactionDeleteBPlusTreeObject: - evtlog.Record(evtlog.FormatHeadhunterRecordTransactionDeleteBPlusTreeObject, volume.volumeName, keys.(uint64)) - default: - logger.Fatalf("headhunter.recordTransaction(transactionType==%v,,) invalid", transactionType) - } - - // TODO: Eventually just remove this (once replayLogFile is mandatory) - if "" == volume.replayLogFileName { - // Replay Log is disabled... simply return - return - } - - switch transactionType { - case transactionPutInodeRec: - singleKey = keys.(uint64) - singleValue = values.([]byte) - bytesNeeded = // transactions begin on a replayLogWriteBufferAlignment boundary - globals.uint64Size + // checksum of everything after this field - globals.uint64Size + // bytes following in this transaction - globals.uint64Size + // last CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - globals.uint64Size + // transactionType == transactionPutInodeRec - globals.uint64Size + // inodeNumber - globals.uint64Size + // len(value) - uint64(len(singleValue)) // value - case transactionPutInodeRecs: - multipleKeys = keys.([]uint64) - multipleValues = values.([][]byte) - if len(multipleKeys) != len(multipleValues) { - logger.Fatalf("headhunter.recordTransaction(transactionType==transactionPutInodeRecs,,) passed len(keys) != len(values)") - } - bytesNeeded = // transactions begin on a replayLogWriteBufferAlignment boundary - globals.uint64Size + // checksum of everything after this field - globals.uint64Size + // bytes following in this transaction - globals.uint64Size + // last CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - globals.uint64Size + // transactionType == transactionPutInodeRecs - globals.uint64Size // len(inodeNumbers) == len(values) - for i = 0; i < len(multipleKeys); i++ { - bytesNeeded += - globals.uint64Size + // inodeNumbers[i] - globals.uint64Size + // len(values[i]) - uint64(len(multipleValues[i])) // values[i] - } - case transactionDeleteInodeRec: - singleKey = keys.(uint64) - if nil != values { - logger.Fatalf("headhunter.recordTransaction(transactionType==transactionDeleteInodeRec,,) passed non-nil values") - } - bytesNeeded = // transactions begin on a replayLogWriteBufferAlignment boundary - globals.uint64Size + // checksum of everything after this field - globals.uint64Size + // bytes following in this transaction - globals.uint64Size + // last CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - globals.uint64Size + // transactionType == transactionDeleteInodeRec - globals.uint64Size // inodeNumber - case transactionPutLogSegmentRec: - singleKey = keys.(uint64) - singleValue = values.([]byte) - bytesNeeded = // transactions begin on a replayLogWriteBufferAlignment boundary - globals.uint64Size + // checksum of everything after this field - globals.uint64Size + // bytes following in this transaction - globals.uint64Size + // last CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - globals.uint64Size + // transactionType == transactionPutLogSegmentRec - globals.uint64Size + // logSegmentNumber - globals.uint64Size + // len(value) - uint64(len(singleValue)) // value - case transactionDeleteLogSegmentRec: - singleKey = keys.(uint64) - if nil != values { - logger.Fatalf("headhunter.recordTransaction(transactionType==transactionDeleteLogSegmentRec,,) passed non-nil values") - } - bytesNeeded = // transactions begin on a replayLogWriteBufferAlignment boundary - globals.uint64Size + // checksum of everything after this field - globals.uint64Size + // bytes following in this transaction - globals.uint64Size + // last CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - globals.uint64Size + // transactionType == transactionDeleteLogSegmentRec - globals.uint64Size // logSegmentNumber - case transactionPutBPlusTreeObject: - singleKey = keys.(uint64) - singleValue = values.([]byte) - bytesNeeded = // transactions begin on a replayLogWriteBufferAlignment boundary - globals.uint64Size + // checksum of everything after this field - globals.uint64Size + // bytes following in this transaction - globals.uint64Size + // last CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - globals.uint64Size + // transactionType == transactionPutBPlusTreeObject - globals.uint64Size + // objectNumber - globals.uint64Size + // len(value) - uint64(len(singleValue)) // value - case transactionDeleteBPlusTreeObject: - singleKey = keys.(uint64) - if nil != values { - logger.Fatalf("headhunter.recordTransaction(transactionType==transactionDeleteBPlusTreeObject,,) passed non-nil values") - } - bytesNeeded = // transactions begin on a replayLogWriteBufferAlignment boundary - globals.uint64Size + // checksum of everything after this field - globals.uint64Size + // bytes following in this transaction - globals.uint64Size + // last CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - globals.uint64Size + // transactionType == transactionDeleteBPlusTreeObject - globals.uint64Size // objectNumber - default: - logger.Fatalf("headhunter.recordTransaction(transactionType==%v,,) invalid", transactionType) - } - - if bytesNeeded <= replayLogWriteBufferDefaultSize { - replayLogWriteBuffer = volume.minimizeReplayLogWriteBuffer(bytesNeeded) - } else { - replayLogWriteBuffer = constructReplayLogWriteBuffer(bytesNeeded) - } - - // For now, leave room for ECMA CRC-64 - - replayLogWriteBufferPosition = globals.uint64Size - - // Fill in bytes following in this transaction - - packedUint64, err = cstruct.Pack(bytesNeeded-globals.uint64Size-globals.uint64Size, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - // Fill in last checkpoint's CheckpointHeaderStruct.checkpointObjectTrailerStructObjectNumber - - packedUint64, err = cstruct.Pack(volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - // Fill in transactionType - - packedUint64, err = cstruct.Pack(transactionType, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - // Fill in remaining transactionType-specific bytes - - switch transactionType { - case transactionPutInodeRec: - // Fill in inodeNumber - - packedUint64, err = cstruct.Pack(singleKey, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - // Fill in len(value) and value - - packedUint64, err = cstruct.Pack(uint64(len(singleValue)), LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], singleValue) - replayLogWriteBufferPosition += uint64(len(singleValue)) - case transactionPutInodeRecs: - // Fill in number of following inodeNumber:value pairs - - packedUint64, err = cstruct.Pack(uint64(len(multipleKeys)), LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - // Fill in each inodeNumber:value pair - - for i = 0; i < len(multipleKeys); i++ { - // Fill in inodeNumber - - packedUint64, err = cstruct.Pack(multipleKeys[i], LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - // Fill in len(value) and value - - packedUint64, err = cstruct.Pack(uint64(len(multipleValues[i])), LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], multipleValues[i]) - replayLogWriteBufferPosition += uint64(len(multipleValues[i])) - } - case transactionDeleteInodeRec: - // Fill in inodeNumber - - packedUint64, err = cstruct.Pack(singleKey, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - case transactionPutLogSegmentRec: - // Fill in logSegmentNumber - - packedUint64, err = cstruct.Pack(singleKey, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - // Fill in len(value) and value - - packedUint64, err = cstruct.Pack(uint64(len(singleValue)), LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], singleValue) - replayLogWriteBufferPosition += uint64(len(singleValue)) - case transactionDeleteLogSegmentRec: - // Fill in logSegmentNumber - - packedUint64, err = cstruct.Pack(singleKey, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - case transactionPutBPlusTreeObject: - // Fill in objectNumber - - packedUint64, err = cstruct.Pack(singleKey, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - // Fill in len(value) and value - - packedUint64, err = cstruct.Pack(uint64(len(singleValue)), LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], singleValue) - replayLogWriteBufferPosition += uint64(len(singleValue)) - case transactionDeleteBPlusTreeObject: - // Fill in objectNumber - - packedUint64, err = cstruct.Pack(singleKey, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer[replayLogWriteBufferPosition:], packedUint64) - replayLogWriteBufferPosition += globals.uint64Size - default: - logger.Fatalf("headhunter.recordTransaction(transactionType==%v,,) invalid", transactionType) - } - - // Compute and fill in ECMA CRC-64 - - packedUint64, err = cstruct.Pack(crc64.Checksum(replayLogWriteBuffer[globals.uint64Size:bytesNeeded], globals.crc64ECMATable), LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack() unexpectedly returned error: %v", err) - } - _ = copy(replayLogWriteBuffer, packedUint64) - - // Finally, write out replayLogWriteBuffer - - if nil == volume.replayLogFile { - // Replay Log not currently open - // - // Either upVolume()'s call to getCheckpoint() found that a clean downVolume() was possible - // or a successful putCheckpoint() has removed the Replay Log. In either case, a fresh - // Replay Log will now be created. - - volume.replayLogFile, err = platform.OpenFileSync(volume.replayLogFileName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) - if nil != err { - logger.FatalfWithError(err, "platform.OpenFileSync(%v,os.O_CREATE|os.O_EXCL|os.O_WRONLY,) failed", volume.replayLogFileName) - } - } else { - // Replay Log is currently open - // - // If this is the first call to recordTransaction() since upVolume() called getCheckpoint(), - // volume.replayLogFile will be positioned for writing just after the last transaction replayed - // following the loading of the checkpoint. If this is not the first call to recordTransaction() - // since the last putCheckpoint(), volume.replayLogFile will be positioned for writing just - // after the prior transaction. - } - - _, err = volume.replayLogFile.Write(replayLogWriteBuffer) - if nil != err { - logger.Fatalf("os.Write() unexpectedly returned error: %v", err) - } - - return -} - -// getEtcdCheckpointHeader gets the JSON-encoded checkpointHeader from etcd returning also its revision. -// -func (volume *volumeStruct) getEtcdCheckpointHeader() (checkpointHeader *CheckpointHeaderStruct, checkpointHeaderEtcdRevision int64, err error) { - var ( - cancel context.CancelFunc - ctx context.Context - getResponse *etcd.GetResponse - ) - - logger.Infof("getEtcdCheckpointHeader: called for volume '%s' etcdkey '%s'", - volume.volumeName, volume.checkpointEtcdKeyName) - - ctx, cancel = context.WithTimeout(context.Background(), globals.etcdOpTimeout) - getResponse, err = globals.etcdKV.Get(ctx, volume.checkpointEtcdKeyName) - cancel() - if nil != err { - err = fmt.Errorf("Error contacting etcd: %v", err) - return - } - - if 1 != getResponse.Count { - err = fmt.Errorf("Could not find %s in etcd", volume.checkpointEtcdKeyName) - return - } - - checkpointHeader = &CheckpointHeaderStruct{} - - err = json.Unmarshal(getResponse.Kvs[0].Value, checkpointHeader) - if nil != err { - err = fmt.Errorf("Error unmarshalling %s's Value (%s): %v", volume.checkpointEtcdKeyName, string(getResponse.Kvs[0].Value[:]), err) - return - } - - checkpointHeaderEtcdRevision = getResponse.Kvs[0].ModRevision - - logger.Infof("getEtcdCheckpointHeader: returning for volume '%s' etcdkey '%s' header %+v", - volume.volumeName, volume.checkpointEtcdKeyName, *checkpointHeader) - return -} - -// putEtcdCheckpointHeader puts the JSON-encoding of the supplied checkpointHeader in etcd. -// If a non-zero oldCheckpointHeaderEtcdRevision, it must match the current revision of the checkpointHeader. -// -func (volume *volumeStruct) putEtcdCheckpointHeader(checkpointHeader *CheckpointHeaderStruct, oldCheckpointHeaderEtcdRevision int64) (newCheckpointHeaderEtcdRevision int64, err error) { - var ( - cancel context.CancelFunc - checkpointHeaderBuf []byte - ctx context.Context - putResponse *etcd.PutResponse - txnResponse *etcd.TxnResponse - ) - - checkpointHeaderBuf, err = json.MarshalIndent(checkpointHeader, "", " ") - if nil != err { - err = fmt.Errorf("Error marshalling checkpointHeader (%#v): %v", checkpointHeader, err) - return - } - - if 0 == oldCheckpointHeaderEtcdRevision { - ctx, cancel = context.WithTimeout(context.Background(), globals.etcdOpTimeout) - putResponse, err = globals.etcdKV.Put(ctx, volume.checkpointEtcdKeyName, string(checkpointHeaderBuf[:])) - cancel() - if nil != err { - err = fmt.Errorf("Error contacting etcd: %v", err) - return - } - - newCheckpointHeaderEtcdRevision = putResponse.Header.Revision - } else { - ctx, cancel = context.WithTimeout(context.Background(), globals.etcdOpTimeout) - txnResponse, err = globals.etcdKV.Txn(ctx).If(etcd.Compare(etcd.ModRevision(volume.checkpointEtcdKeyName), "=", oldCheckpointHeaderEtcdRevision)).Then(etcd.OpPut(volume.checkpointEtcdKeyName, string(checkpointHeaderBuf[:]))).Commit() - cancel() - if nil != err { - err = fmt.Errorf("Error contacting etcd: %v", err) - return - } - - if !txnResponse.Succeeded { - err = fmt.Errorf("Transaction to update %s failed", volume.checkpointEtcdKeyName) - return - } - - newCheckpointHeaderEtcdRevision = txnResponse.Responses[0].GetResponsePut().Header.Revision - } - - return -} - -// fetchCheckpointContainerHeader reads the Checkpoint Container's Checkpoint Header a total of -// FSGlobals.CheckpointHeaderConsensusAttempts times simultaneously. Once all attempts are complete, -// it loops through the results in an attempt to achieve consensus such that a majority of the -// results agree on the most recently fetched Checkpoint Header. -// -func (volume *volumeStruct) fetchCheckpointContainerHeader() (checkpointHeader *CheckpointHeaderStruct, agreements uint16, err error) { - var ( - bestCheckpointHeader *CheckpointHeaderStruct - checkpointHeaders []*CheckpointHeaderStruct - checkpointHeaderConsensusAttempt uint16 - checkpointHeaderConsensusQuorum uint16 - thisCheckpointHeader *CheckpointHeaderStruct - wg sync.WaitGroup - ) - - checkpointHeaders = make([]*CheckpointHeaderStruct, globals.checkpointHeaderConsensusAttempts) - - for checkpointHeaderConsensusAttempt = 0; checkpointHeaderConsensusAttempt < globals.checkpointHeaderConsensusAttempts; checkpointHeaderConsensusAttempt++ { - wg.Add(1) - go volume.fetchCheckpointContainerHeaderAsynchronously(&checkpointHeaders[checkpointHeaderConsensusAttempt], &wg) - } - - wg.Wait() - - checkpointHeaderConsensusQuorum = (globals.checkpointHeaderConsensusAttempts + 1) / 2 - - bestCheckpointHeader = nil - agreements = 0 - - for checkpointHeaderConsensusAttempt = 0; checkpointHeaderConsensusAttempt < globals.checkpointHeaderConsensusAttempts; checkpointHeaderConsensusAttempt++ { - thisCheckpointHeader = checkpointHeaders[checkpointHeaderConsensusAttempt] - if nil == thisCheckpointHeader { - continue - } - - if nil == bestCheckpointHeader { - bestCheckpointHeader = thisCheckpointHeader - agreements = 1 - } else { - if (thisCheckpointHeader.CheckpointObjectTrailerStructObjectNumber == bestCheckpointHeader.CheckpointObjectTrailerStructObjectNumber) && - (thisCheckpointHeader.ReservedToNonce == bestCheckpointHeader.ReservedToNonce) { - agreements++ - } else { - if (thisCheckpointHeader.CheckpointObjectTrailerStructObjectNumber > bestCheckpointHeader.CheckpointObjectTrailerStructObjectNumber) || - (thisCheckpointHeader.ReservedToNonce > bestCheckpointHeader.ReservedToNonce) { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s found to be newer than prior best", volume.volumeName, CheckpointHeaderName) - bestCheckpointHeader = thisCheckpointHeader - agreements = 1 - } else { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s found to be older than prior best", volume.volumeName, CheckpointHeaderName) - } - } - } - } - - if agreements >= checkpointHeaderConsensusQuorum { - logger.Infof("%s.fetchCheckpointContainerHeader() found consensus CheckpointHeader %016X %016X %016X %016X (%d out of %d)", - volume.volumeName, - bestCheckpointHeader.CheckpointVersion, - bestCheckpointHeader.CheckpointObjectTrailerStructObjectNumber, - bestCheckpointHeader.CheckpointObjectTrailerStructObjectLength, - bestCheckpointHeader.ReservedToNonce, - agreements, - globals.checkpointHeaderConsensusAttempts) - checkpointHeader = bestCheckpointHeader - err = nil - } else { - err = fmt.Errorf("%s.fetchCheckpointContainerHeader() could not achieve consensus (%d out of %d)", - volume.volumeName, - agreements, - globals.checkpointHeaderConsensusAttempts) - logger.Errorf("%v", err) - } - - return -} - -// fetchCheckpointContainerHeaderAsynchronously is invoked as a goroutine (indicating completion by -// making a wg.Done() call) to fetch a CheckpointHeaderStruct for the Volume's Swift Account Checkpoint -// Container Header. If the Checkpoint Container is not accessible, is missing a Checkpoint Header, or -// contains an ill-formed Checkpoint Header, nil is written to where checkpointHeaderPtr indicates. -// If a well-formed Checkpoint Header is found, a CheckpointHeaderStruct is created and populated with -// the decoded values and a pointer to it is written to where checkpointHeaderPtr indicates. -// -func (volume *volumeStruct) fetchCheckpointContainerHeaderAsynchronously(checkpointHeaderPtr **CheckpointHeaderStruct, wg *sync.WaitGroup) { - var ( - checkpointContainerHeaders map[string][]string - checkpointHeaderValue string - checkpointHeaderValueSplit []string - checkpointHeaderValues []string - err error - ok bool - thisCheckpointHeader *CheckpointHeaderStruct - ) - - checkpointContainerHeaders, err = swiftclient.ContainerHead(volume.accountName, volume.checkpointContainerName) - if nil != err { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead failed: %v", volume.volumeName, err) - *checkpointHeaderPtr = nil - wg.Done() - return - } - - checkpointHeaderValues, ok = checkpointContainerHeaders[CheckpointHeaderName] - if !ok { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead missing Header %s", volume.volumeName, CheckpointHeaderName) - *checkpointHeaderPtr = nil - wg.Done() - return - } - if 1 != len(checkpointHeaderValues) { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s should have only a single value (had %d)", volume.volumeName, CheckpointHeaderName, len(checkpointHeaderValues)) - *checkpointHeaderPtr = nil - wg.Done() - return - } - - checkpointHeaderValue = checkpointHeaderValues[0] - checkpointHeaderValueSplit = strings.Split(checkpointHeaderValue, " ") - if 4 != len(checkpointHeaderValueSplit) { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s should have four fields (had %d)", volume.volumeName, CheckpointHeaderName, len(checkpointHeaderValueSplit)) - *checkpointHeaderPtr = nil - wg.Done() - return - } - - thisCheckpointHeader = &CheckpointHeaderStruct{} - - thisCheckpointHeader.CheckpointVersion, err = strconv.ParseUint(checkpointHeaderValueSplit[0], 16, 64) - if nil != err { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s contained unparseable CheckpointVersion (%s)", volume.volumeName, CheckpointHeaderName, checkpointHeaderValueSplit[0]) - *checkpointHeaderPtr = nil - wg.Done() - return - } - if CheckpointVersion3 != thisCheckpointHeader.CheckpointVersion { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s contained unsupported CheckpointVersion (%s)", volume.volumeName, CheckpointHeaderName, checkpointHeaderValueSplit[0]) - *checkpointHeaderPtr = nil - wg.Done() - return - } - - thisCheckpointHeader.CheckpointObjectTrailerStructObjectNumber, err = strconv.ParseUint(checkpointHeaderValueSplit[1], 16, 64) - if nil != err { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s contained unparseable CheckpointObjectTrailerStructObjectNumber (%s)", volume.volumeName, CheckpointHeaderName, checkpointHeaderValueSplit[1]) - *checkpointHeaderPtr = nil - wg.Done() - return - } - thisCheckpointHeader.CheckpointObjectTrailerStructObjectLength, err = strconv.ParseUint(checkpointHeaderValueSplit[2], 16, 64) - if nil != err { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s contained unparseable CheckpointObjectTrailerStructObjectLength (%s)", volume.volumeName, CheckpointHeaderName, checkpointHeaderValueSplit[2]) - *checkpointHeaderPtr = nil - wg.Done() - return - } - thisCheckpointHeader.ReservedToNonce, err = strconv.ParseUint(checkpointHeaderValueSplit[3], 16, 64) - if nil != err { - logger.Warnf("%s.fetchCheckpointContainerHeader() ContainerHead Header %s contained unparseable ReservedToNonce (%s)", volume.volumeName, CheckpointHeaderName, checkpointHeaderValueSplit[3]) - *checkpointHeaderPtr = nil - wg.Done() - return - } - - *checkpointHeaderPtr = thisCheckpointHeader - wg.Done() -} - -func (volume *volumeStruct) fetchCheckpointLayoutReport() (layoutReport sortedmap.LayoutReport, err error) { - var ( - checkpointHeader *CheckpointHeaderStruct - ) - - if globals.etcdEnabled { - checkpointHeader, _, err = volume.getEtcdCheckpointHeader() - if nil == err { - // Return layoutReport from checkpointHeader found in etcd - - layoutReport = make(sortedmap.LayoutReport) - layoutReport[checkpointHeader.CheckpointObjectTrailerStructObjectNumber] = checkpointHeader.CheckpointObjectTrailerStructObjectLength - - return - } - - // Fall-through to use the checkpointHeader from Swift - } - - checkpointHeader, _, err = volume.fetchCheckpointContainerHeader() - if nil != err { - return - } - - // Return layoutReport manufactured from the checkpointHeaderValue found in the Checkpoint Container - - layoutReport = make(sortedmap.LayoutReport) - layoutReport[checkpointHeader.CheckpointObjectTrailerStructObjectNumber] = checkpointHeader.CheckpointObjectTrailerStructObjectLength - - return -} - -func (volume *volumeStruct) getCheckpoint(autoFormat bool) (err error) { - var ( - accountHeaderValues []string - accountHeaders map[string][]string - bPlusTreeObjectWrapperBPlusTreeTracker *bPlusTreeTrackerStruct - bytesConsumed uint64 - bytesNeeded uint64 - checkpointContainerHeadAgreements uint16 - checkpointContainerHeaders map[string][]string - checkpointHeader CheckpointHeaderStruct - checkpointHeaderValue string - checkpointHeaderValues []string - checkpointObjectTrailerBuf []byte - checkpointObjectTrailerV3 *CheckpointObjectTrailerV3Struct - computedCRC64 uint64 - containerNameAsValue sortedmap.Value - createdObjectsWrapperBPlusTreeTracker *bPlusTreeTrackerStruct - defaultReplayLogReadBuffer []byte - deletedObjectsWrapperBPlusTreeTracker *bPlusTreeTrackerStruct - elementOfBPlusTreeLayout ElementOfBPlusTreeLayoutStruct - expectedCheckpointObjectTrailerSize uint64 - inodeIndex uint64 - inodeNumber uint64 - inodeRecWrapperBPlusTreeTracker *bPlusTreeTrackerStruct - layoutReportIndex uint64 - logSegmentNumber uint64 - logSegmentRecWrapperBPlusTreeTracker *bPlusTreeTrackerStruct - numInodes uint64 - objectNumber uint64 - ok bool - replayLogReadBuffer []byte - replayLogReadBufferPosition uint64 - replayLogPosition int64 - replayLogSize int64 - replayLogTransactionFixedPart replayLogTransactionFixedPartStruct - snapShotBPlusTreeObjectBPlusTreeObjectLengthStruct uint64Struct - snapShotBPlusTreeObjectBPlusTreeObjectNumberStruct uint64Struct - snapShotBPlusTreeObjectBPlusTreeObjectOffsetStruct uint64Struct - snapShotCreatedObjectsBPlusTreeObjectLengthStruct uint64Struct - snapShotCreatedObjectsBPlusTreeObjectNumberStruct uint64Struct - snapShotCreatedObjectsBPlusTreeObjectOffsetStruct uint64Struct - snapShotDeletedObjectsBPlusTreeObjectLengthStruct uint64Struct - snapShotDeletedObjectsBPlusTreeObjectNumberStruct uint64Struct - snapShotDeletedObjectsBPlusTreeObjectOffsetStruct uint64Struct - snapShotID uint64 - snapShotIDStruct uint64Struct - snapShotInodeRecBPlusTreeObjectLengthStruct uint64Struct - snapShotInodeRecBPlusTreeObjectNumberStruct uint64Struct - snapShotInodeRecBPlusTreeObjectOffsetStruct uint64Struct - snapShotIndex uint64 - snapShotLogSegmentRecBPlusTreeObjectLengthStruct uint64Struct - snapShotLogSegmentRecBPlusTreeObjectNumberStruct uint64Struct - snapShotLogSegmentRecBPlusTreeObjectOffsetStruct uint64Struct - snapShotNameBuf []byte - snapShotNameBufLenStruct uint64Struct - snapShotNonceStruct uint64Struct - snapShotTimeStampBuf []byte - snapShotTimeStampBufLenStruct uint64Struct - storagePolicyHeaderValues []string - value []byte - valueLen uint64 - volumeView *volumeViewStruct - volumeViewAsValue sortedmap.Value - ) - - if globals.etcdEnabled { - volume.checkpointHeader, volume.checkpointHeaderEtcdRevision, err = volume.getEtcdCheckpointHeader() - if nil != err { - logger.Infof("No checkpointHeader found in etcd for volume %s: %v", volume.volumeName, err) - - volume.checkpointHeader, checkpointContainerHeadAgreements, err = volume.fetchCheckpointContainerHeader() - if nil != err { - if 0 != checkpointContainerHeadAgreements { - err = fmt.Errorf("Found no consensus but non-empty checkpointHeader in Swift for volume %s", volume.volumeName) - return - } - if autoFormat { - logger.Infof("No checkpointHeader found in Swift for volume %s: %v", volume.volumeName, err) - } else { - err = fmt.Errorf("No checkpointHeader found in Swift for volume %s: %v", volume.volumeName, err) - return - } - - volume.checkpointHeader = &CheckpointHeaderStruct{ - CheckpointVersion: CheckpointVersion3, - CheckpointObjectTrailerStructObjectNumber: 0, - CheckpointObjectTrailerStructObjectLength: 0, - ReservedToNonce: firstNonceToProvide, // First FetchNonce() will trigger a reserve step - } - - volume.checkpointHeaderEtcdRevision, err = volume.putEtcdCheckpointHeader(volume.checkpointHeader, 0) - if nil != err { - err = fmt.Errorf("Unable to put checkpointHeader in etcd: %v", err) - return - } - - checkpointHeaderValue = fmt.Sprintf("%016X %016X %016X %016X", - checkpointHeader.CheckpointVersion, - checkpointHeader.CheckpointObjectTrailerStructObjectNumber, - checkpointHeader.CheckpointObjectTrailerStructObjectLength, - checkpointHeader.ReservedToNonce, - ) - - checkpointHeaderValues = []string{checkpointHeaderValue} - - storagePolicyHeaderValues = []string{volume.checkpointContainerStoragePolicy} - - checkpointContainerHeaders = make(map[string][]string) - - checkpointContainerHeaders[CheckpointHeaderName] = checkpointHeaderValues - checkpointContainerHeaders[StoragePolicyHeaderName] = storagePolicyHeaderValues - - err = swiftclient.ContainerPut(volume.accountName, volume.checkpointContainerName, checkpointContainerHeaders) - if nil != err { - return - } - - // Mark Account as bi-modal... - // Note: pfs_middleware will actually see this header named AccountHeaderNameTranslated - - accountHeaderValues = []string{AccountHeaderValue} - - accountHeaders = make(map[string][]string) - - accountHeaders[AccountHeaderName] = accountHeaderValues - - err = swiftclient.AccountPost(volume.accountName, accountHeaders) - if nil != err { - return - } - } - } - } else { - volume.checkpointHeader, checkpointContainerHeadAgreements, err = volume.fetchCheckpointContainerHeader() - if nil != err { - if 0 != checkpointContainerHeadAgreements { - err = fmt.Errorf("Found no consensus but non-empty checkpointHeader in Swift for volume %s", volume.volumeName) - return - } - if autoFormat { - logger.Infof("No checkpointHeader found in Swift for volume %s: %v", volume.volumeName, err) - } else { - err = fmt.Errorf("No checkpointHeader found in Swift for volume %s: %v", volume.volumeName, err) - return - } - - volume.checkpointHeader = &CheckpointHeaderStruct{ - CheckpointVersion: CheckpointVersion3, - CheckpointObjectTrailerStructObjectNumber: 0, - CheckpointObjectTrailerStructObjectLength: 0, - ReservedToNonce: firstNonceToProvide, // First FetchNonce() will trigger a reserve step - } - - checkpointHeaderValue = fmt.Sprintf("%016X %016X %016X %016X", - checkpointHeader.CheckpointVersion, - checkpointHeader.CheckpointObjectTrailerStructObjectNumber, - checkpointHeader.CheckpointObjectTrailerStructObjectLength, - checkpointHeader.ReservedToNonce, - ) - - checkpointHeaderValues = []string{checkpointHeaderValue} - - storagePolicyHeaderValues = []string{volume.checkpointContainerStoragePolicy} - - checkpointContainerHeaders = make(map[string][]string) - - checkpointContainerHeaders[CheckpointHeaderName] = checkpointHeaderValues - checkpointContainerHeaders[StoragePolicyHeaderName] = storagePolicyHeaderValues - - err = swiftclient.ContainerPut(volume.accountName, volume.checkpointContainerName, checkpointContainerHeaders) - if nil != err { - return - } - - // Mark Account as bi-modal... - // Note: pfs_middleware will actually see this header named AccountHeaderNameTranslated - - accountHeaderValues = []string{AccountHeaderValue} - - accountHeaders = make(map[string][]string) - - accountHeaders[AccountHeaderName] = accountHeaderValues - - err = swiftclient.AccountPost(volume.accountName, accountHeaders) - if nil != err { - return - } - } - } - - volume.liveView = &volumeViewStruct{volume: volume} - - if CheckpointVersion3 == volume.checkpointHeader.CheckpointVersion { - if 0 == volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber { - // Initialize based on zero-filled CheckpointObjectTrailerV3Struct - - checkpointObjectTrailerV3 = &CheckpointObjectTrailerV3Struct{ - InodeRecBPlusTreeObjectNumber: 0, - InodeRecBPlusTreeObjectOffset: 0, - InodeRecBPlusTreeObjectLength: 0, - InodeRecBPlusTreeLayoutNumElements: 0, - LogSegmentRecBPlusTreeObjectNumber: 0, - LogSegmentRecBPlusTreeObjectOffset: 0, - LogSegmentRecBPlusTreeObjectLength: 0, - LogSegmentRecBPlusTreeLayoutNumElements: 0, - BPlusTreeObjectBPlusTreeObjectNumber: 0, - BPlusTreeObjectBPlusTreeObjectOffset: 0, - BPlusTreeObjectBPlusTreeObjectLength: 0, - BPlusTreeObjectBPlusTreeLayoutNumElements: 0, - CreatedObjectsBPlusTreeLayoutNumElements: 0, - DeletedObjectsBPlusTreeLayoutNumElements: 0, - SnapShotIDNumBits: uint64(volume.snapShotIDNumBits), - SnapShotListNumElements: 0, - SnapShotListTotalSize: 0, - } - - inodeRecWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.inodeRecWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: inodeRecWrapperBPlusTreeTracker, - } - - volume.liveView.inodeRecWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxInodesPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.inodeRecWrapper, - globals.inodeRecCache) - - logSegmentRecWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.logSegmentRecWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: logSegmentRecWrapperBPlusTreeTracker, - } - - volume.liveView.logSegmentRecWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxLogSegmentsPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.logSegmentRecWrapper, - globals.logSegmentRecCache) - - bPlusTreeObjectWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.bPlusTreeObjectWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: bPlusTreeObjectWrapperBPlusTreeTracker, - } - - volume.liveView.bPlusTreeObjectWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxDirFileNodesPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.bPlusTreeObjectWrapper, - globals.bPlusTreeObjectCache) - - createdObjectsWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.createdObjectsWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: createdObjectsWrapperBPlusTreeTracker, - } - - volume.liveView.createdObjectsWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxCreatedDeletedObjectsPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.createdObjectsWrapper, - globals.createdDeletedObjectsCache) - - deletedObjectsWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.deletedObjectsWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: deletedObjectsWrapperBPlusTreeTracker, - } - - volume.liveView.deletedObjectsWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxCreatedDeletedObjectsPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.deletedObjectsWrapper, - globals.createdDeletedObjectsCache) - - // Compute SnapShotID shortcuts - - volume.snapShotIDShift = uint64(64) - uint64(volume.snapShotIDNumBits) - volume.dotSnapShotDirSnapShotID = (uint64(1) << uint64(volume.snapShotIDNumBits)) - uint64(1) - volume.snapShotU64NonceMask = (uint64(1) << volume.snapShotIDShift) - uint64(1) - - // Initialize viewTreeBy{Nonce|ID|Time|Name} - - volume.viewTreeByNonce = sortedmap.NewLLRBTree(sortedmap.CompareUint64, nil) - volume.viewTreeByID = sortedmap.NewLLRBTree(sortedmap.CompareUint64, nil) - volume.viewTreeByTime = sortedmap.NewLLRBTree(sortedmap.CompareTime, nil) - volume.viewTreeByName = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - volume.priorView = nil - - // Initialize list of available SnapShotIDs - - volume.availableSnapShotIDList = list.New() - - for snapShotID = uint64(1); snapShotID < volume.dotSnapShotDirSnapShotID; snapShotID++ { - volume.availableSnapShotIDList.PushBack(snapShotID) - } - } else { - // Read in CheckpointObjectTrailerV3Struct - - checkpointObjectTrailerBuf, err = - swiftclient.ObjectTail( - volume.accountName, - volume.checkpointContainerName, - utils.Uint64ToHexStr(volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber), - volume.checkpointHeader.CheckpointObjectTrailerStructObjectLength) - if nil != err { - return - } - - checkpointObjectTrailerV3 = &CheckpointObjectTrailerV3Struct{} - - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, checkpointObjectTrailerV3, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // Load liveView.{inodeRec|logSegmentRec|bPlusTreeObject}Wrapper B+Trees - - inodeRecWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.inodeRecWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: inodeRecWrapperBPlusTreeTracker, - } - - if 0 == checkpointObjectTrailerV3.InodeRecBPlusTreeObjectNumber { - volume.liveView.inodeRecWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxInodesPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.inodeRecWrapper, - globals.inodeRecCache) - } else { - volume.liveView.inodeRecWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - checkpointObjectTrailerV3.InodeRecBPlusTreeObjectNumber, - checkpointObjectTrailerV3.InodeRecBPlusTreeObjectOffset, - checkpointObjectTrailerV3.InodeRecBPlusTreeObjectLength, - sortedmap.CompareUint64, - volume.liveView.inodeRecWrapper, - globals.inodeRecCache) - if nil != err { - return - } - } - - logSegmentRecWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.logSegmentRecWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: logSegmentRecWrapperBPlusTreeTracker, - } - - if 0 == checkpointObjectTrailerV3.LogSegmentRecBPlusTreeObjectNumber { - volume.liveView.logSegmentRecWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxLogSegmentsPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.logSegmentRecWrapper, - globals.logSegmentRecCache) - } else { - volume.liveView.logSegmentRecWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - checkpointObjectTrailerV3.LogSegmentRecBPlusTreeObjectNumber, - checkpointObjectTrailerV3.LogSegmentRecBPlusTreeObjectOffset, - checkpointObjectTrailerV3.LogSegmentRecBPlusTreeObjectLength, - sortedmap.CompareUint64, - volume.liveView.logSegmentRecWrapper, - globals.logSegmentRecCache) - if nil != err { - return - } - } - - bPlusTreeObjectWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.bPlusTreeObjectWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: bPlusTreeObjectWrapperBPlusTreeTracker, - } - - if 0 == checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeObjectNumber { - volume.liveView.bPlusTreeObjectWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxDirFileNodesPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.bPlusTreeObjectWrapper, - globals.bPlusTreeObjectCache) - } else { - volume.liveView.bPlusTreeObjectWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeObjectNumber, - checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeObjectOffset, - checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeObjectLength, - sortedmap.CompareUint64, - volume.liveView.bPlusTreeObjectWrapper, - globals.bPlusTreeObjectCache) - if nil != err { - return - } - } - - // Initialize liveView.{createdObjects|deletedObjects}Wrapper B+Trees - - createdObjectsWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.createdObjectsWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: createdObjectsWrapperBPlusTreeTracker, - } - - volume.liveView.createdObjectsWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxCreatedDeletedObjectsPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.createdObjectsWrapper, - globals.createdDeletedObjectsCache) - - deletedObjectsWrapperBPlusTreeTracker = &bPlusTreeTrackerStruct{bPlusTreeLayout: make(sortedmap.LayoutReport)} - - volume.liveView.deletedObjectsWrapper = &bPlusTreeWrapperStruct{ - volumeView: volume.liveView, - bPlusTreeTracker: deletedObjectsWrapperBPlusTreeTracker, - } - - volume.liveView.deletedObjectsWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxCreatedDeletedObjectsPerMetadataNode, - sortedmap.CompareUint64, - volume.liveView.deletedObjectsWrapper, - globals.createdDeletedObjectsCache) - - // Validate size of checkpointObjectTrailerBuf - - expectedCheckpointObjectTrailerSize = checkpointObjectTrailerV3.InodeRecBPlusTreeLayoutNumElements - expectedCheckpointObjectTrailerSize += checkpointObjectTrailerV3.LogSegmentRecBPlusTreeLayoutNumElements - expectedCheckpointObjectTrailerSize += checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeLayoutNumElements - expectedCheckpointObjectTrailerSize += checkpointObjectTrailerV3.CreatedObjectsBPlusTreeLayoutNumElements - expectedCheckpointObjectTrailerSize += checkpointObjectTrailerV3.DeletedObjectsBPlusTreeLayoutNumElements - expectedCheckpointObjectTrailerSize *= globals.ElementOfBPlusTreeLayoutStructSize - expectedCheckpointObjectTrailerSize += checkpointObjectTrailerV3.SnapShotListTotalSize - - if uint64(len(checkpointObjectTrailerBuf)) != expectedCheckpointObjectTrailerSize { - err = fmt.Errorf("checkpointObjectTrailer for volume %v does not match required size", volume.volumeName) - return - } - - // Deserialize liveView.{inodeRec|logSegmentRec|bPlusTreeObject}Wrapper LayoutReports - - for layoutReportIndex = 0; layoutReportIndex < checkpointObjectTrailerV3.InodeRecBPlusTreeLayoutNumElements; layoutReportIndex++ { - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volume.liveView.inodeRecWrapper.bPlusTreeTracker.bPlusTreeLayout[elementOfBPlusTreeLayout.ObjectNumber] = elementOfBPlusTreeLayout.ObjectBytes - } - - for layoutReportIndex = 0; layoutReportIndex < checkpointObjectTrailerV3.LogSegmentRecBPlusTreeLayoutNumElements; layoutReportIndex++ { - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volume.liveView.logSegmentRecWrapper.bPlusTreeTracker.bPlusTreeLayout[elementOfBPlusTreeLayout.ObjectNumber] = elementOfBPlusTreeLayout.ObjectBytes - } - - for layoutReportIndex = 0; layoutReportIndex < checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeLayoutNumElements; layoutReportIndex++ { - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volume.liveView.bPlusTreeObjectWrapper.bPlusTreeTracker.bPlusTreeLayout[elementOfBPlusTreeLayout.ObjectNumber] = elementOfBPlusTreeLayout.ObjectBytes - } - - for layoutReportIndex = 0; layoutReportIndex < checkpointObjectTrailerV3.CreatedObjectsBPlusTreeLayoutNumElements; layoutReportIndex++ { - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volume.liveView.createdObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout[elementOfBPlusTreeLayout.ObjectNumber] = elementOfBPlusTreeLayout.ObjectBytes - } - - for layoutReportIndex = 0; layoutReportIndex < checkpointObjectTrailerV3.DeletedObjectsBPlusTreeLayoutNumElements; layoutReportIndex++ { - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volume.liveView.deletedObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout[elementOfBPlusTreeLayout.ObjectNumber] = elementOfBPlusTreeLayout.ObjectBytes - } - - // Compute SnapShotID shortcuts - - volume.snapShotIDShift = uint64(64) - uint64(volume.snapShotIDNumBits) - volume.dotSnapShotDirSnapShotID = (uint64(1) << uint64(volume.snapShotIDNumBits)) - uint64(1) - volume.snapShotU64NonceMask = (uint64(1) << volume.snapShotIDShift) - uint64(1) - - // Load SnapShotList - - volume.viewTreeByNonce = sortedmap.NewLLRBTree(sortedmap.CompareUint64, nil) - volume.viewTreeByID = sortedmap.NewLLRBTree(sortedmap.CompareUint64, nil) - volume.viewTreeByTime = sortedmap.NewLLRBTree(sortedmap.CompareTime, nil) - volume.viewTreeByName = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - // Load of viewTreeBy{Nonce|ID|Time|Name} - - for snapShotIndex = 0; snapShotIndex < checkpointObjectTrailerV3.SnapShotListNumElements; snapShotIndex++ { - volumeView = &volumeViewStruct{volume: volume} - - // ElementOfSnapShotListStruct.Nonce - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the nonce", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotNonceStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - volumeView.nonce = snapShotNonceStruct.U64 - _, ok, err = volume.viewTreeByNonce.GetByKey(volumeView.nonce) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByNonce.GetByKey(%v) for SnapShotList element %v failed: %v", volume.volumeName, volumeView.nonce, snapShotIndex, err) - } - if ok { - err = fmt.Errorf("Volume %v's viewTreeByNonce already contained nonce %v for SnapShotList element %v ", volume.volumeName, volumeView.nonce, snapShotIndex) - return - } - - // ElementOfSnapShotListStruct.ID - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the id", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotIDStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - volumeView.snapShotID = snapShotIDStruct.U64 - if volumeView.snapShotID >= volume.dotSnapShotDirSnapShotID { - err = fmt.Errorf("Invalid volumeView.snapShotID (%v) for configured volume.snapShotIDNumBits (%v)", volumeView.snapShotID, volume.snapShotIDNumBits) - return - } - _, ok, err = volume.viewTreeByID.GetByKey(volumeView.snapShotID) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByID.GetByKey(%v) for SnapShotList element %v failed: %v", volume.volumeName, volumeView.snapShotID, snapShotIndex, err) - } - if ok { - err = fmt.Errorf("Volume %v's viewTreeByID already contained snapShotID %v for SnapShotList element %v ", volume.volumeName, volumeView.snapShotID, snapShotIndex) - return - } - - // ElementOfSnapShotListStruct.TimeStamp - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the timeStamp len", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotTimeStampBufLenStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - if uint64(len(checkpointObjectTrailerBuf)) < snapShotTimeStampBufLenStruct.U64 { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the timeStamp", volume.volumeName, snapShotIndex) - return - } - snapShotTimeStampBuf = checkpointObjectTrailerBuf[:snapShotTimeStampBufLenStruct.U64] - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[snapShotTimeStampBufLenStruct.U64:] - err = volumeView.snapShotTime.UnmarshalBinary(snapShotTimeStampBuf) - if nil != err { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v's' timeStamp (err: %v)", volume.volumeName, snapShotIndex, err) - return - } - _, ok, err = volume.viewTreeByTime.GetByKey(volumeView.snapShotTime) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByTime.GetByKey(%v) for SnapShotList element %v failed: %v", volume.volumeName, volumeView.snapShotTime, snapShotIndex, err) - } - if ok { - err = fmt.Errorf("Volume %v's viewTreeByTime already contained snapShotTime %v for SnapShotList element %v ", volume.volumeName, volumeView.snapShotTime, snapShotIndex) - return - } - - // ElementOfSnapShotListStruct.Name - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the name len", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotNameBufLenStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - if uint64(len(checkpointObjectTrailerBuf)) < snapShotNameBufLenStruct.U64 { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the name", volume.volumeName, snapShotIndex) - return - } - snapShotNameBuf = checkpointObjectTrailerBuf[:snapShotNameBufLenStruct.U64] - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[snapShotNameBufLenStruct.U64:] - volumeView.snapShotName = utils.ByteSliceToString(snapShotNameBuf) - _, ok, err = volume.viewTreeByName.GetByKey(volumeView.snapShotName) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByName.GetByKey(%v) for SnapShotList element %v failed: %v", volume.volumeName, volumeView.snapShotName, snapShotIndex, err) - } - if ok { - err = fmt.Errorf("Volume %v's viewTreeByName already contained snapShotName %v for SnapShotList element %v ", volume.volumeName, volumeView.snapShotName, snapShotIndex) - return - } - - // ElementOfSnapShotListStruct.InodeRecBPlusTreeObjectNumber - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the inodeRecBPlusTreeObjectNumber", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotInodeRecBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.InodeRecBPlusTreeObjectOffset - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the inodeRecBPlusTreeObjectOffset", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotInodeRecBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.InodeRecBPlusTreeObjectLength - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the inodeRecBPlusTreeObjectLength", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotInodeRecBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volumeView.inodeRecWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: nil, - } - - if 0 == snapShotInodeRecBPlusTreeObjectNumberStruct.U64 { - volumeView.inodeRecWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxInodesPerMetadataNode, - sortedmap.CompareUint64, - volumeView.inodeRecWrapper, - globals.inodeRecCache) - } else { - volumeView.inodeRecWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - snapShotInodeRecBPlusTreeObjectNumberStruct.U64, - snapShotInodeRecBPlusTreeObjectOffsetStruct.U64, - snapShotInodeRecBPlusTreeObjectLengthStruct.U64, - sortedmap.CompareUint64, - volumeView.inodeRecWrapper, - globals.inodeRecCache) - if nil != err { - return - } - } - - // ElementOfSnapShotListStruct.LogSegmentRecBPlusTreeObjectNumber - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the logSegmentRecBPlusTreeObjectNumber", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotLogSegmentRecBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.LogSegmentRecBPlusTreeObjectOffset - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the logSegmentRecBPlusTreeObjectOffset", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotLogSegmentRecBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.LogSegmentRecBPlusTreeObjectLength - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the logSegmentRecBPlusTreeObjectLength", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotLogSegmentRecBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volumeView.logSegmentRecWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: nil, - } - - if 0 == snapShotLogSegmentRecBPlusTreeObjectNumberStruct.U64 { - volumeView.logSegmentRecWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxLogSegmentsPerMetadataNode, - sortedmap.CompareUint64, - volumeView.logSegmentRecWrapper, - globals.logSegmentRecCache) - } else { - volumeView.logSegmentRecWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - snapShotLogSegmentRecBPlusTreeObjectNumberStruct.U64, - snapShotLogSegmentRecBPlusTreeObjectOffsetStruct.U64, - snapShotLogSegmentRecBPlusTreeObjectLengthStruct.U64, - sortedmap.CompareUint64, - volumeView.logSegmentRecWrapper, - globals.logSegmentRecCache) - if nil != err { - return - } - } - - // ElementOfSnapShotListStruct.BPlusTreeObjectBPlusTreeObjectNumber - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the bPlusTreeObjectBPlusTreeObjectNumber", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotBPlusTreeObjectBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.BPlusTreeObjectBPlusTreeObjectOffset - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the bPlusTreeObjectBPlusTreeObjectOffset", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotBPlusTreeObjectBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.BPlusTreeObjectBPlusTreeObjectLength - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the bPlusTreeObjectBPlusTreeObjectLength", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotBPlusTreeObjectBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volumeView.bPlusTreeObjectWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: nil, - } - - if 0 == snapShotBPlusTreeObjectBPlusTreeObjectNumberStruct.U64 { - volumeView.bPlusTreeObjectWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxDirFileNodesPerMetadataNode, - sortedmap.CompareUint64, - volumeView.bPlusTreeObjectWrapper, - globals.logSegmentRecCache) - } else { - volumeView.bPlusTreeObjectWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - snapShotBPlusTreeObjectBPlusTreeObjectNumberStruct.U64, - snapShotBPlusTreeObjectBPlusTreeObjectOffsetStruct.U64, - snapShotBPlusTreeObjectBPlusTreeObjectLengthStruct.U64, - sortedmap.CompareUint64, - volumeView.bPlusTreeObjectWrapper, - globals.logSegmentRecCache) - if nil != err { - return - } - } - - // ElementOfSnapShotListStruct.CreatedObjectsBPlusTreeObjectNumber - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the createdObjectsBPlusTreeObjectNumber", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotCreatedObjectsBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.CreatedObjectsBPlusTreeObjectOffset - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the createdObjectsBPlusTreeObjectOffset", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotCreatedObjectsBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.CreatedObjectsBPlusTreeObjectLength - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the createdObjectsBPlusTreeObjectLength", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotCreatedObjectsBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volumeView.createdObjectsWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: volumeView.volume.liveView.createdObjectsWrapper.bPlusTreeTracker, - } - - if 0 == snapShotCreatedObjectsBPlusTreeObjectNumberStruct.U64 { - volumeView.createdObjectsWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxCreatedDeletedObjectsPerMetadataNode, - sortedmap.CompareUint64, - volumeView.createdObjectsWrapper, - globals.createdDeletedObjectsCache) - } else { - volumeView.createdObjectsWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - snapShotCreatedObjectsBPlusTreeObjectNumberStruct.U64, - snapShotCreatedObjectsBPlusTreeObjectOffsetStruct.U64, - snapShotCreatedObjectsBPlusTreeObjectLengthStruct.U64, - sortedmap.CompareUint64, - volumeView.createdObjectsWrapper, - globals.createdDeletedObjectsCache) - if nil != err { - return - } - } - - // ElementOfSnapShotListStruct.DeletedObjectsBPlusTreeObjectNumber - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the deletedObjectsBPlusTreeObjectNumber", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotDeletedObjectsBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.DeletedObjectsBPlusTreeObjectOffset - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the deletedObjectsBPlusTreeObjectOffset", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotDeletedObjectsBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - // ElementOfSnapShotListStruct.DeletedObjectsBPlusTreeObjectLength - - if uint64(len(checkpointObjectTrailerBuf)) < globals.uint64Size { - err = fmt.Errorf("Cannot parse volume %v's checkpointObjectTrailer's SnapShotList element %v...no room for the deletedObjectsBPlusTreeObjectLength", volume.volumeName, snapShotIndex) - return - } - bytesConsumed, err = cstruct.Unpack(checkpointObjectTrailerBuf, &snapShotDeletedObjectsBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - return - } - checkpointObjectTrailerBuf = checkpointObjectTrailerBuf[bytesConsumed:] - - volumeView.deletedObjectsWrapper = &bPlusTreeWrapperStruct{ - volumeView: volumeView, - bPlusTreeTracker: volumeView.volume.liveView.deletedObjectsWrapper.bPlusTreeTracker, - } - - if 0 == snapShotDeletedObjectsBPlusTreeObjectNumberStruct.U64 { - volumeView.deletedObjectsWrapper.bPlusTree = - sortedmap.NewBPlusTree( - volume.maxCreatedDeletedObjectsPerMetadataNode, - sortedmap.CompareUint64, - volumeView.deletedObjectsWrapper, - globals.createdDeletedObjectsCache) - } else { - volumeView.deletedObjectsWrapper.bPlusTree, err = - sortedmap.OldBPlusTree( - snapShotDeletedObjectsBPlusTreeObjectNumberStruct.U64, - snapShotDeletedObjectsBPlusTreeObjectOffsetStruct.U64, - snapShotDeletedObjectsBPlusTreeObjectLengthStruct.U64, - sortedmap.CompareUint64, - volumeView.deletedObjectsWrapper, - globals.createdDeletedObjectsCache) - if nil != err { - return - } - } - - // Insert volumeView into viewTreeBy{Nonce|ID|Time|Name} - - _, err = volume.viewTreeByNonce.Put(volumeView.nonce, volumeView) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByNonce.Put() for SnapShotList element %v failed: %v", volume.volumeName, snapShotIndex, err) - } - _, err = volume.viewTreeByID.Put(volumeView.snapShotID, volumeView) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByID.Put() for SnapShotList element %v failed: %v", volume.volumeName, snapShotIndex, err) - } - _, err = volume.viewTreeByTime.Put(volumeView.snapShotTime, volumeView) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByTime.Put() for SnapShotList element %v failed: %v", volume.volumeName, snapShotIndex, err) - } - _, err = volume.viewTreeByName.Put(volumeView.snapShotName, volumeView) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByName.Put() for SnapShotList element %v failed: %v", volume.volumeName, snapShotIndex, err) - } - } - - if 0 == checkpointObjectTrailerV3.SnapShotListNumElements { - volume.priorView = nil - } else { - _, volumeViewAsValue, ok, err = volume.viewTreeByNonce.GetByIndex(int(checkpointObjectTrailerV3.SnapShotListNumElements) - 1) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByID.GetByIndex() failed: %v", volume.volumeName, err) - } - if !ok { - logger.Fatalf("Logic error - volume %v's viewTreeByID.GetByIndex() returned !ok", volume.volumeName) - } - volume.priorView, ok = volumeViewAsValue.(*volumeViewStruct) - if !ok { - logger.Fatalf("Logic error - volume %v's volumeViewAsValue.(*volumeViewStruct) returned !ok", volume.volumeName) - } - } - - // Validate checkpointObjectTrailerBuf was entirely consumed - - if 0 != len(checkpointObjectTrailerBuf) { - err = fmt.Errorf("Extra %v bytes found in volume %v's checkpointObjectTrailer", len(checkpointObjectTrailerBuf), volume.volumeName) - return - } - - // Derive available SnapShotIDs - - volume.availableSnapShotIDList = list.New() - - for snapShotID = uint64(1); snapShotID < volume.dotSnapShotDirSnapShotID; snapShotID++ { - _, ok, err = volume.viewTreeByID.GetByKey(snapShotID) - if nil != err { - logger.Fatalf("Logic error - volume %v's viewTreeByID.GetByKey() failed: %v", volume.volumeName, err) - } - if !ok { - volume.availableSnapShotIDList.PushBack(snapShotID) - } - } - } - } else { - // Note that, currently, fetchCheckpointContainerHeader() would have error'd preventing us to reach here - err = fmt.Errorf("CheckpointHeader.CheckpointVersion (%v) for volume %s not supported", volume.checkpointHeader.CheckpointVersion, volume.volumeName) - return - } - - volume.maxNonce = (1 << (64 - volume.snapShotIDNumBits)) - 1 - volume.nextNonce = volume.checkpointHeader.ReservedToNonce - - // Indicate that we've had (at least) one checkpointTriggeringEvent - - volume.checkpointTriggeringEvents++ - - // Check for the need to process a Replay Log - - if "" == volume.replayLogFileName { - // Replay Log is disabled... simply return now - err = nil - return - } - - volume.replayLogFile, err = platform.OpenFileSync(volume.replayLogFileName, os.O_RDWR, 0600) - if nil != err { - if os.IsNotExist(err) { - // No Replay Log found... simply return now - err = nil - return - } - logger.FatalfWithError(err, "platform.OpenFileSync(%v,os.O_RDWR,) failed", volume.replayLogFileName) - } - - // Compute current end of Replay Log and round it down to replayLogWriteBufferAlignment multiple if necessary - - replayLogSize, err = volume.replayLogFile.Seek(0, 2) - if nil != err { - return - } - replayLogSize = int64(uintptr(replayLogSize) & ^(replayLogWriteBufferAlignment - 1)) - - // Seek back to start of Replay Log - - _, err = volume.replayLogFile.Seek(0, 0) - replayLogPosition = 0 - - defaultReplayLogReadBuffer = constructReplayLogWriteBuffer(globals.replayLogTransactionFixedPartStructSize) - - for replayLogPosition < replayLogSize { - // Read next Transaction Header from Replay Log - - _, err = io.ReadFull(volume.replayLogFile, defaultReplayLogReadBuffer) - if nil != err { - return - } - - _, err = cstruct.Unpack(defaultReplayLogReadBuffer, &replayLogTransactionFixedPart, LittleEndian) - if nil != err { - // Logic error - we should never fail cstruct.Unpack() call - - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - - // Ensure entire Transaction is in replayLogReadBuffer and we are positioned correctly - - bytesNeeded = globals.uint64Size + globals.uint64Size + replayLogTransactionFixedPart.BytesFollowing - - if bytesNeeded <= uint64(len(defaultReplayLogReadBuffer)) { - // We've already read the entire Transaction - - replayLogReadBuffer = defaultReplayLogReadBuffer - } else { - // Back up and read entire Transaction into fresh replayLogReadBuffer - - _, err = volume.replayLogFile.Seek(replayLogPosition, 0) - - replayLogReadBuffer = constructReplayLogWriteBuffer(bytesNeeded) - - _, err = io.ReadFull(volume.replayLogFile, replayLogReadBuffer) - if nil != err { - return - } - } - - // Validate ECMA CRC-64 of Transaction - - computedCRC64 = crc64.Checksum(replayLogReadBuffer[globals.uint64Size:bytesNeeded], globals.crc64ECMATable) - if computedCRC64 != replayLogTransactionFixedPart.CRC64 { - // Corruption in replayLogTransactionFixedPart - so exit as if Replay Log ended here - - logger.Infof("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - - _, err = volume.replayLogFile.Seek(replayLogPosition, 0) - if nil != err { - return - } - err = volume.replayLogFile.Truncate(replayLogPosition) - return - } - - // Replay Transaction - - volume.checkpointTriggeringEvents++ - - replayLogReadBufferPosition = globals.replayLogTransactionFixedPartStructSize - - switch replayLogTransactionFixedPart.TransactionType { - case transactionPutInodeRec: - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &inodeNumber, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &valueLen, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - value = make([]byte, valueLen) - copy(value, replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+valueLen]) - ok, err = volume.liveView.inodeRecWrapper.bPlusTree.PatchByKey(inodeNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.inodeRecWrapper.bPlusTree.PatchByKey() failure: %v", volume.volumeName, err) - } - if !ok { - _, err = volume.liveView.inodeRecWrapper.bPlusTree.Put(inodeNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.inodeRecWrapper.bPlusTree.Put() failure: %v", volume.volumeName, err) - } - } - case transactionPutInodeRecs: - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &numInodes, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - for inodeIndex = 0; inodeIndex < numInodes; inodeIndex++ { - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &inodeNumber, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &valueLen, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - value = make([]byte, valueLen) - copy(value, replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+valueLen]) - replayLogReadBufferPosition += valueLen - ok, err = volume.liveView.inodeRecWrapper.bPlusTree.PatchByKey(inodeNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.inodeRecWrapper.bPlusTree.PatchByKey() failure: %v", volume.volumeName, err) - } - if !ok { - _, err = volume.liveView.inodeRecWrapper.bPlusTree.Put(inodeNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.inodeRecWrapper.bPlusTree.Put() failure: %v", volume.volumeName, err) - } - } - } - case transactionDeleteInodeRec: - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &inodeNumber, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - _, err = volume.liveView.inodeRecWrapper.bPlusTree.DeleteByKey(inodeNumber) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.inodeRecWrapper.bPlusTree.DeleteByKey() failure: %v", volume.volumeName, err) - } - case transactionPutLogSegmentRec: - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &logSegmentNumber, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &valueLen, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - value = make([]byte, valueLen) - copy(value, replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+valueLen]) - ok, err = volume.liveView.logSegmentRecWrapper.bPlusTree.PatchByKey(logSegmentNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.logSegmentRecWrapper.bPlusTree.PatchByKey() failure: %v", volume.volumeName, err) - } - if !ok { - _, err = volume.liveView.logSegmentRecWrapper.bPlusTree.Put(logSegmentNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.logSegmentRecWrapper.bPlusTree.Put() failure: %v", volume.volumeName, err) - } - } - if nil != volume.priorView { - _, err = volume.priorView.createdObjectsWrapper.bPlusTree.Put(logSegmentNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.priorView.createdObjectsWrapper.bPlusTree.Put() failure: %v", volume.volumeName, err) - } - } - case transactionDeleteLogSegmentRec: - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &logSegmentNumber, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - containerNameAsValue, ok, err = volume.liveView.logSegmentRecWrapper.bPlusTree.GetByKey(logSegmentNumber) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.logSegmentRecWrapper.bPlusTree.GetByKey() failure: %v", volume.volumeName, err) - } - if !ok { - logger.Fatalf("Replay Log for Volume %s hit unexpected missing logSegmentNumber (0x%016X) in LogSegmentRecB+Tree", volume.volumeName, logSegmentNumber) - } - _, err = volume.liveView.logSegmentRecWrapper.bPlusTree.DeleteByKey(logSegmentNumber) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.logSegmentRecWrapper.bPlusTree.DeleteByKey() failure: %v", volume.volumeName, err) - } - if nil == volume.priorView { - _, err = volume.liveView.deletedObjectsWrapper.bPlusTree.Put(logSegmentNumber, containerNameAsValue) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.deletedObjectsWrapper.bPlusTree.Put() failure: %v", volume.volumeName, err) - } - } else { - ok, err = volume.priorView.createdObjectsWrapper.bPlusTree.DeleteByKey(logSegmentNumber) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.priorView.createdObjectsWrapper.bPlusTree.DeleteByKey() failure: %v", volume.volumeName, err) - } - if ok { - _, err = volume.liveView.deletedObjectsWrapper.bPlusTree.Put(logSegmentNumber, containerNameAsValue) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.deletedObjectsWrapper.bPlusTree.Put() failure: %v", volume.volumeName, err) - } - } else { - _, err = volume.priorView.deletedObjectsWrapper.bPlusTree.Put(logSegmentNumber, containerNameAsValue) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.priorView.deletedObjectsWrapper.bPlusTree.Put() failure: %v", volume.volumeName, err) - } - } - } - case transactionPutBPlusTreeObject: - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &objectNumber, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &valueLen, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - replayLogReadBufferPosition += globals.uint64Size - value = make([]byte, valueLen) - copy(value, replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+valueLen]) - ok, err = volume.liveView.bPlusTreeObjectWrapper.bPlusTree.PatchByKey(objectNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.bPlusTreeObjectWrapper.bPlusTree.PatchByKey() failure: %v", volume.volumeName, err) - } - if !ok { - _, err = volume.liveView.bPlusTreeObjectWrapper.bPlusTree.Put(objectNumber, value) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.bPlusTreeObjectWrapper.bPlusTree.Put() failure: %v", volume.volumeName, err) - } - } - case transactionDeleteBPlusTreeObject: - _, err = cstruct.Unpack(replayLogReadBuffer[replayLogReadBufferPosition:replayLogReadBufferPosition+globals.uint64Size], &objectNumber, LittleEndian) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected cstruct.Unpack() failure: %v", volume.volumeName, err) - } - _, err = volume.liveView.bPlusTreeObjectWrapper.bPlusTree.DeleteByKey(objectNumber) - if nil != err { - logger.Fatalf("Reply Log for Volume %s hit unexpected volume.liveView.bPlusTreeObjectWrapper.bPlusTree.DeleteByKey() failure: %v", volume.volumeName, err) - } - default: - // Corruption in replayLogTransactionFixedPart - so exit as if Replay Log ended here - - logger.Infof("Reply Log for Volume %s hit unexpected replayLogTransactionFixedPart.TransactionType == %v", volume.volumeName, replayLogTransactionFixedPart.TransactionType) - - _, err = volume.replayLogFile.Seek(replayLogPosition, 0) - if nil != err { - return - } - err = volume.replayLogFile.Truncate(replayLogPosition) - return - } - - // Finally, make replayLogPosition match where we actually are in volume.replayLogFile - - replayLogPosition += int64(len(replayLogReadBuffer)) - } - - err = nil - return -} - -// putCheckpoint does the dirty work of creating a ProxyFS checkpoint -- its complicated. -// -// putCheckpoint collects bucketized statistics of the total time taken by the -// checkpoint and the number of bytes written to Swift to store the checkpoint. -// In addition, the time required by, and number of bytes transferred by, each step -// along the way is recorded in individual PutCheckpoint* bucketstats. -// -// Note that startTime2 is recorded as the starting time of an interval several -// times in this routine, and each time it is used to measure time.Since(startTime2) -// it is reset to the current time. -// -func (volume *volumeStruct) putCheckpoint() (err error) { - - // measure entire time and bytes for putCheckpoint - startTime := time.Now() - var chunkedPutBytes uint64 - defer func() { - globals.PutCheckpointUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - if err != nil { - globals.PutCheckpointErrors.Add(1) - return - } - globals.PutCheckpointBytes.Add(chunkedPutBytes) - }() - - var ( - bytesUsedCumulative uint64 - bytesUsedThisBPlusTree uint64 - checkpointContainerHeaders map[string][]string - checkpointHeaderValue string - checkpointHeaderValues []string - checkpointObjectTrailer *CheckpointObjectTrailerV3Struct - checkpointObjectTrailerBeginningOffset uint64 - checkpointObjectTrailerEndingOffset uint64 - checkpointTrailerBuf []byte - combinedBPlusTreeLayout sortedmap.LayoutReport - containerNameAsByteSlice []byte - containerNameAsValue sortedmap.Value - delayedObjectDeleteList []delayedObjectDeleteStruct - elementOfBPlusTreeLayout ElementOfBPlusTreeLayoutStruct - elementOfBPlusTreeLayoutBuf []byte - elementOfSnapShotListBuf []byte - logSegmentObjectsToDelete int - objectNumber uint64 - objectNumberAsKey sortedmap.Key - ok bool - postponedCreatedObjectNumber uint64 - postponedCreatedObjectsFound bool - snapShotBPlusTreeObjectBPlusTreeObjectLengthBuf []byte - snapShotBPlusTreeObjectBPlusTreeObjectLengthStruct uint64Struct - snapShotBPlusTreeObjectBPlusTreeObjectNumberBuf []byte - snapShotBPlusTreeObjectBPlusTreeObjectNumberStruct uint64Struct - snapShotBPlusTreeObjectBPlusTreeObjectOffsetBuf []byte - snapShotBPlusTreeObjectBPlusTreeObjectOffsetStruct uint64Struct - snapShotCreatedObjectsBPlusTreeObjectLengthBuf []byte - snapShotCreatedObjectsBPlusTreeObjectLengthStruct uint64Struct - snapShotCreatedObjectsBPlusTreeObjectNumberBuf []byte - snapShotCreatedObjectsBPlusTreeObjectNumberStruct uint64Struct - snapShotCreatedObjectsBPlusTreeObjectOffsetBuf []byte - snapShotCreatedObjectsBPlusTreeObjectOffsetStruct uint64Struct - snapShotDeletedObjectsBPlusTreeObjectLengthBuf []byte - snapShotDeletedObjectsBPlusTreeObjectLengthStruct uint64Struct - snapShotDeletedObjectsBPlusTreeObjectNumberBuf []byte - snapShotDeletedObjectsBPlusTreeObjectNumberStruct uint64Struct - snapShotDeletedObjectsBPlusTreeObjectOffsetBuf []byte - snapShotDeletedObjectsBPlusTreeObjectOffsetStruct uint64Struct - snapShotIDBuf []byte - snapShotIDStruct uint64Struct - snapShotInodeRecBPlusTreeObjectLengthBuf []byte - snapShotInodeRecBPlusTreeObjectLengthStruct uint64Struct - snapShotInodeRecBPlusTreeObjectNumberBuf []byte - snapShotInodeRecBPlusTreeObjectNumberStruct uint64Struct - snapShotInodeRecBPlusTreeObjectOffsetBuf []byte - snapShotInodeRecBPlusTreeObjectOffsetStruct uint64Struct - snapShotListBuf []byte - snapShotLogSegmentRecBPlusTreeObjectLengthBuf []byte - snapShotLogSegmentRecBPlusTreeObjectLengthStruct uint64Struct - snapShotLogSegmentRecBPlusTreeObjectNumberBuf []byte - snapShotLogSegmentRecBPlusTreeObjectNumberStruct uint64Struct - snapShotLogSegmentRecBPlusTreeObjectOffsetBuf []byte - snapShotLogSegmentRecBPlusTreeObjectOffsetStruct uint64Struct - snapShotNameBuf []byte - snapShotNameBufLenBuf []byte - snapShotNameBufLenStruct uint64Struct - snapShotNonceBuf []byte - snapShotNonceStruct uint64Struct - snapShotTimeStampBuf []byte - snapShotTimeStampBufLenBuf []byte - snapShotTimeStampBufLenStruct uint64Struct - treeLayoutBuf []byte - treeLayoutBufSize uint64 - volumeView *volumeViewStruct - volumeViewAsValue sortedmap.Value - volumeViewCount int - volumeViewIndex int - ) - - if 0 == volume.checkpointTriggeringEvents { - stats.IncrementOperations(&stats.SkippedCheckpoints) - err = nil - return - } - - stats.IncrementOperations(&stats.AttemptedCheckpoints) - - checkpointObjectTrailer = &CheckpointObjectTrailerV3Struct{} - - startTime2 := time.Now() - volume.liveView.inodeRecWrapper.ClearCounters() - - checkpointObjectTrailer.InodeRecBPlusTreeObjectNumber, - checkpointObjectTrailer.InodeRecBPlusTreeObjectOffset, - checkpointObjectTrailer.InodeRecBPlusTreeObjectLength, - err = volume.liveView.inodeRecWrapper.bPlusTree.Flush(false) - if nil != err { - return - } - globals.PutCheckpointInodeRecUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - globals.PutCheckpointInodeRecNodes.Add(volume.liveView.inodeRecWrapper.totalPutNodes) - globals.PutCheckpointInodeRecBytes.Add(volume.liveView.inodeRecWrapper.totalPutBytes) - chunkedPutBytes += volume.liveView.inodeRecWrapper.totalPutBytes - - startTime2 = time.Now() - volume.liveView.logSegmentRecWrapper.ClearCounters() - - checkpointObjectTrailer.LogSegmentRecBPlusTreeObjectNumber, - checkpointObjectTrailer.LogSegmentRecBPlusTreeObjectOffset, - checkpointObjectTrailer.LogSegmentRecBPlusTreeObjectLength, - err = volume.liveView.logSegmentRecWrapper.bPlusTree.Flush(false) - if nil != err { - return - } - globals.PutCheckpointLogSegmentUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - globals.PutCheckpointLogSegmentNodes.Add(volume.liveView.logSegmentRecWrapper.totalPutNodes) - globals.PutCheckpointLogSegmentBytes.Add(volume.liveView.logSegmentRecWrapper.totalPutBytes) - chunkedPutBytes += volume.liveView.logSegmentRecWrapper.totalPutBytes - - startTime2 = time.Now() - volume.liveView.bPlusTreeObjectWrapper.ClearCounters() - - checkpointObjectTrailer.BPlusTreeObjectBPlusTreeObjectNumber, - checkpointObjectTrailer.BPlusTreeObjectBPlusTreeObjectOffset, - checkpointObjectTrailer.BPlusTreeObjectBPlusTreeObjectLength, - err = volume.liveView.bPlusTreeObjectWrapper.bPlusTree.Flush(false) - if nil != err { - return - } - globals.PutCheckpointbPlusTreeObjectUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - globals.PutCheckpointbPlusTreeObjectNodes.Add(volume.liveView.bPlusTreeObjectWrapper.totalPutNodes) - globals.PutCheckpointbPlusTreeObjectBytes.Add(volume.liveView.bPlusTreeObjectWrapper.totalPutBytes) - chunkedPutBytes += volume.liveView.bPlusTreeObjectWrapper.totalPutBytes - - startTime2 = time.Now() - volumeViewCount, err = volume.viewTreeByNonce.Len() - if nil != err { - logger.Fatalf("volume.viewTreeByNonce.Len() failed: %v", err) - } - - for volumeViewIndex = 0; volumeViewIndex < volumeViewCount; volumeViewIndex++ { - _, volumeViewAsValue, ok, err = volume.viewTreeByNonce.GetByIndex(volumeViewIndex) - if nil != err { - logger.Fatalf("volume.viewTreeByNonce.GetByIndex(%v) failed: %v", volumeViewIndex, err) - } - if !ok { - logger.Fatalf("volume.viewTreeByNonce.GetByIndex(%v) returned !ok", volumeViewIndex) - } - - volumeView, ok = volumeViewAsValue.(*volumeViewStruct) - if !ok { - logger.Fatalf("volume.viewTreeByNonce.GetByIndex(%v) returned something other than a *volumeViewStruct", volumeViewIndex) - } - - if volumeView == volume.priorView { - // We must avoid a deadlock that would occur if, during Flush() of volume.priorView's - // createdObjectsWrapper.bPlusTree, we needed to do a Put() into it due to the creation - // of a new Checkpoint Object. The following sequence postpones those Put() calls - // until after the Flush() completes, performs them, then retries the Flush() call. - - volume.postponePriorViewCreatedObjectsPuts = true - postponedCreatedObjectsFound = true - - for postponedCreatedObjectsFound { - _, _, _, err = volumeView.createdObjectsWrapper.bPlusTree.Flush(false) - if nil != err { - volume.postponePriorViewCreatedObjectsPuts = false - volume.postponedPriorViewCreatedObjectsPuts = make(map[uint64]struct{}) - return - } - - postponedCreatedObjectsFound = 0 < len(volume.postponedPriorViewCreatedObjectsPuts) - - if postponedCreatedObjectsFound { - for postponedCreatedObjectNumber = range volume.postponedPriorViewCreatedObjectsPuts { - ok, err = volume.priorView.createdObjectsWrapper.bPlusTree.Put(postponedCreatedObjectNumber, []byte(volume.checkpointContainerName)) - if nil != err { - volume.postponePriorViewCreatedObjectsPuts = false - volume.postponedPriorViewCreatedObjectsPuts = make(map[uint64]struct{}) - return - } - if !ok { - volume.postponePriorViewCreatedObjectsPuts = false - volume.postponedPriorViewCreatedObjectsPuts = make(map[uint64]struct{}) - err = fmt.Errorf("volume.priorView.createdObjectsWrapper.bPlusTree.Put() returned !ok") - return - } - } - - volume.postponedPriorViewCreatedObjectsPuts = make(map[uint64]struct{}) - } - } - - volume.postponePriorViewCreatedObjectsPuts = false - } else { - _, _, _, err = volumeView.createdObjectsWrapper.bPlusTree.Flush(false) - if nil != err { - return - } - } - - err = volumeView.createdObjectsWrapper.bPlusTree.Prune() - if nil != err { - return - } - - _, _, _, err = volumeView.deletedObjectsWrapper.bPlusTree.Flush(false) - if nil != err { - return - } - - err = volumeView.deletedObjectsWrapper.bPlusTree.Prune() - if nil != err { - return - } - } - - err = volume.liveView.inodeRecWrapper.bPlusTree.Prune() - if nil != err { - return - } - err = volume.liveView.logSegmentRecWrapper.bPlusTree.Prune() - if nil != err { - return - } - err = volume.liveView.bPlusTreeObjectWrapper.bPlusTree.Prune() - if nil != err { - return - } - globals.PutCheckpointSnapshotFlushUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - - startTime2 = time.Now() - checkpointObjectTrailer.InodeRecBPlusTreeLayoutNumElements = uint64(len(volume.liveView.inodeRecWrapper.bPlusTreeTracker.bPlusTreeLayout)) - checkpointObjectTrailer.LogSegmentRecBPlusTreeLayoutNumElements = uint64(len(volume.liveView.logSegmentRecWrapper.bPlusTreeTracker.bPlusTreeLayout)) - checkpointObjectTrailer.BPlusTreeObjectBPlusTreeLayoutNumElements = uint64(len(volume.liveView.bPlusTreeObjectWrapper.bPlusTreeTracker.bPlusTreeLayout)) - checkpointObjectTrailer.CreatedObjectsBPlusTreeLayoutNumElements = uint64(len(volume.liveView.createdObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout)) - checkpointObjectTrailer.DeletedObjectsBPlusTreeLayoutNumElements = uint64(len(volume.liveView.deletedObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout)) - - treeLayoutBufSize = checkpointObjectTrailer.InodeRecBPlusTreeLayoutNumElements - treeLayoutBufSize += checkpointObjectTrailer.LogSegmentRecBPlusTreeLayoutNumElements - treeLayoutBufSize += checkpointObjectTrailer.BPlusTreeObjectBPlusTreeLayoutNumElements - treeLayoutBufSize += checkpointObjectTrailer.CreatedObjectsBPlusTreeLayoutNumElements - treeLayoutBufSize += checkpointObjectTrailer.DeletedObjectsBPlusTreeLayoutNumElements - treeLayoutBufSize *= globals.ElementOfBPlusTreeLayoutStructSize - - treeLayoutBuf = make([]byte, 0, treeLayoutBufSize) - - for elementOfBPlusTreeLayout.ObjectNumber, elementOfBPlusTreeLayout.ObjectBytes = range volume.liveView.inodeRecWrapper.bPlusTreeTracker.bPlusTreeLayout { - elementOfBPlusTreeLayoutBuf, err = cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) for volume %v inodeRec failed: %v", volume.volumeName, err) - } - treeLayoutBuf = append(treeLayoutBuf, elementOfBPlusTreeLayoutBuf...) - } - - for elementOfBPlusTreeLayout.ObjectNumber, elementOfBPlusTreeLayout.ObjectBytes = range volume.liveView.logSegmentRecWrapper.bPlusTreeTracker.bPlusTreeLayout { - elementOfBPlusTreeLayoutBuf, err = cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) for volume %v logSegmentRec failed: %v", volume.volumeName, err) - } - treeLayoutBuf = append(treeLayoutBuf, elementOfBPlusTreeLayoutBuf...) - } - - for elementOfBPlusTreeLayout.ObjectNumber, elementOfBPlusTreeLayout.ObjectBytes = range volume.liveView.bPlusTreeObjectWrapper.bPlusTreeTracker.bPlusTreeLayout { - elementOfBPlusTreeLayoutBuf, err = cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) for volume %v bPlusTreeObject failed: %v", volume.volumeName, err) - } - treeLayoutBuf = append(treeLayoutBuf, elementOfBPlusTreeLayoutBuf...) - } - - for elementOfBPlusTreeLayout.ObjectNumber, elementOfBPlusTreeLayout.ObjectBytes = range volume.liveView.createdObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout { - elementOfBPlusTreeLayoutBuf, err = cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) for volume %v createdObjects failed: %v", volume.volumeName, err) - } - treeLayoutBuf = append(treeLayoutBuf, elementOfBPlusTreeLayoutBuf...) - } - - for elementOfBPlusTreeLayout.ObjectNumber, elementOfBPlusTreeLayout.ObjectBytes = range volume.liveView.deletedObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout { - elementOfBPlusTreeLayoutBuf, err = cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(&elementOfBPlusTreeLayout, LittleEndian) for volume %v deletedObjects failed: %v", volume.volumeName, err) - } - treeLayoutBuf = append(treeLayoutBuf, elementOfBPlusTreeLayoutBuf...) - } - globals.PutCheckpointTreeLayoutUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - - startTime2 = time.Now() - checkpointObjectTrailer.SnapShotIDNumBits = uint64(volume.snapShotIDNumBits) - - checkpointObjectTrailer.SnapShotListNumElements = uint64(volumeViewCount) - - snapShotListBuf = make([]byte, 0) - - for volumeViewIndex = 0; volumeViewIndex < volumeViewCount; volumeViewIndex++ { - _, volumeViewAsValue, ok, err = volume.viewTreeByNonce.GetByIndex(volumeViewIndex) - if nil != err { - logger.Fatalf("volume.viewTreeByNonce.GetByIndex(%v) failed: %v", volumeViewIndex, err) - } - if !ok { - logger.Fatalf("volume.viewTreeByNonce.GetByIndex(%v) returned !ok", volumeViewIndex) - } - - volumeView, ok = volumeViewAsValue.(*volumeViewStruct) - if !ok { - logger.Fatalf("volume.viewTreeByNonce.GetByIndex(%v) returned something other than a *volumeViewStruct", volumeViewIndex) - } - - snapShotNonceStruct.U64 = volumeView.nonce - snapShotNonceBuf, err = cstruct.Pack(snapShotNonceStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotNonceStruct, LittleEndian) failed: %v", err) - } - - snapShotIDStruct.U64 = volumeView.snapShotID - snapShotIDBuf, err = cstruct.Pack(snapShotIDStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotIDStruct, LittleEndian) failed: %v", err) - } - - snapShotTimeStampBuf, err = volumeView.snapShotTime.MarshalBinary() - if nil != err { - logger.Fatalf("volumeView.snapShotTime.MarshalBinary()for volumeViewIndex %v failed: %v", volumeViewIndex, err) - } - snapShotTimeStampBufLenStruct.U64 = uint64(len(snapShotTimeStampBuf)) - snapShotTimeStampBufLenBuf, err = cstruct.Pack(snapShotTimeStampBufLenStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotTimeStampBufLenStruct, LittleEndian) failed: %v", err) - } - - snapShotNameBuf = utils.StringToByteSlice(volumeView.snapShotName) - snapShotNameBufLenStruct.U64 = uint64(len(snapShotNameBuf)) - snapShotNameBufLenBuf, err = cstruct.Pack(snapShotNameBufLenStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotNameBufLenStruct, LittleEndian) failed: %v", err) - } - - snapShotInodeRecBPlusTreeObjectNumberStruct.U64, - snapShotInodeRecBPlusTreeObjectOffsetStruct.U64, - snapShotInodeRecBPlusTreeObjectLengthStruct.U64 = volumeView.inodeRecWrapper.bPlusTree.FetchLocation() - snapShotInodeRecBPlusTreeObjectNumberBuf, err = cstruct.Pack(snapShotInodeRecBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotInodeRecBPlusTreeObjectNumberStruct, LittleEndian) failed: %v", err) - } - snapShotInodeRecBPlusTreeObjectOffsetBuf, err = cstruct.Pack(snapShotInodeRecBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotInodeRecBPlusTreeObjectOffsetStruct, LittleEndian) failed: %v", err) - } - snapShotInodeRecBPlusTreeObjectLengthBuf, err = cstruct.Pack(snapShotInodeRecBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotInodeRecBPlusTreeObjectLengthStruct, LittleEndian) failed: %v", err) - } - - snapShotLogSegmentRecBPlusTreeObjectNumberStruct.U64, - snapShotLogSegmentRecBPlusTreeObjectOffsetStruct.U64, - snapShotLogSegmentRecBPlusTreeObjectLengthStruct.U64 = volumeView.logSegmentRecWrapper.bPlusTree.FetchLocation() - snapShotLogSegmentRecBPlusTreeObjectNumberBuf, err = cstruct.Pack(snapShotLogSegmentRecBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotLogSegmentRecBPlusTreeObjectNumberStruct, LittleEndian) failed: %v", err) - } - snapShotLogSegmentRecBPlusTreeObjectOffsetBuf, err = cstruct.Pack(snapShotLogSegmentRecBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotLogSegmentRecBPlusTreeObjectOffsetStruct, LittleEndian) failed: %v", err) - } - snapShotLogSegmentRecBPlusTreeObjectLengthBuf, err = cstruct.Pack(snapShotLogSegmentRecBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotLogSegmentRecBPlusTreeObjectLengthStruct, LittleEndian) failed: %v", err) - } - - snapShotBPlusTreeObjectBPlusTreeObjectNumberStruct.U64, - snapShotBPlusTreeObjectBPlusTreeObjectOffsetStruct.U64, - snapShotBPlusTreeObjectBPlusTreeObjectLengthStruct.U64 = volumeView.bPlusTreeObjectWrapper.bPlusTree.FetchLocation() - snapShotBPlusTreeObjectBPlusTreeObjectNumberBuf, err = cstruct.Pack(snapShotBPlusTreeObjectBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotBPlusTreeObjectBPlusTreeObjectNumberStruct, LittleEndian) failed: %v", err) - } - snapShotBPlusTreeObjectBPlusTreeObjectOffsetBuf, err = cstruct.Pack(snapShotBPlusTreeObjectBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotBPlusTreeObjectBPlusTreeObjectOffsetStruct, LittleEndian) failed: %v", err) - } - snapShotBPlusTreeObjectBPlusTreeObjectLengthBuf, err = cstruct.Pack(snapShotBPlusTreeObjectBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotBPlusTreeObjectBPlusTreeObjectLengthStruct, LittleEndian) failed: %v", err) - } - - snapShotCreatedObjectsBPlusTreeObjectNumberStruct.U64, - snapShotCreatedObjectsBPlusTreeObjectOffsetStruct.U64, - snapShotCreatedObjectsBPlusTreeObjectLengthStruct.U64 = volumeView.createdObjectsWrapper.bPlusTree.FetchLocation() - snapShotCreatedObjectsBPlusTreeObjectNumberBuf, err = cstruct.Pack(snapShotCreatedObjectsBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotCreatedObjectsBPlusTreeObjectNumberStruct, LittleEndian) failed: %v", err) - } - snapShotCreatedObjectsBPlusTreeObjectOffsetBuf, err = cstruct.Pack(snapShotCreatedObjectsBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotCreatedObjectsBPlusTreeObjectOffsetStruct, LittleEndian) failed: %v", err) - } - snapShotCreatedObjectsBPlusTreeObjectLengthBuf, err = cstruct.Pack(snapShotCreatedObjectsBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotCreatedObjectsBPlusTreeObjectLengthStruct, LittleEndian) failed: %v", err) - } - - snapShotDeletedObjectsBPlusTreeObjectNumberStruct.U64, - snapShotDeletedObjectsBPlusTreeObjectOffsetStruct.U64, - snapShotDeletedObjectsBPlusTreeObjectLengthStruct.U64 = volumeView.deletedObjectsWrapper.bPlusTree.FetchLocation() - snapShotDeletedObjectsBPlusTreeObjectNumberBuf, err = cstruct.Pack(snapShotDeletedObjectsBPlusTreeObjectNumberStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotDeletedObjectsBPlusTreeObjectNumberStruct, LittleEndian) failed: %v", err) - } - snapShotDeletedObjectsBPlusTreeObjectOffsetBuf, err = cstruct.Pack(snapShotDeletedObjectsBPlusTreeObjectOffsetStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotDeletedObjectsBPlusTreeObjectOffsetStruct, LittleEndian) failed: %v", err) - } - snapShotDeletedObjectsBPlusTreeObjectLengthBuf, err = cstruct.Pack(snapShotDeletedObjectsBPlusTreeObjectLengthStruct, LittleEndian) - if nil != err { - logger.Fatalf("cstruct.Pack(snapShotDeletedObjectsBPlusTreeObjectLengthStruct, LittleEndian) failed: %v", err) - } - - elementOfSnapShotListBuf = make([]byte, 0, 19*globals.uint64Size+snapShotTimeStampBufLenStruct.U64+snapShotNameBufLenStruct.U64) - - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotNonceBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotIDBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotTimeStampBufLenBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotTimeStampBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotNameBufLenBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotNameBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotInodeRecBPlusTreeObjectNumberBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotInodeRecBPlusTreeObjectOffsetBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotInodeRecBPlusTreeObjectLengthBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotLogSegmentRecBPlusTreeObjectNumberBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotLogSegmentRecBPlusTreeObjectOffsetBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotLogSegmentRecBPlusTreeObjectLengthBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotBPlusTreeObjectBPlusTreeObjectNumberBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotBPlusTreeObjectBPlusTreeObjectOffsetBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotBPlusTreeObjectBPlusTreeObjectLengthBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotCreatedObjectsBPlusTreeObjectNumberBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotCreatedObjectsBPlusTreeObjectOffsetBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotCreatedObjectsBPlusTreeObjectLengthBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotDeletedObjectsBPlusTreeObjectNumberBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotDeletedObjectsBPlusTreeObjectOffsetBuf...) - elementOfSnapShotListBuf = append(elementOfSnapShotListBuf, snapShotDeletedObjectsBPlusTreeObjectLengthBuf...) - - snapShotListBuf = append(snapShotListBuf, elementOfSnapShotListBuf...) - } - checkpointObjectTrailer.SnapShotListTotalSize = uint64(len(snapShotListBuf)) - - globals.PutCheckpointSnapshotListUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - - startTime2 = time.Now() - - checkpointTrailerBuf, err = cstruct.Pack(checkpointObjectTrailer, LittleEndian) - if nil != err { - return - } - - err = volume.openCheckpointChunkedPutContextIfNecessary() - if nil != err { - return - } - - checkpointObjectTrailerBeginningOffset, err = volume.bytesPutToCheckpointChunkedPutContext() - if nil != err { - return - } - - err = volume.sendChunkToCheckpointChunkedPutContext(checkpointTrailerBuf) - if nil != err { - return - } - globals.PutCheckpointCheckpointTrailerBytes.Add(uint64(len(checkpointTrailerBuf))) - chunkedPutBytes += uint64(len(checkpointTrailerBuf)) - - err = volume.sendChunkToCheckpointChunkedPutContext(treeLayoutBuf) - if nil != err { - return - } - globals.PutCheckpointTreeLayoutBytes.Add(uint64(len(treeLayoutBuf))) - chunkedPutBytes += uint64(len(treeLayoutBuf)) - - if 0 < len(snapShotListBuf) { - err = volume.sendChunkToCheckpointChunkedPutContext(snapShotListBuf) - if nil != err { - return - } - } - globals.PutCheckpointSnapshotListBytes.Add(uint64(len(snapShotListBuf))) - chunkedPutBytes += uint64(len(snapShotListBuf)) - - checkpointObjectTrailerEndingOffset, err = volume.bytesPutToCheckpointChunkedPutContext() - if nil != err { - return - } - - err = volume.closeCheckpointChunkedPutContext() - if nil != err { - return - } - globals.PutCheckpointChunkedPutUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - - startTime2 = time.Now() - - // Before updating checkpointHeader, start accounting for unreferencing of prior checkpointTrailer - - combinedBPlusTreeLayout = make(sortedmap.LayoutReport) - - if 0 != volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber { - combinedBPlusTreeLayout[volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber] = 0 - } - - // Now update checkpointHeader atomically indicating checkpoint is complete - - volume.checkpointHeader.CheckpointVersion = CheckpointVersion3 - - volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber = volume.checkpointChunkedPutContextObjectNumber - volume.checkpointHeader.CheckpointObjectTrailerStructObjectLength = checkpointObjectTrailerEndingOffset - checkpointObjectTrailerBeginningOffset - - checkpointHeaderValue = fmt.Sprintf("%016X %016X %016X %016X", - volume.checkpointHeader.CheckpointVersion, - volume.checkpointHeader.CheckpointObjectTrailerStructObjectNumber, - volume.checkpointHeader.CheckpointObjectTrailerStructObjectLength, - volume.checkpointHeader.ReservedToNonce, - ) - - if globals.etcdEnabled { - volume.checkpointHeaderEtcdRevision, err = volume.putEtcdCheckpointHeader(volume.checkpointHeader, volume.checkpointHeaderEtcdRevision) - if nil != err { - return - } - } - - checkpointHeaderValues = []string{checkpointHeaderValue} - - checkpointContainerHeaders = make(map[string][]string) - - checkpointContainerHeaders[CheckpointHeaderName] = checkpointHeaderValues - - err = swiftclient.ContainerPost(volume.accountName, volume.checkpointContainerName, checkpointContainerHeaders) - if nil != err { - return - } - if globals.logCheckpointHeaderPosts { - logger.Infof("POST'd checkpointHeaderValue %s for volume %s from putCheckpoint()", checkpointHeaderValue, volume.volumeName) - } - globals.PutCheckpointPostAndEtcdUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - - volume.checkpointTriggeringEvents = 0 - - stats.IncrementOperations(&stats.CompletedCheckpoints) - - startTime2 = time.Now() - // Remove replayLogFile if necessary - - if nil != volume.replayLogFile { - err = volume.replayLogFile.Close() - if nil != err { - return - } - volume.replayLogFile = nil - } - - if "" != volume.replayLogFileName { - err = os.Remove(volume.replayLogFileName) - if nil != err { - if !os.IsNotExist(err) { - return - } - } - } - - // Now continue computing what checkpoint objects may be deleted - - for objectNumber, bytesUsedThisBPlusTree = range volume.liveView.inodeRecWrapper.bPlusTreeTracker.bPlusTreeLayout { - bytesUsedCumulative, ok = combinedBPlusTreeLayout[objectNumber] - if ok { - combinedBPlusTreeLayout[objectNumber] = bytesUsedCumulative + bytesUsedThisBPlusTree - } else { - combinedBPlusTreeLayout[objectNumber] = bytesUsedThisBPlusTree - } - } - for objectNumber, bytesUsedThisBPlusTree = range volume.liveView.logSegmentRecWrapper.bPlusTreeTracker.bPlusTreeLayout { - bytesUsedCumulative, ok = combinedBPlusTreeLayout[objectNumber] - if ok { - combinedBPlusTreeLayout[objectNumber] = bytesUsedCumulative + bytesUsedThisBPlusTree - } else { - combinedBPlusTreeLayout[objectNumber] = bytesUsedThisBPlusTree - } - } - for objectNumber, bytesUsedThisBPlusTree = range volume.liveView.bPlusTreeObjectWrapper.bPlusTreeTracker.bPlusTreeLayout { - bytesUsedCumulative, ok = combinedBPlusTreeLayout[objectNumber] - if ok { - combinedBPlusTreeLayout[objectNumber] = bytesUsedCumulative + bytesUsedThisBPlusTree - } else { - combinedBPlusTreeLayout[objectNumber] = bytesUsedThisBPlusTree - } - } - for objectNumber, bytesUsedThisBPlusTree = range volume.liveView.createdObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout { - bytesUsedCumulative, ok = combinedBPlusTreeLayout[objectNumber] - if ok { - combinedBPlusTreeLayout[objectNumber] = bytesUsedCumulative + bytesUsedThisBPlusTree - } else { - combinedBPlusTreeLayout[objectNumber] = bytesUsedThisBPlusTree - } - } - for objectNumber, bytesUsedThisBPlusTree = range volume.liveView.deletedObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout { - bytesUsedCumulative, ok = combinedBPlusTreeLayout[objectNumber] - if ok { - combinedBPlusTreeLayout[objectNumber] = bytesUsedCumulative + bytesUsedThisBPlusTree - } else { - combinedBPlusTreeLayout[objectNumber] = bytesUsedThisBPlusTree - } - } - - logSegmentObjectsToDelete, err = volume.liveView.deletedObjectsWrapper.bPlusTree.Len() - if nil != err { - logger.Fatalf("volume.liveView.deletedObjectsWrapper.bPlusTree.Len() failed: %v", err) - } - - delayedObjectDeleteList = make([]delayedObjectDeleteStruct, 0, len(combinedBPlusTreeLayout)+logSegmentObjectsToDelete) - - for objectNumber, bytesUsedCumulative = range combinedBPlusTreeLayout { - if 0 == bytesUsedCumulative { - delete(volume.liveView.inodeRecWrapper.bPlusTreeTracker.bPlusTreeLayout, objectNumber) - delete(volume.liveView.logSegmentRecWrapper.bPlusTreeTracker.bPlusTreeLayout, objectNumber) - delete(volume.liveView.bPlusTreeObjectWrapper.bPlusTreeTracker.bPlusTreeLayout, objectNumber) - delete(volume.liveView.createdObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout, objectNumber) - delete(volume.liveView.deletedObjectsWrapper.bPlusTreeTracker.bPlusTreeLayout, objectNumber) - - if nil == volume.priorView { - delayedObjectDeleteList = append(delayedObjectDeleteList, delayedObjectDeleteStruct{containerName: volume.checkpointContainerName, objectNumber: objectNumber}) - } else { - ok, err = volume.priorView.createdObjectsWrapper.bPlusTree.DeleteByKey(objectNumber) - if nil != err { - logger.Fatalf("volume.priorView.createdObjectsWrapper.bPlusTree.DeleteByKey(objectNumber==0x%016X) failed: %v", objectNumber, err) - } - if ok { - delayedObjectDeleteList = append(delayedObjectDeleteList, delayedObjectDeleteStruct{containerName: volume.checkpointContainerName, objectNumber: objectNumber}) - } else { - ok, err = volume.priorView.deletedObjectsWrapper.bPlusTree.Put(objectNumber, utils.StringToByteSlice(volume.checkpointContainerName)) - if nil != err { - logger.Fatalf("volume.priorView.deletedObjectsWrapper.bPlusTree.Put(objectNumber==0x%016X,%s) failed: %v", objectNumber, volume.checkpointContainerName, err) - } - if !ok { - logger.Fatalf("volume.priorView.deletedObjectsWrapper.bPlusTree.Put(objectNumber==0x%016X,%s) returned !ok", objectNumber, volume.checkpointContainerName) - } - } - } - } - } - - for ; logSegmentObjectsToDelete > 0; logSegmentObjectsToDelete-- { - objectNumberAsKey, containerNameAsValue, ok, err = volume.liveView.deletedObjectsWrapper.bPlusTree.GetByIndex(0) - if nil != err { - logger.Fatalf("volume.liveView.deletedObjectsWrapper.bPlusTree.GetByIndex(0) failed: %v", err) - } - if !ok { - logger.Fatalf("volume.liveView.deletedObjectsWrapper.bPlusTree.GetByIndex(0) returned !ok") - } - - objectNumber, ok = objectNumberAsKey.(uint64) - if !ok { - logger.Fatalf("objectNumberAsKey.(uint64) returned !ok") - } - - containerNameAsByteSlice, ok = containerNameAsValue.([]byte) - if !ok { - logger.Fatalf("containerNameAsValue.([]byte) returned !ok") - } - - delayedObjectDeleteList = append(delayedObjectDeleteList, delayedObjectDeleteStruct{containerName: string(containerNameAsByteSlice[:]), objectNumber: objectNumber}) - - ok, err = volume.liveView.deletedObjectsWrapper.bPlusTree.DeleteByIndex(0) - if nil != err { - logger.Fatalf("volume.liveView.deletedObjectsWrapper.bPlusTree.DeleteByIndex(0) failed: %v", err) - } - if !ok { - logger.Fatalf("volume.liveView.deletedObjectsWrapper.bPlusTree.DeleteByIndex(0) returned !ok") - } - } - - if 0 < len(delayedObjectDeleteList) { - volume.backgroundObjectDeleteWG.Add(1) - go volume.performDelayedObjectDeletes(delayedObjectDeleteList) - } - globals.PutCheckpointObjectCleanupUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - - err = nil - return -} - -func (volume *volumeStruct) performDelayedObjectDeletes(delayedObjectDeleteList []delayedObjectDeleteStruct) { - var ( - delayedObjectDelete delayedObjectDeleteStruct - delayedObjectDeleteName string - err error - ) - -RetryCheckForObjectDeleteEnabled: - globals.backgroundObjectDeleteActiveWG.Add(1) - - globals.backgroundObjectDeleteRWMutex.RLock() - - if !globals.backgroundObjectDeleteEnabled { - // We must block...and retry - globals.backgroundObjectDeleteActiveWG.Done() - globals.backgroundObjectDeleteRWMutex.RUnlock() - globals.backgroundObjectDeleteEnabledWG.Wait() - goto RetryCheckForObjectDeleteEnabled - } - - globals.backgroundObjectDeleteRWMutex.RUnlock() - - for _, delayedObjectDelete = range delayedObjectDeleteList { - delayedObjectDeleteName = utils.Uint64ToHexStr(delayedObjectDelete.objectNumber) - if globals.metadataRecycleBin && (delayedObjectDelete.containerName == volume.checkpointContainerName) { - err = swiftclient.ObjectPost( - volume.accountName, - delayedObjectDelete.containerName, - delayedObjectDeleteName, - globals.metadataRecycleBinHeader) - if nil != err { - logger.Errorf("POST %s/%s/%s failed with err: %v", volume.accountName, delayedObjectDelete.containerName, delayedObjectDeleteName, err) - } - } else { - err = swiftclient.ObjectDelete( - volume.accountName, - delayedObjectDelete.containerName, - delayedObjectDeleteName, - swiftclient.SkipRetry) - if nil != err { - logger.Errorf("DELETE %s/%s/%s failed with err: %v", volume.accountName, delayedObjectDelete.containerName, delayedObjectDeleteName, err) - } - } - } - - volume.backgroundObjectDeleteWG.Done() - globals.backgroundObjectDeleteActiveWG.Done() -} - -func (volume *volumeStruct) openCheckpointChunkedPutContextIfNecessary() (err error) { - var ( - ok bool - ) - - if nil == volume.checkpointChunkedPutContext { - volume.checkpointChunkedPutContextObjectNumber = volume.fetchNonceWhileLocked() - volume.checkpointChunkedPutContext, err = - swiftclient.ObjectFetchChunkedPutContext(volume.accountName, - volume.checkpointContainerName, - utils.Uint64ToHexStr(volume.checkpointChunkedPutContextObjectNumber), - volume.volumeName) - if nil != err { - return - } - if nil != volume.priorView { - if volume.postponePriorViewCreatedObjectsPuts { - _, ok = volume.postponedPriorViewCreatedObjectsPuts[volume.checkpointChunkedPutContextObjectNumber] - if ok { - err = fmt.Errorf("volume.postponedPriorViewCreatedObjectsPuts[volume.checkpointChunkedPutContextObjectNumber] check returned ok") - return - } - volume.postponedPriorViewCreatedObjectsPuts[volume.checkpointChunkedPutContextObjectNumber] = struct{}{} - } else { - ok, err = volume.priorView.createdObjectsWrapper.bPlusTree.Put(volume.checkpointChunkedPutContextObjectNumber, []byte(volume.checkpointContainerName)) - if nil != err { - return - } - if !ok { - err = fmt.Errorf("volume.priorView.createdObjectsWrapper.bPlusTree.Put() returned !ok") - return - } - } - } - } - err = nil - return -} - -func (volume *volumeStruct) bytesPutToCheckpointChunkedPutContext() (bytesPut uint64, err error) { - if nil == volume.checkpointChunkedPutContext { - err = fmt.Errorf("bytesPutToCheckpointChunkedPutContext() called while volume.checkpointChunkedPutContext == nil") - } else { - bytesPut, err = volume.checkpointChunkedPutContext.BytesPut() - } - return // err set as appropriate regardless of path -} - -func (volume *volumeStruct) sendChunkToCheckpointChunkedPutContext(buf []byte) (err error) { - if nil == volume.checkpointChunkedPutContext { - err = fmt.Errorf("sendChunkToCheckpointChunkedPutContext() called while volume.checkpointChunkedPutContext == nil") - } else { - err = volume.checkpointChunkedPutContext.SendChunk(buf) - } - return // err set as appropriate regardless of path -} - -func (volume *volumeStruct) closeCheckpointChunkedPutContextIfNecessary() (err error) { - var ( - bytesPut uint64 - ) - - if nil == volume.checkpointChunkedPutContext { - err = nil - } else { - bytesPut, err = volume.checkpointChunkedPutContext.BytesPut() - if nil == err { - if bytesPut >= volume.maxFlushSize { - err = volume.checkpointChunkedPutContext.Close() - volume.checkpointChunkedPutContext = nil - } - } - } - return // err set as appropriate regardless of path -} - -func (volume *volumeStruct) closeCheckpointChunkedPutContext() (err error) { - if nil == volume.checkpointChunkedPutContext { - err = fmt.Errorf("closeCheckpointChunkedPutContext() called while volume.checkpointChunkedPutContext == nil") - } else { - err = volume.checkpointChunkedPutContext.Close() - volume.checkpointChunkedPutContext = nil - } - return // err set as appropriate regardless of path -} - -// checkpointDaemon periodically and upon request persists a checkpoint/snapshot. -func (volume *volumeStruct) checkpointDaemon() { - var ( - bPlusTreeObjectCacheHitsDelta uint64 - bPlusTreeObjectCacheMissesDelta uint64 - bPlusTreeObjectCacheStats *sortedmap.BPlusTreeCacheStats - checkpointListener VolumeEventListener - checkpointListeners []VolumeEventListener - checkpointRequest *checkpointRequestStruct - checkpointRequesters []*checkpointRequestStruct - createdDeletedObjectsCacheHitsDelta uint64 - createdDeletedObjectsCacheMissesDelta uint64 - createdDeletedObjectsCacheStats *sortedmap.BPlusTreeCacheStats - exitOnCompletion bool - inodeRecCacheHitsDelta uint64 - inodeRecCacheMissesDelta uint64 - inodeRecCacheStats *sortedmap.BPlusTreeCacheStats - logSegmentRecCacheHitsDelta uint64 - logSegmentRecCacheMissesDelta uint64 - logSegmentRecCacheStats *sortedmap.BPlusTreeCacheStats - ) - - for { - select { - case checkpointRequest = <-volume.checkpointRequestChan: - // Explicitly requested checkpoint... use it below - case <-time.After(volume.checkpointInterval): - // Time to automatically do a checkpoint... so dummy up a checkpointRequest - checkpointRequest = &checkpointRequestStruct{exitOnCompletion: false} - checkpointRequest.waitGroup.Add(1) // ...even though we won't be waiting on it... - } - - // measure the time required to perform the checkpoint - startTime := time.Now() - - // measure the time required to get the volume lock for the checkpoint - startTime2 := startTime - - volume.Lock() - globals.DaemonPerCheckpointLockWaitUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - - // measure the time while the lock is held for the checkpoint - startTime2 = time.Now() - - evtlog.Record(evtlog.FormatHeadhunterCheckpointStart, volume.volumeName) - - checkpointRequest.err = volume.putCheckpoint() - - if nil == checkpointRequest.err { - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndSuccess, volume.volumeName) - } else { - // As part of conducting the checkpoint - and depending upon where the early non-nil - // error was reported - it is highly likely that e.g. pages of the B+Trees have been - // marked clean even though either their dirty data has not been successfully posted - // to Swift and/or the Checkpoint Header that points to it has not been successfully - // recorded in Swift. In either case, a subsequent checkpoint may, indeed, appear to - // succeed and quite probably miss some of the references nodes of the B+Trees not - // having made it to Swift... and, yet, wrongly presume all is (now) well. - - // It should also be noted that other activity (e.g. garbage collection of usually - // now unreferenced data) awaiting completion of this checkpoint should not have - // been allowed to proceed. - - // For now, we will instead promptly fail right here thus preventing that subsequent - // checkpoint from masking the data loss. While there are alternatives (e.g. going - // back and marking every node of the B+Trees as being dirty - or at least those that - // were marked clean), such an approach will not be pursued at this time. - - evtlog.Record(evtlog.FormatHeadhunterCheckpointEndFailure, volume.volumeName, checkpointRequest.err.Error()) - logger.FatalfWithError(checkpointRequest.err, "Shutting down to prevent subsequent checkpoints from corrupting Swift") - } - - // Collect any outstanding requests for a checkpoint. - // - // The volume lock has been held since the checkpoint started - // and the volume lock is required to add any metadata that can - // be part of the checkpoint, so any metadata that the - // requesters wanted flushed has been flushed. - checkpointRequesters = make([]*checkpointRequestStruct, 0, 1) - moreRequests := true - for moreRequests { - checkpointRequesters = append(checkpointRequesters, checkpointRequest) - - select { - case checkpointRequest = <-volume.checkpointRequestChan: - default: - moreRequests = false - } - } - - checkpointListeners = make([]VolumeEventListener, 0, len(volume.eventListeners)) - for checkpointListener = range volume.eventListeners { - checkpointListeners = append(checkpointListeners, checkpointListener) - } - - volume.Unlock() - globals.DaemonPerCheckpointLockedUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - - // Measure time spent updating statistics (including lock wait - // time), time spent waking waiters and calling back listeners - // (note that one listener also updates statistics). - startTime2 = time.Now() - - exitOnCompletion = false - for _, checkpointRequest = range checkpointRequesters { - if checkpointRequest.exitOnCompletion { - exitOnCompletion = true - } - checkpointRequest.waitGroup.Done() // Awake the checkpoint requester - } - - for _, checkpointListener = range checkpointListeners { - checkpointListener.CheckpointCompleted() - } - - // Update Global B+Tree Cache stats now - - inodeRecCacheStats = globals.inodeRecCache.Stats() - logSegmentRecCacheStats = globals.logSegmentRecCache.Stats() - bPlusTreeObjectCacheStats = globals.bPlusTreeObjectCache.Stats() - createdDeletedObjectsCacheStats = globals.createdDeletedObjectsCache.Stats() - - globals.Lock() - inodeRecCacheHitsDelta = inodeRecCacheStats.CacheHits - globals.inodeRecCachePriorCacheHits - inodeRecCacheMissesDelta = inodeRecCacheStats.CacheMisses - globals.inodeRecCachePriorCacheMisses - - logSegmentRecCacheHitsDelta = logSegmentRecCacheStats.CacheHits - globals.logSegmentRecCachePriorCacheHits - logSegmentRecCacheMissesDelta = logSegmentRecCacheStats.CacheMisses - globals.logSegmentRecCachePriorCacheMisses - - bPlusTreeObjectCacheHitsDelta = bPlusTreeObjectCacheStats.CacheHits - globals.bPlusTreeObjectCachePriorCacheHits - bPlusTreeObjectCacheMissesDelta = bPlusTreeObjectCacheStats.CacheMisses - globals.bPlusTreeObjectCachePriorCacheMisses - - createdDeletedObjectsCacheHitsDelta = createdDeletedObjectsCacheStats.CacheHits - globals.createdDeletedObjectsCachePriorCacheHits - createdDeletedObjectsCacheMissesDelta = createdDeletedObjectsCacheStats.CacheMisses - globals.createdDeletedObjectsCachePriorCacheMisses - - if 0 != inodeRecCacheHitsDelta { - stats.IncrementOperationsBy(&stats.InodeRecCacheHits, inodeRecCacheHitsDelta) - globals.inodeRecCachePriorCacheHits = inodeRecCacheStats.CacheHits - } - if 0 != inodeRecCacheMissesDelta { - stats.IncrementOperationsBy(&stats.InodeRecCacheMisses, inodeRecCacheMissesDelta) - globals.inodeRecCachePriorCacheMisses = inodeRecCacheStats.CacheMisses - } - - if 0 != logSegmentRecCacheHitsDelta { - stats.IncrementOperationsBy(&stats.LogSegmentRecCacheHits, logSegmentRecCacheHitsDelta) - globals.logSegmentRecCachePriorCacheHits = logSegmentRecCacheStats.CacheHits - } - if 0 != logSegmentRecCacheMissesDelta { - stats.IncrementOperationsBy(&stats.LogSegmentRecCacheMisses, logSegmentRecCacheMissesDelta) - globals.logSegmentRecCachePriorCacheMisses = logSegmentRecCacheStats.CacheMisses - } - - if 0 != bPlusTreeObjectCacheHitsDelta { - stats.IncrementOperationsBy(&stats.BPlusTreeObjectCacheHits, bPlusTreeObjectCacheHitsDelta) - globals.bPlusTreeObjectCachePriorCacheHits = bPlusTreeObjectCacheStats.CacheHits - } - if 0 != bPlusTreeObjectCacheMissesDelta { - stats.IncrementOperationsBy(&stats.BPlusTreeObjectCacheMisses, bPlusTreeObjectCacheMissesDelta) - globals.bPlusTreeObjectCachePriorCacheMisses = bPlusTreeObjectCacheStats.CacheMisses - } - - if 0 != createdDeletedObjectsCacheHitsDelta { - stats.IncrementOperationsBy(&stats.CreatedDeletedObjectsCacheHits, createdDeletedObjectsCacheHitsDelta) - globals.createdDeletedObjectsCachePriorCacheHits = createdDeletedObjectsCacheStats.CacheHits - } - if 0 != createdDeletedObjectsCacheMissesDelta { - stats.IncrementOperationsBy(&stats.CreatedDeletedObjectsCacheMisses, createdDeletedObjectsCacheMissesDelta) - globals.createdDeletedObjectsCachePriorCacheMisses = createdDeletedObjectsCacheStats.CacheMisses - } - - globals.Unlock() - globals.DaemonPerCheckpointStatsUpdateUsec.Add(uint64(time.Since(startTime2) / time.Microsecond)) - globals.DaemonPerCheckpointUsec.Add(uint64(time.Since(startTime) / time.Microsecond)) - - if exitOnCompletion { - return - } - } -} diff --git a/headhunter/config.go b/headhunter/config.go deleted file mode 100644 index cd684bbc..00000000 --- a/headhunter/config.go +++ /dev/null @@ -1,882 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package headhunter - -import ( - "container/list" - "fmt" - "hash/crc64" - "os" - "sync" - "time" - - etcd "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" - - "github.com/NVIDIA/cstruct" - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/etcdclient" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -const ( - firstNonceToProvide = uint64(2) // Must skip useful values: 0 == unassigned and 1 == RootDirInodeNumber -) - -const ( - liveSnapShotID = uint64(0) -) - -const ( - AccountHeaderName = "X-ProxyFS-BiModal" - AccountHeaderNameTranslated = "X-Account-Sysmeta-Proxyfs-Bimodal" - AccountHeaderValue = "true" - CheckpointHeaderName = "X-Container-Meta-Checkpoint" - StoragePolicyHeaderName = "X-Storage-Policy" -) - -const ( - MetadataRecycleBinHeaderName = "X-Object-Meta-Recycle-Bin" - MetadataRecycleBinHeaderValue = "true" -) - -type bPlusTreeTrackerStruct struct { - bPlusTreeLayout sortedmap.LayoutReport -} - -type bPlusTreeWrapperStruct struct { - volumeView *volumeViewStruct - bPlusTree sortedmap.BPlusTree - bPlusTreeTracker *bPlusTreeTrackerStruct // For inodeRecWrapper, logSegmentRecWrapper, & bPlusTreeObjectWrapper: - // only valid for liveView... nil otherwise - // For createdObjectsWrapper & deletedObjectsWrapper: - // all volumeView's share the corresponding one created for liveView - totalPutNodes uint64 // each call to PutNode() increments this - totalPutBytes uint64 // each call to PutNode() adds the buffer size -} - -type volumeViewStruct struct { - volume *volumeStruct - nonce uint64 // supplies strict time-ordering of views regardless of timebase resets - snapShotID uint64 // in the range [1:2^SnapShotIDNumBits-2] - // ID == 0 reserved for the "live" view - // ID == 2^SnapShotIDNumBits-1 reserved for the .snapshot subdir of a dir - snapShotTime time.Time - snapShotName string - inodeRecWrapper *bPlusTreeWrapperStruct - logSegmentRecWrapper *bPlusTreeWrapperStruct - bPlusTreeObjectWrapper *bPlusTreeWrapperStruct - createdObjectsWrapper *bPlusTreeWrapperStruct // if volumeView is the liveView, should be empty - // if volumeView is not the liveView, tracks objects created between this and the next volumeView - deletedObjectsWrapper *bPlusTreeWrapperStruct // if volumeView is the liveView, tracks objects to be deleted at next checkpoint - // if volumeView is not the liveView, tracks objects deleted between this and the next volumeView - // upon object creation: - // if prior volumeView exists: - // add object to that volumeView's createdObjects - // else: - // do nothing - // - // upon object deletion: - // if prior volumeView exists: - // if object in prior volumeView createdObjects: - // remove object from prior volumeView createdObjects - // add object to liveView deletedObjects - // else: - // add object to prior volumeView deletedObjects - // else: - // add object to liveView deletedObjects - // - // upon next checkpoint: - // remove all objects from liveView deletedObjects - // schedule background deletion of those removed objects - // - // upon SnapShot creation: - // take a fresh checkpoint (will drain liveView deletedObjects) - // create a fresh volumeView - // take a fresh checkpoint (will record volumeView in this checkpoint) - // - // upon volumeView deletion: - // if deleted volumeView is oldest: - // discard volumeView createdObjects - // schedule background deletion of volumeView deletedObjects - // discard volumeView deletedObjects - // else: - // createdObjects merged into prior volumeView - // discard volumeView createdObjects - // for each object in deletedObjects: - // if object in prior volumeView createdObjects: - // remove object from prior volumeView createdObjects - // schedule background deletion of object - // else: - // add object to prior volumeView deletedObjects - // discard volumeView deletedObjects - // discard volumeView -} - -type volumeStruct struct { - trackedlock.Mutex - volumeName string - accountName string - maxFlushSize uint64 - nonceValuesToReserve uint16 - maxInodesPerMetadataNode uint64 - maxLogSegmentsPerMetadataNode uint64 - maxDirFileNodesPerMetadataNode uint64 - maxCreatedDeletedObjectsPerMetadataNode uint64 - checkpointEtcdKeyName string - checkpointContainerName string - checkpointContainerStoragePolicy string - checkpointInterval time.Duration - replayLogFileName string // if != "", use replay log to reduce RPO to zero - replayLogFile *os.File // opened on first Put or Delete after checkpoint - // closed/deleted on successful checkpoint - volumeGroup *volumeGroupStruct - served bool - defaultReplayLogWriteBuffer []byte // used for O_DIRECT writes to replay log - checkpointTriggeringEvents uint64 // count of events modifying metadata since last checkpoint - checkpointChunkedPutContext swiftclient.ChunkedPutContext - checkpointChunkedPutContextObjectNumber uint64 // ultimately copied to CheckpointObjectTrailerStructObjectNumber - eventListeners map[VolumeEventListener]struct{} - snapShotIDNumBits uint16 - snapShotIDShift uint64 // e.g. inodeNumber >> snapShotIDNumBits == snapShotID - dotSnapShotDirSnapShotID uint64 // .snapshot/ pseudo directory Inode's snapShotID - snapShotU64NonceMask uint64 // used to mask of snapShotID bits - maxNonce uint64 - nextNonce uint64 - checkpointRequestChan chan *checkpointRequestStruct - checkpointHeader *CheckpointHeaderStruct - checkpointHeaderEtcdRevision int64 - liveView *volumeViewStruct - priorView *volumeViewStruct - postponePriorViewCreatedObjectsPuts bool - postponedPriorViewCreatedObjectsPuts map[uint64]struct{} - viewTreeByNonce sortedmap.LLRBTree // key == volumeViewStruct.Nonce; value == *volumeViewStruct - viewTreeByID sortedmap.LLRBTree // key == volumeViewStruct.ID; value == *volumeViewStruct - viewTreeByTime sortedmap.LLRBTree // key == volumeViewStruct.Time; value == *volumeViewStruct - viewTreeByName sortedmap.LLRBTree // key == volumeViewStruct.Name; value == *volumeViewStruct - availableSnapShotIDList *list.List - backgroundObjectDeleteWG sync.WaitGroup -} - -type volumeGroupStruct struct { - name string - volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName -} - -type globalsStruct struct { - trackedlock.Mutex - - crc64ECMATable *crc64.Table - uint64Size uint64 - ElementOfBPlusTreeLayoutStructSize uint64 - replayLogTransactionFixedPartStructSize uint64 - - inodeRecCache sortedmap.BPlusTreeCache - inodeRecCachePriorCacheHits uint64 - inodeRecCachePriorCacheMisses uint64 - logSegmentRecCache sortedmap.BPlusTreeCache - logSegmentRecCachePriorCacheHits uint64 - logSegmentRecCachePriorCacheMisses uint64 - bPlusTreeObjectCache sortedmap.BPlusTreeCache - bPlusTreeObjectCachePriorCacheHits uint64 - bPlusTreeObjectCachePriorCacheMisses uint64 - createdDeletedObjectsCache sortedmap.BPlusTreeCache - createdDeletedObjectsCachePriorCacheHits uint64 - createdDeletedObjectsCachePriorCacheMisses uint64 - - checkpointHeaderConsensusAttempts uint16 - mountRetryLimit uint16 - mountRetryDelay []time.Duration - - logCheckpointHeaderPosts bool - - etcdEnabled bool - etcdEndpoints []string - etcdAutoSyncInterval time.Duration - etcdCertDir string - etcdDialTimeout time.Duration - etcdOpTimeout time.Duration - - metadataRecycleBin bool - metadataRecycleBinHeader map[string][]string - - etcdClient *etcd.Client - etcdKV etcd.KV - - volumeGroupMap map[string]*volumeGroupStruct // key == volumeGroupStruct.name - volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName - - backgroundObjectDeleteRWMutex trackedlock.RWMutex - backgroundObjectDeleteEnabled bool // used to make {Disable|Enable}ObjectDeletions() idempotent - backgroundObjectDeleteEnabledWG sync.WaitGroup // use to awaken blocked performDelayedObjectDeletes() after EnableObjectDeletions() called - backgroundObjectDeleteActiveWG sync.WaitGroup // used to hold off returning from DisableObjectDeletions() until all deletions cease - - FetchNonceUsec bucketstats.BucketLog2Round - GetInodeRecUsec bucketstats.BucketLog2Round - GetInodeRecBytes bucketstats.BucketLog2Round - PutInodeRecUsec bucketstats.BucketLog2Round - PutInodeRecBytes bucketstats.BucketLog2Round - PutInodeRecsUsec bucketstats.BucketLog2Round - PutInodeRecsBytes bucketstats.BucketLog2Round - DeleteInodeRecUsec bucketstats.BucketLog2Round - IndexedInodeNumberUsec bucketstats.BucketLog2Round - NextInodeNumberUsec bucketstats.BucketLog2Round - GetLogSegmentRecUsec bucketstats.BucketLog2Round - PutLogSegmentRecUsec bucketstats.BucketLog2Round - DeleteLogSegmentRecUsec bucketstats.BucketLog2Round - IndexedLogSegmentNumberUsec bucketstats.BucketLog2Round - GetBPlusTreeObjectUsec bucketstats.BucketLog2Round - GetBPlusTreeObjectBytes bucketstats.BucketLog2Round - PutBPlusTreeObjectUsec bucketstats.BucketLog2Round - PutBPlusTreeObjectBytes bucketstats.BucketLog2Round - DeleteBPlusTreeObjectUsec bucketstats.BucketLog2Round - IndexedBPlusTreeObjectNumberUsec bucketstats.BucketLog2Round - DoCheckpointUsec bucketstats.BucketLog2Round - DaemonPerCheckpointUsec bucketstats.BucketLog2Round - DaemonPerCheckpointLockWaitUsec bucketstats.BucketLog2Round - DaemonPerCheckpointLockedUsec bucketstats.BucketLog2Round - DaemonPerCheckpointStatsUpdateUsec bucketstats.BucketLog2Round - - PutCheckpointUsec bucketstats.BucketLog2Round - PutCheckpointBytes bucketstats.BucketLog2Round - PutCheckpointInodeRecUsec bucketstats.BucketLog2Round - PutCheckpointInodeRecBytes bucketstats.BucketLog2Round - PutCheckpointInodeRecNodes bucketstats.BucketLog2Round - PutCheckpointLogSegmentUsec bucketstats.BucketLog2Round - PutCheckpointLogSegmentBytes bucketstats.BucketLog2Round - PutCheckpointLogSegmentNodes bucketstats.BucketLog2Round - PutCheckpointbPlusTreeObjectUsec bucketstats.BucketLog2Round - PutCheckpointbPlusTreeObjectBytes bucketstats.BucketLog2Round - PutCheckpointbPlusTreeObjectNodes bucketstats.BucketLog2Round - PutCheckpointSnapshotFlushUsec bucketstats.BucketLog2Round - PutCheckpointTreeLayoutUsec bucketstats.BucketLog2Round - PutCheckpointTreeLayoutBytes bucketstats.BucketLog2Round - PutCheckpointCheckpointTrailerBytes bucketstats.BucketLog2Round - PutCheckpointSnapshotListUsec bucketstats.BucketLog2Round - PutCheckpointSnapshotListBytes bucketstats.BucketLog2Round - PutCheckpointChunkedPutUsec bucketstats.BucketLog2Round - PutCheckpointPostAndEtcdUsec bucketstats.BucketLog2Round - PutCheckpointObjectCleanupUsec bucketstats.BucketLog2Round - - FetchLayoutReportUsec bucketstats.BucketLog2Round - SnapShotCreateByInodeLayerUsec bucketstats.BucketLog2Round - SnapShotDeleteByInodeLayerUsec bucketstats.BucketLog2Round - SnapShotCountUsec bucketstats.BucketLog2Round - SnapShotLookupByNameUsec bucketstats.BucketLog2Round - SnapShotListByIDUsec bucketstats.BucketLog2Round - SnapShotListByTimeUsec bucketstats.BucketLog2Round - SnapShotListByNameUsec bucketstats.BucketLog2Round - SnapShotU64DecodeUsec bucketstats.BucketLog2Round - SnapShotIDAndNonceEncodeUsec bucketstats.BucketLog2Round - SnapShotTypeDotSnapShotAndNonceEncodeUsec bucketstats.BucketLog2Round - - GetInodeRecErrors bucketstats.Total - PutInodeRecErrors bucketstats.Total - PutInodeRecsErrors bucketstats.Total - DeleteInodeRecErrors bucketstats.Total - IndexedInodeNumberErrors bucketstats.Total - NextInodeNumberErrors bucketstats.Total - GetLogSegmentRecErrors bucketstats.Total - PutLogSegmentRecErrors bucketstats.Total - DeleteLogSegmentRecErrors bucketstats.Total - IndexedLogSegmentNumberErrors bucketstats.Total - GetBPlusTreeObjectErrors bucketstats.Total - PutBPlusTreeObjectErrors bucketstats.Total - DeleteBPlusTreeObjectErrors bucketstats.Total - IndexedBPlusTreeObjectNumberErrors bucketstats.Total - DoCheckpointErrors bucketstats.Total - PutCheckpointErrors bucketstats.Total - FetchLayoutReportErrors bucketstats.Total - SnapShotCreateByInodeLayerErrors bucketstats.Total - SnapShotDeleteByInodeLayerErrors bucketstats.Total - SnapShotCountErrors bucketstats.Total - SnapShotLookupByNameErrors bucketstats.Total -} - -var globals globalsStruct - -func init() { - transitions.Register("headhunter", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - var ( - bPlusTreeObjectCacheEvictHighLimit uint64 - bPlusTreeObjectCacheEvictLowLimit uint64 - createdDeletedObjectsCacheEvictHighLimit uint64 - createdDeletedObjectsCacheEvictLowLimit uint64 - dummyElementOfBPlusTreeLayoutStruct ElementOfBPlusTreeLayoutStruct - dummyReplayLogTransactionFixedPartStruct replayLogTransactionFixedPartStruct - dummyUint64 uint64 - inodeRecCacheEvictHighLimit uint64 - inodeRecCacheEvictLowLimit uint64 - logSegmentRecCacheEvictHighLimit uint64 - logSegmentRecCacheEvictLowLimit uint64 - mountRetryDelay time.Duration - mountRetryExpBackoff float64 - mountRetryIndex uint16 - nextMountRetryDelay time.Duration - ) - - bucketstats.Register("proxyfs.headhunter", "", &globals) - - // Pre-compute crc64 ECMA Table & useful cstruct sizes - - globals.crc64ECMATable = crc64.MakeTable(crc64.ECMA) - - globals.uint64Size, _, err = cstruct.Examine(dummyUint64) - if nil != err { - return - } - - globals.ElementOfBPlusTreeLayoutStructSize, _, err = cstruct.Examine(dummyElementOfBPlusTreeLayoutStruct) - if nil != err { - return - } - - globals.replayLogTransactionFixedPartStructSize, _, err = cstruct.Examine(dummyReplayLogTransactionFixedPartStruct) - if nil != err { - return - } - - // Initialize globals.volume{|Group}Map's - - globals.volumeGroupMap = make(map[string]*volumeGroupStruct) - globals.volumeMap = make(map[string]*volumeStruct) - - // Initialize B+Tree caches - - inodeRecCacheEvictLowLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "InodeRecCacheEvictLowLimit") - if nil != err { - return - } - inodeRecCacheEvictHighLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "InodeRecCacheEvictHighLimit") - if nil != err { - return - } - - globals.inodeRecCache = sortedmap.NewBPlusTreeCache(inodeRecCacheEvictLowLimit, inodeRecCacheEvictHighLimit) - - globals.inodeRecCachePriorCacheHits = 0 - globals.inodeRecCachePriorCacheMisses = 0 - - logSegmentRecCacheEvictLowLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "LogSegmentRecCacheEvictLowLimit") - if nil != err { - return - } - logSegmentRecCacheEvictHighLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "LogSegmentRecCacheEvictHighLimit") - if nil != err { - return - } - - globals.logSegmentRecCache = sortedmap.NewBPlusTreeCache(logSegmentRecCacheEvictLowLimit, logSegmentRecCacheEvictHighLimit) - - globals.logSegmentRecCachePriorCacheHits = 0 - globals.logSegmentRecCachePriorCacheMisses = 0 - - bPlusTreeObjectCacheEvictLowLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "BPlusTreeObjectCacheEvictLowLimit") - if nil != err { - return - } - bPlusTreeObjectCacheEvictHighLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "BPlusTreeObjectCacheEvictHighLimit") - if nil != err { - return - } - - globals.bPlusTreeObjectCache = sortedmap.NewBPlusTreeCache(bPlusTreeObjectCacheEvictLowLimit, bPlusTreeObjectCacheEvictHighLimit) - - globals.bPlusTreeObjectCachePriorCacheHits = 0 - globals.bPlusTreeObjectCachePriorCacheMisses = 0 - - createdDeletedObjectsCacheEvictLowLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "CreatedDeletedObjectsCacheEvictLowLimit") - if nil != err { - createdDeletedObjectsCacheEvictLowLimit = logSegmentRecCacheEvictLowLimit // TODO: Eventually just return - } - createdDeletedObjectsCacheEvictHighLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "CreatedDeletedObjectsCacheEvictHighLimit") - if nil != err { - createdDeletedObjectsCacheEvictHighLimit = logSegmentRecCacheEvictHighLimit // TODO: Eventually just return - } - - globals.createdDeletedObjectsCache = sortedmap.NewBPlusTreeCache(createdDeletedObjectsCacheEvictLowLimit, createdDeletedObjectsCacheEvictHighLimit) - - globals.createdDeletedObjectsCachePriorCacheHits = 0 - globals.createdDeletedObjectsCachePriorCacheMisses = 0 - - // Record mount retry parameters and compute retry delays - - globals.checkpointHeaderConsensusAttempts, err = confMap.FetchOptionValueUint16("FSGlobals", "CheckpointHeaderConsensusAttempts") - if nil != err { - globals.checkpointHeaderConsensusAttempts = 5 // TODO: Eventually just return - } - - globals.mountRetryLimit, err = confMap.FetchOptionValueUint16("FSGlobals", "MountRetryLimit") - if nil != err { - globals.mountRetryLimit = 6 // TODO: Eventually just return - } - mountRetryDelay, err = confMap.FetchOptionValueDuration("FSGlobals", "MountRetryDelay") - if nil != err { - mountRetryDelay = time.Duration(1 * time.Second) // TODO: Eventually just return - } - mountRetryExpBackoff, err = confMap.FetchOptionValueFloat64("FSGlobals", "MountRetryExpBackoff") - if nil != err { - mountRetryExpBackoff = 2 // TODO: Eventually just return - } - - globals.mountRetryDelay = make([]time.Duration, globals.mountRetryLimit) - - nextMountRetryDelay = mountRetryDelay - - for mountRetryIndex = 0; mountRetryIndex < globals.mountRetryLimit; mountRetryIndex++ { - globals.mountRetryDelay[mountRetryIndex] = nextMountRetryDelay - nextMountRetryDelay = time.Duration(float64(nextMountRetryDelay) * mountRetryExpBackoff) - } - - // Fetch CheckpointHeader logging setting - - globals.logCheckpointHeaderPosts, err = confMap.FetchOptionValueBool("FSGlobals", "LogCheckpointHeaderPosts") - if nil != err { - globals.logCheckpointHeaderPosts = true // TODO: Eventually just return - } - - // Record etcd parameters - - globals.etcdEnabled, err = confMap.FetchOptionValueBool("FSGlobals", "EtcdEnabled") - if nil != err { - globals.etcdEnabled = false // Current default - } - - if globals.etcdEnabled { - globals.etcdEndpoints, err = confMap.FetchOptionValueStringSlice("FSGlobals", "EtcdEndpoints") - if nil != err { - return - } - globals.etcdAutoSyncInterval, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdAutoSyncInterval") - if nil != err { - return - } - globals.etcdCertDir, err = confMap.FetchOptionValueString("FSGlobals", "EtcdCertDir") - if nil != err { - return - } - globals.etcdDialTimeout, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdDialTimeout") - if nil != err { - return - } - globals.etcdOpTimeout, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdOpTimeout") - if nil != err { - return - } - - // Initialize etcd Client & KV objects - - tlsInfo := transport.TLSInfo{ - CertFile: etcdclient.GetCertFilePath(globals.etcdCertDir), - KeyFile: etcdclient.GetKeyFilePath(globals.etcdCertDir), - TrustedCAFile: etcdclient.GetCA(globals.etcdCertDir), - } - - globals.etcdClient, err = etcdclient.New(&tlsInfo, globals.etcdEndpoints, - globals.etcdAutoSyncInterval, globals.etcdDialTimeout) - if nil != err { - return - } - - globals.etcdKV = etcd.NewKV(globals.etcdClient) - } - - // Record MetadataRecycleBin setting - - globals.metadataRecycleBin, err = confMap.FetchOptionValueBool("FSGlobals", "MetadataRecycleBin") - if nil != err { - globals.metadataRecycleBin = false // TODO: Eventually just return or, perhaps, set to true - } - if globals.metadataRecycleBin { - globals.metadataRecycleBinHeader = make(map[string][]string) - globals.metadataRecycleBinHeader[MetadataRecycleBinHeaderName] = []string{MetadataRecycleBinHeaderValue} - } - - // Start off with background object deletions enabled - - globals.backgroundObjectDeleteEnabled = true - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - volumeGroup := &volumeGroupStruct{name: volumeGroupName, volumeMap: make(map[string]*volumeStruct)} - globals.Lock() - _, ok := globals.volumeGroupMap[volumeGroupName] - if ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeGroupCreated() called for preexisting VolumeGroup (%s)", volumeGroupName) - return - } - globals.volumeGroupMap[volumeGroupName] = volumeGroup - globals.Unlock() - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - globals.Lock() - _, ok := globals.volumeGroupMap[volumeGroupName] - if !ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeGroupMoved() called for nonexistent VolumeGroup (%s)", volumeGroupName) - return - } - globals.Unlock() - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - globals.Lock() - volumeGroup, ok := globals.volumeGroupMap[volumeGroupName] - if !ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeGroupDestroyed() called for nonexistent VolumeGroup (%s)", volumeGroupName) - return - } - if 0 != len(volumeGroup.volumeMap) { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeGroupDestroyed() called for non-empty VolumeGroup (%s)", volumeGroupName) - return - } - delete(globals.volumeGroupMap, volumeGroupName) - globals.Unlock() - err = nil - return -} - -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - volume := &volumeStruct{volumeName: volumeName, served: false, checkpointTriggeringEvents: 0} - globals.Lock() - volumeGroup, ok := globals.volumeGroupMap[volumeGroupName] - if !ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeCreated() called for Volume (%s) to be added to nonexistent VolumeGroup (%s)", volumeName, volumeGroupName) - return - } - _, ok = globals.volumeMap[volumeName] - if ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeCreated() called for preexiting Volume (%s)", volumeName) - return - } - _, ok = volumeGroup.volumeMap[volumeName] - if ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeCreated() called for preexiting Volume (%s) to be added to VolumeGroup (%s)", volumeName, volumeGroupName) - return - } - volume.volumeGroup = volumeGroup - volumeGroup.volumeMap[volumeName] = volume - globals.volumeMap[volumeName] = volume - globals.Unlock() - err = nil - return -} - -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - globals.Lock() - volume, ok := globals.volumeMap[volumeName] - if !ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeMoved() called for nonexistent Volume (%s)", volumeName) - return - } - if volume.served { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeMoved() called for Volume (%s) being actively served", volumeName) - return - } - newVolumeGroup, ok := globals.volumeGroupMap[volumeGroupName] - if !ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeMoved() called for Volume (%s) to be moved to nonexistent VolumeGroup (%s)", volumeName, volumeGroupName) - return - } - _, ok = newVolumeGroup.volumeMap[volumeName] - if ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeMoved() called for Volume (%s) to be moved to VolumeGroup (%s) already containing the Volume", volumeName, volumeGroupName) - return - } - oldVolumeGroup := volume.volumeGroup - delete(oldVolumeGroup.volumeMap, volumeName) - newVolumeGroup.volumeMap[volumeName] = volume - volume.volumeGroup = newVolumeGroup - globals.Unlock() - err = nil - return -} - -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - globals.Lock() - volume, ok := globals.volumeMap[volumeName] - if !ok { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeDestroyed() called for nonexistent Volume (%s)", volumeName) - return - } - if volume.served { - globals.Unlock() - err = fmt.Errorf("headhunter.VolumeDestroyed() called for Volume (%s) being actively served", volumeName) - return - } - delete(volume.volumeGroup.volumeMap, volumeName) - delete(globals.volumeMap, volumeName) - globals.Unlock() - err = nil - return -} - -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - globals.Lock() - volume, ok := globals.volumeMap[volumeName] - if !ok { - globals.Unlock() - err = fmt.Errorf("headhunter.ServeVolume() called for nonexistent Volume (%s)", volumeName) - return - } - if volume.served { - globals.Unlock() - err = fmt.Errorf("headhunter.ServeVolume() called for Volume (%s) already being served", volumeName) - return - } - volume.served = true - globals.Unlock() - err = volume.up(confMap) - if nil != err { - err = fmt.Errorf("headhunter.ServeVolume() failed to \"up\" Volume (%s): %v", volumeName, err) - return - } - err = nil - return -} - -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - globals.Lock() - volume, ok := globals.volumeMap[volumeName] - if !ok { - globals.Unlock() - err = fmt.Errorf("headhunter.UnserveVolume() called for nonexistent Volume (%s)", volumeName) - return - } - if !volume.served { - globals.Unlock() - err = fmt.Errorf("headhunter.UnserveVolume() called for Volume (%s) not being served", volumeName) - return - } - volume.served = false - globals.Unlock() - err = volume.down(confMap) - if nil != err { - err = fmt.Errorf("headhunter.UnserveVolume() failed to \"down\" Volume (%s): %v", volumeName, err) - return - } - err = nil - return -} - -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - EnableObjectDeletions() // Otherwise, we will hang - return nil -} -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - return nil -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - if globals.etcdEnabled { - globals.etcdKV = nil - - err = globals.etcdClient.Close() - if nil != err { - return - } - } - - if 0 != len(globals.volumeGroupMap) { - err = fmt.Errorf("headhunter.Down() called with 0 != len(globals.volumeGroupMap") - return - } - if 0 != len(globals.volumeMap) { - err = fmt.Errorf("headhunter.Down() called with 0 != len(globals.volumeMap") - return - } - - bucketstats.UnRegister("proxyfs.headhunter", "") - - err = nil - return -} - -func (volume *volumeStruct) up(confMap conf.ConfMap) (err error) { - var ( - autoFormat bool - autoFormatStringSlice []string - mountRetryIndex uint16 - volumeSectionName string - ) - - volumeSectionName = "Volume:" + volume.volumeName - - volume.checkpointChunkedPutContext = nil - volume.eventListeners = make(map[VolumeEventListener]struct{}) - volume.checkpointRequestChan = make(chan *checkpointRequestStruct, 1) - volume.postponePriorViewCreatedObjectsPuts = false - volume.postponedPriorViewCreatedObjectsPuts = make(map[uint64]struct{}) - - volume.accountName, err = confMap.FetchOptionValueString(volumeSectionName, "AccountName") - if nil != err { - return - } - - volume.maxFlushSize, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxFlushSize") - if nil != err { - return - } - - volume.nonceValuesToReserve, err = confMap.FetchOptionValueUint16(volumeSectionName, "NonceValuesToReserve") - if nil != err { - return - } - - volume.maxInodesPerMetadataNode, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxInodesPerMetadataNode") - if nil != err { - return - } - - volume.maxLogSegmentsPerMetadataNode, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxLogSegmentsPerMetadataNode") - if nil != err { - return - } - - volume.maxDirFileNodesPerMetadataNode, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxDirFileNodesPerMetadataNode") - if nil != err { - return - } - - volume.maxCreatedDeletedObjectsPerMetadataNode, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxCreatedDeletedObjectsPerMetadataNode") - if nil != err { - volume.maxCreatedDeletedObjectsPerMetadataNode = volume.maxLogSegmentsPerMetadataNode // TODO: Eventually just return - } - - if globals.etcdEnabled { - volume.checkpointEtcdKeyName, err = confMap.FetchOptionValueString(volumeSectionName, "CheckpointEtcdKeyName") - if nil != err { - return - } - } - - volume.checkpointContainerName, err = confMap.FetchOptionValueString(volumeSectionName, "CheckpointContainerName") - if nil != err { - return - } - - volume.checkpointContainerStoragePolicy, err = confMap.FetchOptionValueString(volumeSectionName, "CheckpointContainerStoragePolicy") - if nil != err { - return - } - - volume.checkpointInterval, err = confMap.FetchOptionValueDuration(volumeSectionName, "CheckpointInterval") - if nil != err { - return - } - - volume.replayLogFileName, err = confMap.FetchOptionValueString(volumeSectionName, "ReplayLogFileName") - if nil == err { - // Provision aligned buffer used to write to Replay Log - volume.defaultReplayLogWriteBuffer = constructReplayLogWriteBuffer(replayLogWriteBufferDefaultSize) - } else { - // Disable Replay Log - volume.replayLogFileName = "" - } - - volume.snapShotIDNumBits, err = confMap.FetchOptionValueUint16(volumeSectionName, "SnapShotIDNumBits") - if nil != err { - volume.snapShotIDNumBits = 10 // TODO: Eventually just return - } - if 2 > volume.snapShotIDNumBits { - err = fmt.Errorf("[%v]SnapShotIDNumBits must be at least 2", volumeSectionName) - return - } - if 32 < volume.snapShotIDNumBits { - err = fmt.Errorf("[%v]SnapShotIDNumBits must be no more than 32", volumeSectionName) - return - } - - autoFormatStringSlice, err = confMap.FetchOptionValueStringSlice(volumeSectionName, "AutoFormat") - if nil == err { - if 1 != len(autoFormatStringSlice) { - err = fmt.Errorf("If specified, [%v]AutoFormat must be single-valued and either true or false", volumeSectionName) - return - } - autoFormat, err = confMap.FetchOptionValueBool(volumeSectionName, "AutoFormat") - if nil != err { - return - } - } else { - autoFormat = false // Default to false if not present - } - - err = volume.getCheckpoint(autoFormat) - if nil != err { - logger.Warnf("Initial attempt to mount Volume %s failed: %v", volume.volumeName, err) - mountRetryIndex = 0 - for { - if mountRetryIndex == globals.mountRetryLimit { - err = fmt.Errorf("MountRetryLimit (%d) for Volume %s exceeded", globals.mountRetryLimit, volume.volumeName) - return - } - - time.Sleep(globals.mountRetryDelay[mountRetryIndex]) - - err = volume.getCheckpoint(autoFormat) - if nil == err { - break - } - - mountRetryIndex++ // Pre-increment to identify first retry as #1 in following logger.Warnf() call - - logger.Warnf("Mount Retry #%d failed: %v", mountRetryIndex, err) - } - } - - go volume.checkpointDaemon() - - err = nil - return -} - -func (volume *volumeStruct) down(confMap conf.ConfMap) (err error) { - var ( - checkpointRequest checkpointRequestStruct - ) - - checkpointRequest.exitOnCompletion = true - checkpointRequest.waitGroup.Add(1) - - volume.checkpointRequestChan <- &checkpointRequest - - checkpointRequest.waitGroup.Wait() - - err = checkpointRequest.err - - volume.backgroundObjectDeleteWG.Wait() - - return -} diff --git a/headhunter/sortedmap_helper.go b/headhunter/sortedmap_helper.go deleted file mode 100644 index 7b447afc..00000000 --- a/headhunter/sortedmap_helper.go +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package headhunter - -import ( - "fmt" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/utils" -) - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsUint64, ok := key.(uint64) - if !ok { - err = fmt.Errorf("headhunter.bPlusTreeWrapper.DumpKey() could not parse key as a uint64") - return - } - - keyAsString = fmt.Sprintf("0x%016X", keyAsUint64) - - err = nil - return -} - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - valueAsByteSlice, ok := value.([]byte) - if !ok { - err = fmt.Errorf("headhunter.bPlusTreeWrapper.DumpValue() count not parse value as []byte") - return - } - - if 0 == len(valueAsByteSlice) { - valueAsString = "" - err = nil - return - } - - valueAsHexDigitByteSlice := make([]byte, 2+(3*(len(valueAsByteSlice)-1))) - - for i, u8 := range valueAsByteSlice { - if i == 0 { - valueAsHexDigitByteSlice[0] = utils.ByteToHexDigit(u8 >> 4) - valueAsHexDigitByteSlice[1] = utils.ByteToHexDigit(u8 & 0x0F) - } else { - valueAsHexDigitByteSlice[(3*i)-1] = ' ' - valueAsHexDigitByteSlice[(3*i)+0] = utils.ByteToHexDigit(u8 >> 4) - valueAsHexDigitByteSlice[(3*i)+1] = utils.ByteToHexDigit(u8 & 0x0F) - } - } - - valueAsString = string(valueAsHexDigitByteSlice[:]) - - err = nil - return -} - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { - stats.IncrementOperations(&stats.HeadhunterBPlusTreeNodeFaults) - evtlog.Record(evtlog.FormatHeadhunterBPlusTreeNodeFault, bPlusTreeWrapper.volumeView.volume.volumeName, objectNumber, objectOffset, objectLength) - - nodeByteSlice, err = - swiftclient.ObjectGet( - bPlusTreeWrapper.volumeView.volume.accountName, - bPlusTreeWrapper.volumeView.volume.checkpointContainerName, - utils.Uint64ToHexStr(objectNumber), - objectOffset, - objectLength) - - return -} - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { - var ( - bytesUsed uint64 - ok bool - ) - - // track data written - bPlusTreeWrapper.totalPutNodes++ - bPlusTreeWrapper.totalPutBytes += uint64(len(nodeByteSlice)) - - err = bPlusTreeWrapper.volumeView.volume.openCheckpointChunkedPutContextIfNecessary() - if nil != err { - return - } - - objectNumber = bPlusTreeWrapper.volumeView.volume.checkpointChunkedPutContextObjectNumber - - objectOffset, err = bPlusTreeWrapper.volumeView.volume.bytesPutToCheckpointChunkedPutContext() - if nil != err { - return - } - - err = bPlusTreeWrapper.volumeView.volume.sendChunkToCheckpointChunkedPutContext(nodeByteSlice) - if nil != err { - return - } - - if nil != bPlusTreeWrapper.bPlusTreeTracker { - bytesUsed, ok = bPlusTreeWrapper.bPlusTreeTracker.bPlusTreeLayout[objectNumber] - - if ok { - bytesUsed += uint64(len(nodeByteSlice)) - } else { - bytesUsed = uint64(len(nodeByteSlice)) - } - - bPlusTreeWrapper.bPlusTreeTracker.bPlusTreeLayout[objectNumber] = bytesUsed - } - - err = bPlusTreeWrapper.volumeView.volume.closeCheckpointChunkedPutContextIfNecessary() - - return // err set as appropriate -} - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { - var ( - bytesUsed uint64 - ok bool - ) - - if nil != bPlusTreeWrapper.bPlusTreeTracker { - bytesUsed, ok = bPlusTreeWrapper.bPlusTreeTracker.bPlusTreeLayout[objectNumber] - - if ok { - if objectLength > bytesUsed { - err = fmt.Errorf("Logic error: bPlusTreeWrapper.DiscardNode() called referencing too many bytes (0x%016X) in objectNumber 0x%016X", objectLength, objectNumber) - logger.ErrorfWithError(err, "disk corruption or logic error [case 1]") - } else { - // Decrement bytesUsed... leaving a zero value to indicate object may be deleted at end of checkpoint - bytesUsed -= objectLength - bPlusTreeWrapper.bPlusTreeTracker.bPlusTreeLayout[objectNumber] = bytesUsed - err = nil - } - } else { - err = fmt.Errorf("Logic error: bPlusTreeWrapper.DiscardNode() called referencing bytes (0x%016X) in unreferenced objectNumber 0x%016X", objectLength, objectNumber) - logger.ErrorfWithError(err, "disk corruption or logic error [case 2]") - } - } - - return // err set as appropriate regardless of path -} - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { - keyAsUint64, ok := key.(uint64) - if !ok { - err = fmt.Errorf("*bPlusTreeWrapper.PackKey(key == %v) failed to convert key to uint64", key) - return - } - packedKey = utils.Uint64ToByteSlice(keyAsUint64) - err = nil - return -} - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { - if 8 > len(payloadData) { - err = fmt.Errorf("*bPlusTreeWrapper.UnpackKey(payloadData) failed - len(payloadData) must be atleast 8 (was %v)", len(payloadData)) - return - } - keyAsUint64, ok := utils.ByteSliceToUint64(payloadData[:8]) - if !ok { - err = fmt.Errorf("*bPlusTreeWrapper.UnpackKey(payloadData) failed in call to utils.ByteSliceToUint64()") - return - } - key = keyAsUint64 - bytesConsumed = 8 - err = nil - return -} - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { - valueAsByteSlice, ok := value.([]byte) - if !ok { - err = fmt.Errorf("*bPlusTreeWrapper.PackValue() failed - value isn't a []byte") - return - } - packedValue = utils.Uint64ToByteSlice(uint64(len(valueAsByteSlice))) - packedValue = append(packedValue, valueAsByteSlice...) - err = nil - return -} - -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { - if 8 > len(payloadData) { - err = fmt.Errorf("*bPlusTreeWrapper.UnpackValue(payloadData) failed - len(payloadData) must be atleast 8 (was %v)", len(payloadData)) - return - } - valueSize, ok := utils.ByteSliceToUint64(payloadData[:8]) - if !ok { - err = fmt.Errorf("*bPlusTreeWrapper.UnpackValue(payloadData) failed in call to utils.ByteSliceToUint64()") - return - } - valueAsByteSlice := make([]byte, valueSize) - if 0 < valueSize { - copy(valueAsByteSlice, payloadData[8:(8+valueSize)]) - } - value = valueAsByteSlice - bytesConsumed = 8 + valueSize - err = nil - return -} - -// ClearCounters clears the counters that track the number of bytes and number -// of nodes passed to PutNode(). -// -func (bPlusTreeWrapper *bPlusTreeWrapperStruct) ClearCounters() { - - bPlusTreeWrapper.totalPutNodes = 0 - bPlusTreeWrapper.totalPutBytes = 0 -} diff --git a/headhunter/stress_test.go b/headhunter/stress_test.go deleted file mode 100644 index 76ebe3f6..00000000 --- a/headhunter/stress_test.go +++ /dev/null @@ -1,746 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package headhunter - -import ( - "bytes" - cryptoRand "crypto/rand" - "math/big" - mathRand "math/rand" - "sync" - "testing" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/ramswift" - "github.com/NVIDIA/proxyfs/transitions" -) - -const ( - testNumFetchNonceCalls = 1000 - - testNumInodeRecs = 1000 - testMaxInodeRecSize = 100 - testMaxNewInodesBetweenRePutInodes = 5 - testMaxPutInodeRecCallsBetweenPutInodeRecsCalls = 5 - testMaxInodesPerPutInodeRecsCall = 5 - - testNumLogSegmentRecs = 1000 - testMaxLogSegmentRecSize = 20 - - testNumBPlusTreeObjects = 1000 - testMaxBPlusTreeObjectSize = 1000 - - testMaxPutDeleteCallsPerGet = 5 - - testMaxPutDeleteCallsPerCheckpoint = 10 - - testMaxCheckpointsPerDownUp = 5 - - testPseudoRandom = false - testPseudoRandomSeed = int64(0) -) - -var testMathRandSource *mathRand.Rand // A source for pseudo-random numbers (if selected) - -func testRandU64FLessThanN(t *testing.T, n uint64) (r uint64) { - var ( - bigN *big.Int - bigR *big.Int - err error - ) - - if testPseudoRandom { - if nil == testMathRandSource { - testMathRandSource = mathRand.New(mathRand.NewSource(testPseudoRandomSeed)) - } - r = uint64(testMathRandSource.Int63n(int64(n))) - } else { - bigN = big.NewInt(int64(n)) - bigR, err = cryptoRand.Int(cryptoRand.Reader, bigN) - if nil != err { - t.Fatalf("cryptoRand.Int(cryptoRand.Reader, bigN) returned error: %v", err) - } - r = bigR.Uint64() - } - - return -} - -func testRandByteSlice(t *testing.T, maxLen uint64) (slice []byte) { - var ( - i uint64 - sliceLen uint64 - ) - - sliceLen = testRandU64FLessThanN(t, maxLen) + 1 - - slice = make([]byte, sliceLen) - - for i = uint64(0); i < sliceLen; i++ { - slice[i] = byte(testRandU64FLessThanN(t, uint64(0x100))) - } - - return -} - -func testKnuthShuffledU64Slice(t *testing.T, n uint64) (u64Slice []uint64) { - var ( - i uint64 - swapFrom uint64 - swapTo uint64 - ) - - u64Slice = make([]uint64, n) - for i = uint64(0); i < n; i++ { - u64Slice[i] = i - } - - for swapFrom = uint64(n - 1); swapFrom > uint64(0); swapFrom-- { - swapTo = testRandU64FLessThanN(t, swapFrom) - u64Slice[swapFrom], u64Slice[swapTo] = u64Slice[swapTo], u64Slice[swapFrom] - } - - return -} - -func TestHeadHunterStress(t *testing.T) { - var ( - bPlusTreeObjectMap map[uint64][]byte - bPlusTreeObjectNumbers []uint64 - checkpointsSinceDownUp uint64 - checkpointsSinceDownUpCap uint64 - confMap conf.ConfMap - confStrings []string - deleteIndex uint64 - deleteKey uint64 - doneChan chan bool - err error - fetchNonceIndex uint64 - getIndex uint64 - getKey uint64 - inodeMap map[uint64][]byte - inodeNumbers []uint64 - logSegmentMap map[uint64][]byte - logSegmentNumbers []uint64 - mapKey uint64 - newInodesSinceRePutInode uint64 - newInodesSinceRePutInodeCap uint64 - numInodeRecs uint64 - ok bool - putDeleteCallsSinceCheckpoint uint64 - putDeleteCallsSinceCheckpointCap uint64 - putDeleteCallsSinceGet uint64 - putDeleteCallsSinceGetCap uint64 - putIndex uint64 - putIndexPrev uint64 - putInodeRecCallsSincePutInodeRecsCall uint64 - putInodeRecCallsSincePutInodeRecsCallCap uint64 - putInodeRecsIndex uint64 - putKey uint64 - putKeys []uint64 - putValues [][]byte - signalHandlerIsArmedWG sync.WaitGroup - slice []byte - volumeHandle VolumeHandle - ) - - confStrings = []string{ - "Logging.LogFilePath=/dev/null", - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=9999", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=0", - "SwiftClient.RetryLimitObject=0", - "SwiftClient.RetryDelay=1s", - "SwiftClient.RetryDelayObject=1s", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=64", - "SwiftClient.NonChunkedConnectionPoolSize=32", - "Cluster.WhoAmI=Peer0", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - "Volume:TestVolume.AccountName=TestAccount", - "Volume:TestVolume.CheckpointContainerName=.__checkpoint__", - "Volume:TestVolume.CheckpointContainerStoragePolicy=gold", - "Volume:TestVolume.CheckpointInterval=10h", // We never want a time-based checkpoint - "Volume:TestVolume.MaxFlushSize=10000000", - "Volume:TestVolume.NonceValuesToReserve=1", // We want to force worst-case nonce fetching - "Volume:TestVolume.MaxInodesPerMetadataNode=32", - "Volume:TestVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:TestVolume.MaxDirFileNodesPerMetadataNode=16", - "VolumeGroup:TestVolumeGroup.VolumeList=TestVolume", - "VolumeGroup:TestVolumeGroup.VirtualIPAddr=", - "VolumeGroup:TestVolumeGroup.PrimaryPeer=Peer0", - "FSGlobals.VolumeGroupList=TestVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - } - - // Launch a ramswift instance - - signalHandlerIsArmedWG.Add(1) - doneChan = make(chan bool, 1) // Must be buffered to avoid race - - go ramswift.Daemon("/dev/null", confStrings, &signalHandlerIsArmedWG, doneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - confMap, err = conf.MakeConfMapFromStrings(confStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned error: %v", err) - } - - // Schedule a Format of TestVolume on first Up() - - err = confMap.UpdateFromString("Volume:TestVolume.AutoFormat=true") - if nil != err { - t.Fatalf("conf.UpdateFromString(\"Volume:TestVolume.AutoFormat=true\") returned error: %v", err) - } - - // Up packages (TestVolume will be formatted) - - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - - // Unset AutoFormat for all subsequent uses of ConfMap - - err = confMap.UpdateFromString("Volume:TestVolume.AutoFormat=false") - if nil != err { - t.Fatalf("conf.UpdateFromString(\"Volume:TestVolume.AutoFormat=false\") returned error: %v", err) - } - - // Fetch a volumeHandle to use - - volumeHandle, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("headhunter.FetchVolumeHandle(\"TestVolume\") returned error: %v", err) - } - - // Stress FetchNonce() - - for fetchNonceIndex = uint64(0); fetchNonceIndex < testNumFetchNonceCalls; fetchNonceIndex++ { - _ = volumeHandle.FetchNonce() - } - - // Stress *InodeRec[|s]() - - inodeMap = make(map[uint64][]byte) - - for mapKey = uint64(0); mapKey < testNumInodeRecs; mapKey++ { - inodeMap[mapKey] = testRandByteSlice(t, testMaxInodeRecSize) - } - - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - newInodesSinceRePutInodeCap = testRandU64FLessThanN(t, testMaxNewInodesBetweenRePutInodes) + 1 - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - putInodeRecCallsSincePutInodeRecsCallCap = testRandU64FLessThanN(t, testMaxPutInodeRecCallsBetweenPutInodeRecsCalls) + 1 - - checkpointsSinceDownUp = 0 - newInodesSinceRePutInode = 0 - putDeleteCallsSinceCheckpoint = 0 - putDeleteCallsSinceGet = 0 - putInodeRecCallsSincePutInodeRecsCall = 0 - - inodeNumbers = testKnuthShuffledU64Slice(t, testNumInodeRecs) - - putIndex = 0 - - for putIndex < testNumInodeRecs { - putIndexPrev = putIndex - - if putInodeRecCallsSincePutInodeRecsCall == putInodeRecCallsSincePutInodeRecsCallCap { - numInodeRecs = testRandU64FLessThanN(t, testMaxInodesPerPutInodeRecsCall) + 1 - if (putIndex + numInodeRecs) > testNumInodeRecs { - numInodeRecs = testNumInodeRecs - putIndex - } - putKeys = make([]uint64, numInodeRecs) - putValues = make([][]byte, numInodeRecs) - for putInodeRecsIndex = 0; putInodeRecsIndex < numInodeRecs; putInodeRecsIndex++ { - putKey = inodeNumbers[putIndex] - putKeys[putInodeRecsIndex] = putKey - putValues[putInodeRecsIndex] = inodeMap[putKey] - putIndex++ - } - err = volumeHandle.PutInodeRecs(putKeys, putValues) - if nil != err { - t.Fatalf("headhunter.PutInodeRecs() returned error: %v", err) - } - putInodeRecCallsSincePutInodeRecsCallCap = testRandU64FLessThanN(t, testMaxPutInodeRecCallsBetweenPutInodeRecsCalls) + 1 - putInodeRecCallsSincePutInodeRecsCall = 0 - } else { - putKey = inodeNumbers[putIndex] - err = volumeHandle.PutInodeRec(putKey, inodeMap[putKey]) - if nil != err { - t.Fatalf("headhunter.PutInodeRec() returned error: %v", err) - } - putInodeRecCallsSincePutInodeRecsCall++ - putIndex++ - } - - putDeleteCallsSinceGet++ - if putDeleteCallsSinceGet == putDeleteCallsSinceGetCap { - getIndex = testRandU64FLessThanN(t, putIndex) - getKey = inodeNumbers[getIndex] - slice, ok, err = volumeHandle.GetInodeRec(getKey) - if nil != err { - t.Fatalf("headhunter.GetInodeRec() returned error: %v", err) - } - if !ok { - t.Fatalf("headhunter.GetInodeRec() returned !ok") - } - if 0 != bytes.Compare(slice, inodeMap[getKey]) { - t.Fatalf("headhunter.GetInodeRec() returned unexpected slice") - } - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - putDeleteCallsSinceGet = 0 - } - - putDeleteCallsSinceCheckpoint++ - if putDeleteCallsSinceCheckpoint == putDeleteCallsSinceCheckpointCap { - err = volumeHandle.DoCheckpoint() - if nil != err { - t.Fatalf("headhunter.DoCheckpoint() returned error: %v", err) - } - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceCheckpoint = 0 - - checkpointsSinceDownUp++ - if checkpointsSinceDownUp == checkpointsSinceDownUpCap { - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - volumeHandle, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("headhunter.FetchVolumeHandle(\"TestVolume\") returned error: %v", err) - } - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - checkpointsSinceDownUp = 0 - } - } - - newInodesSinceRePutInode++ - if newInodesSinceRePutInode == newInodesSinceRePutInodeCap { - for putIndex > putIndexPrev { - putIndex-- - inodeMap[mapKey] = testRandByteSlice(t, testMaxInodeRecSize) - } - newInodesSinceRePutInodeCap = testRandU64FLessThanN(t, testMaxNewInodesBetweenRePutInodes) + 1 - newInodesSinceRePutInode = 0 - } - } - - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - - checkpointsSinceDownUp = 0 - putDeleteCallsSinceCheckpoint = 0 - putDeleteCallsSinceGet = 0 - - inodeNumbers = testKnuthShuffledU64Slice(t, testNumInodeRecs) - - deleteIndex = 0 - - for deleteIndex < testNumInodeRecs { - deleteKey = inodeNumbers[deleteIndex] - err = volumeHandle.DeleteInodeRec(deleteKey) - if nil != err { - t.Fatalf("headhunter.DeleteInodeRec() returned error: %v", err) - } - - deleteIndex++ - - if deleteIndex < testNumInodeRecs { - putDeleteCallsSinceGet++ - if putDeleteCallsSinceGet == putDeleteCallsSinceGetCap { - getIndex = testRandU64FLessThanN(t, testNumInodeRecs-deleteIndex) + deleteIndex - getKey = inodeNumbers[getIndex] - slice, ok, err = volumeHandle.GetInodeRec(getKey) - if nil != err { - t.Fatalf("headhunter.GetInodeRec() returned error: %v", err) - } - if !ok { - t.Fatalf("headhunter.GetInodeRec() returned !ok") - } - if 0 != bytes.Compare(slice, inodeMap[getKey]) { - t.Fatalf("headhunter.GetInodeRec() returned unexpected slice") - } - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - putDeleteCallsSinceGet = 0 - } - } - - putDeleteCallsSinceCheckpoint++ - if putDeleteCallsSinceCheckpoint == putDeleteCallsSinceCheckpointCap { - err = volumeHandle.DoCheckpoint() - if nil != err { - t.Fatalf("headhunter.DoCheckpoint() returned error: %v", err) - } - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceCheckpoint = 0 - - checkpointsSinceDownUp++ - if checkpointsSinceDownUp == checkpointsSinceDownUpCap { - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - volumeHandle, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("headhunter.FetchVolumeHandle(\"TestVolume\") returned error: %v", err) - } - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - checkpointsSinceDownUp = 0 - } - } - } - - // Stress *LogSegmentRec() - - logSegmentMap = make(map[uint64][]byte) - - for mapKey = uint64(0); mapKey < testNumLogSegmentRecs; mapKey++ { - logSegmentMap[mapKey] = testRandByteSlice(t, testMaxLogSegmentRecSize) - } - - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - - checkpointsSinceDownUp = 0 - putDeleteCallsSinceCheckpoint = 0 - putDeleteCallsSinceGet = 0 - - logSegmentNumbers = testKnuthShuffledU64Slice(t, testNumLogSegmentRecs) - - putIndex = 0 - - for putIndex < testNumLogSegmentRecs { - putKey = logSegmentNumbers[putIndex] - err = volumeHandle.PutLogSegmentRec(putKey, logSegmentMap[putKey]) - if nil != err { - t.Fatalf("headhunter.PutLogSegmentRec() returned error: %v", err) - } - - putIndex++ - - putDeleteCallsSinceGet++ - if putDeleteCallsSinceGet == putDeleteCallsSinceGetCap { - getIndex = testRandU64FLessThanN(t, putIndex) - getKey = logSegmentNumbers[getIndex] - slice, err = volumeHandle.GetLogSegmentRec(getKey) - if nil != err { - t.Fatalf("headhunter.GetLogSegmentRec() returned error: %v", err) - } - if 0 != bytes.Compare(slice, logSegmentMap[getKey]) { - t.Fatalf("headhunter.GetLogSegmentRec() returned unexpected slice") - } - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - putDeleteCallsSinceGet = 0 - } - - putDeleteCallsSinceCheckpoint++ - if putDeleteCallsSinceCheckpoint == putDeleteCallsSinceCheckpointCap { - err = volumeHandle.DoCheckpoint() - if nil != err { - t.Fatalf("headhunter.DoCheckpoint() returned error: %v", err) - } - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceCheckpoint = 0 - - checkpointsSinceDownUp++ - if checkpointsSinceDownUp == checkpointsSinceDownUpCap { - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - volumeHandle, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("headhunter.FetchVolumeHandle(\"TestVolume\") returned error: %v", err) - } - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - checkpointsSinceDownUp = 0 - } - } - } - - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - - checkpointsSinceDownUp = 0 - putDeleteCallsSinceCheckpoint = 0 - putDeleteCallsSinceGet = 0 - - logSegmentNumbers = testKnuthShuffledU64Slice(t, testNumLogSegmentRecs) - - deleteIndex = 0 - - for deleteIndex < testNumLogSegmentRecs { - deleteKey = logSegmentNumbers[deleteIndex] - err = volumeHandle.DeleteLogSegmentRec(deleteKey) - if nil != err { - t.Fatalf("headhunter.DeleteLogSegmentRec() returned error: %v", err) - } - - deleteIndex++ - - if deleteIndex < testNumLogSegmentRecs { - putDeleteCallsSinceGet++ - if putDeleteCallsSinceGet == putDeleteCallsSinceGetCap { - getIndex = testRandU64FLessThanN(t, testNumLogSegmentRecs-deleteIndex) + deleteIndex - getKey = logSegmentNumbers[getIndex] - slice, err = volumeHandle.GetLogSegmentRec(getKey) - if nil != err { - t.Fatalf("headhunter.GetLogSegmentRec() returned error: %v", err) - } - if 0 != bytes.Compare(slice, logSegmentMap[getKey]) { - t.Fatalf("headhunter.GetLogSegmentRec() returned unexpected slice") - } - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - putDeleteCallsSinceGet = 0 - } - } - - putDeleteCallsSinceCheckpoint++ - if putDeleteCallsSinceCheckpoint == putDeleteCallsSinceCheckpointCap { - err = volumeHandle.DoCheckpoint() - if nil != err { - t.Fatalf("headhunter.DoCheckpoint() returned error: %v", err) - } - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceCheckpoint = 0 - - checkpointsSinceDownUp++ - if checkpointsSinceDownUp == checkpointsSinceDownUpCap { - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - volumeHandle, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("headhunter.FetchVolumeHandle(\"TestVolume\") returned error: %v", err) - } - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - checkpointsSinceDownUp = 0 - } - } - } - - // Stress *BPlusTreeObject() - - bPlusTreeObjectMap = make(map[uint64][]byte) - - for mapKey = uint64(0); mapKey < testNumBPlusTreeObjects; mapKey++ { - bPlusTreeObjectMap[mapKey] = testRandByteSlice(t, testMaxBPlusTreeObjectSize) - } - - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - - checkpointsSinceDownUp = 0 - putDeleteCallsSinceCheckpoint = 0 - putDeleteCallsSinceGet = 0 - - bPlusTreeObjectNumbers = testKnuthShuffledU64Slice(t, testNumBPlusTreeObjects) - - putIndex = 0 - - for putIndex < testNumBPlusTreeObjects { - putKey = bPlusTreeObjectNumbers[putIndex] - err = volumeHandle.PutBPlusTreeObject(putKey, bPlusTreeObjectMap[putKey]) - if nil != err { - t.Fatalf("headhunter.PutBPlusTreeObject() returned error: %v", err) - } - - putIndex++ - - putDeleteCallsSinceGet++ - if putDeleteCallsSinceGet == putDeleteCallsSinceGetCap { - getIndex = testRandU64FLessThanN(t, putIndex) - getKey = bPlusTreeObjectNumbers[getIndex] - slice, err = volumeHandle.GetBPlusTreeObject(getKey) - if nil != err { - t.Fatalf("headhunter.GetBPlusTreeObject() returned error: %v", err) - } - if 0 != bytes.Compare(slice, bPlusTreeObjectMap[getKey]) { - t.Fatalf("headhunter.GetBPlusTreeObject() returned unexpected slice") - } - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - putDeleteCallsSinceGet = 0 - } - - putDeleteCallsSinceCheckpoint++ - if putDeleteCallsSinceCheckpoint == putDeleteCallsSinceCheckpointCap { - err = volumeHandle.DoCheckpoint() - if nil != err { - t.Fatalf("headhunter.DoCheckpoint() returned error: %v", err) - } - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceCheckpoint = 0 - - checkpointsSinceDownUp++ - if checkpointsSinceDownUp == checkpointsSinceDownUpCap { - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - volumeHandle, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("headhunter.FetchVolumeHandle(\"TestVolume\") returned error: %v", err) - } - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - checkpointsSinceDownUp = 0 - } - } - } - - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - - checkpointsSinceDownUp = 0 - putDeleteCallsSinceCheckpoint = 0 - putDeleteCallsSinceGet = 0 - - bPlusTreeObjectNumbers = testKnuthShuffledU64Slice(t, testNumBPlusTreeObjects) - - deleteIndex = 0 - - for deleteIndex < testNumBPlusTreeObjects { - deleteKey = bPlusTreeObjectNumbers[deleteIndex] - err = volumeHandle.DeleteBPlusTreeObject(deleteKey) - if nil != err { - t.Fatalf("headhunter.DeleteBPlusTreeObject() returned error: %v", err) - } - - deleteIndex++ - - if deleteIndex < testNumBPlusTreeObjects { - putDeleteCallsSinceGet++ - if putDeleteCallsSinceGet == putDeleteCallsSinceGetCap { - getIndex = testRandU64FLessThanN(t, testNumBPlusTreeObjects-deleteIndex) + deleteIndex - getKey = bPlusTreeObjectNumbers[getIndex] - slice, err = volumeHandle.GetBPlusTreeObject(getKey) - if nil != err { - t.Fatalf("headhunter.GetBPlusTreeObject() returned error: %v", err) - } - if 0 != bytes.Compare(slice, bPlusTreeObjectMap[getKey]) { - t.Fatalf("headhunter.GetBPlusTreeObject() returned unexpected slice") - } - putDeleteCallsSinceGetCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerGet) + 1 - putDeleteCallsSinceGet = 0 - } - } - - putDeleteCallsSinceCheckpoint++ - if putDeleteCallsSinceCheckpoint == putDeleteCallsSinceCheckpointCap { - err = volumeHandle.DoCheckpoint() - if nil != err { - t.Fatalf("headhunter.DoCheckpoint() returned error: %v", err) - } - putDeleteCallsSinceCheckpointCap = testRandU64FLessThanN(t, testMaxPutDeleteCallsPerCheckpoint) + 1 - putDeleteCallsSinceCheckpoint = 0 - - checkpointsSinceDownUp++ - if checkpointsSinceDownUp == checkpointsSinceDownUpCap { - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - volumeHandle, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("headhunter.FetchVolumeHandle(\"TestVolume\") returned error: %v", err) - } - checkpointsSinceDownUpCap = testRandU64FLessThanN(t, testMaxCheckpointsPerDownUp) + 1 - checkpointsSinceDownUp = 0 - } - } - } - - // Cleanly shutdown and restart - - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - - // TODO: Place stress reading test steps here... - // GetInodeRec(inodeNumber uint64) (value []byte, ok bool, err error) - // GetLogSegmentRec(logSegmentNumber uint64) (value []byte, err error) - // GetBPlusTreeObject(objectNumber uint64) (value []byte, err error) - - // Cleanly shutdown - - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - - // Send ourself a SIGTERM to terminate ramswift.Daemon() - - unix.Kill(unix.Getpid(), unix.SIGTERM) - - _ = <-doneChan -} diff --git a/httpserver/.gitignore b/httpserver/.gitignore deleted file mode 100644 index 3a694af3..00000000 --- a/httpserver/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -bootstrap_dot_min_dot_css_.go -bootstrap_dot_min_dot_js_.go -jquery_dot_min_dot_js_.go -jquery_underline_3_dot_2_dot_1_dot_min_dot_js_.go -jsontree_dot_js_.go -open_iconic_bootstrap_dot_css_.go -open_iconic_dot_eot_.go -open_iconic_dot_otf_.go -open_iconic_dot_svg_.go -open_iconic_dot_ttf_.go -open_iconic_dot_woff_.go -popper_dot_min_dot_js_.go -styles_dot_css_.go diff --git a/httpserver/Makefile b/httpserver/Makefile deleted file mode 100644 index 1f29bddd..00000000 --- a/httpserver/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/httpserver - -generatedfiles := \ - bootstrap_dot_min_dot_css_.go \ - bootstrap_dot_min_dot_js_.go \ - jquery_dot_min_dot_js_.go \ - jquery_underline_3_dot_2_dot_1_dot_min_dot_js_.go \ - jsontree_dot_js_.go \ - open_iconic_bootstrap_dot_css_.go \ - open_iconic_dot_eot_.go \ - open_iconic_dot_otf_.go \ - open_iconic_dot_svg_.go \ - open_iconic_dot_ttf_.go \ - open_iconic_dot_woff_.go \ - popper_dot_min_dot_js_.go \ - styles_dot_css_.go - -include ../GoMakefile diff --git a/httpserver/config.go b/httpserver/config.go deleted file mode 100644 index 0c026f5f..00000000 --- a/httpserver/config.go +++ /dev/null @@ -1,317 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package httpserver - -import ( - "fmt" - "net" - "strconv" - "sync" - "time" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/transitions" - - // Force importing of the following next "top-most" packages - _ "github.com/NVIDIA/proxyfs/fuse" - _ "github.com/NVIDIA/proxyfs/jrpcfs" - _ "github.com/NVIDIA/proxyfs/statslogger" - "github.com/NVIDIA/proxyfs/trackedlock" -) - -type ExtentMapElementStruct struct { - FileOffset uint64 `json:"file_offset"` - ContainerName string `json:"container_name"` - ObjectName string `json:"object_name"` - ObjectOffset uint64 `json:"object_offset"` - Length uint64 `json:"length"` -} - -type jobState uint8 - -const ( - jobRunning jobState = iota - jobHalted - jobCompleted -) - -type jobTypeType uint8 - -const ( - fsckJobType jobTypeType = iota - scrubJobType - limitJobType -) - -type jobStruct struct { - id uint64 - volume *volumeStruct - jobHandle fs.JobHandle - state jobState - startTime time.Time - endTime time.Time -} - -// JobStatusJSONPackedStruct describes all the possible fields returned in JSON-encoded job GET body -type JobStatusJSONPackedStruct struct { - StartTime string `json:"start time"` - HaltTime string `json:"halt time"` - DoneTime string `json:"done time"` - ErrorList []string `json:"error list"` - InfoList []string `json:"info list"` -} - -type volumeStruct struct { - trackedlock.Mutex - name string - fsVolumeHandle fs.VolumeHandle - inodeVolumeHandle inode.VolumeHandle - headhunterVolumeHandle headhunter.VolumeHandle - fsckActiveJob *jobStruct - fsckJobs sortedmap.LLRBTree // Key == jobStruct.id, Value == *jobStruct - scrubActiveJob *jobStruct - scrubJobs sortedmap.LLRBTree // Key == jobStruct.id, Value == *jobStruct - activeDefragInodeNumberSet map[inode.InodeNumber]struct{} -} - -type globalsStruct struct { - trackedlock.Mutex - active bool - jobHistoryMaxSize uint32 - whoAmI string - ipAddr string - tcpPort uint16 - ipAddrTCPPort string - netListener net.Listener - wg sync.WaitGroup - confMap conf.ConfMap - volumeLLRB sortedmap.LLRBTree // Key == volumeStruct.name, Value == *volumeStruct -} - -var globals globalsStruct - -func init() { - transitions.Register("httpserver", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - globals.volumeLLRB = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - globals.jobHistoryMaxSize, err = confMap.FetchOptionValueUint32("HTTPServer", "JobHistoryMaxSize") - if nil != err { - /* - TODO: Eventually change this to: - err = fmt.Errorf("confMap.FetchOptionValueString(\"HTTPServer\", \"JobHistoryMaxSize\") failed: %v", err) - return - */ - globals.jobHistoryMaxSize = 5 - } - - globals.whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueString(\"Cluster\", \"WhoAmI\") failed: %v", err) - return - } - - globals.ipAddr, err = confMap.FetchOptionValueString("Peer:"+globals.whoAmI, "PrivateIPAddr") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueString(\"\", \"PrivateIPAddr\") failed: %v", err) - return - } - - globals.tcpPort, err = confMap.FetchOptionValueUint16("HTTPServer", "TCPPort") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueUint16(\"HTTPServer\", \"TCPPort\") failed: %v", err) - return - } - - globals.ipAddrTCPPort = net.JoinHostPort(globals.ipAddr, strconv.Itoa(int(globals.tcpPort))) - - globals.netListener, err = net.Listen("tcp", globals.ipAddrTCPPort) - if nil != err { - err = fmt.Errorf("net.Listen(\"tcp\", \"%s\") failed: %v", globals.ipAddrTCPPort, err) - return - } - - globals.active = false - - globals.wg.Add(1) - go serveHTTP() - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} - -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - ok bool - volume *volumeStruct - ) - - volume = &volumeStruct{ - name: volumeName, - fsckActiveJob: nil, - fsckJobs: sortedmap.NewLLRBTree(sortedmap.CompareUint64, nil), - scrubActiveJob: nil, - scrubJobs: sortedmap.NewLLRBTree(sortedmap.CompareUint64, nil), - activeDefragInodeNumberSet: make(map[inode.InodeNumber]struct{}), - } - - volume.fsVolumeHandle, err = fs.FetchVolumeHandleByVolumeName(volume.name) - if nil != err { - return - } - - volume.inodeVolumeHandle, err = inode.FetchVolumeHandle(volume.name) - if nil != err { - return - } - - volume.headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(volume.name) - if nil != err { - return - } - - globals.Lock() - - ok, err = globals.volumeLLRB.Put(volumeName, volume) - if nil != err { - globals.Unlock() - err = fmt.Errorf("globals.volumeLLRB.Put(%s,) failed: %v", volumeName, err) - return - } - if !ok { - globals.Unlock() - err = fmt.Errorf("globals.volumeLLRB.Put(%s,) returned ok == false", volumeName) - return - } - - globals.Unlock() - - return // return err from globals.volumeLLRB.Put() sufficient -} - -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - ok bool - volume *volumeStruct - volumeAsValue sortedmap.Value - ) - - globals.Lock() - - volumeAsValue, ok, err = globals.volumeLLRB.GetByKey(volumeName) - if nil != err { - globals.Unlock() - err = fmt.Errorf("globals.volumeLLRB.Get(%v) failed: %v", volumeName, err) - return - } - - volume, ok = volumeAsValue.(*volumeStruct) - if !ok { - globals.Unlock() - err = fmt.Errorf("volumeAsValue.(*volumeStruct) returned ok == false for volume %s", volumeName) - return - } - - if !ok { - globals.Unlock() - return // return err from globals.volumeLLRB.GetByKey() sufficient - } - - volume.Lock() - if nil != volume.fsckActiveJob { - volume.fsckActiveJob.jobHandle.Cancel() - volume.fsckActiveJob.state = jobHalted - volume.fsckActiveJob.endTime = time.Now() - volume.fsckActiveJob = nil - } - if nil != volume.scrubActiveJob { - volume.scrubActiveJob.jobHandle.Cancel() - volume.scrubActiveJob.state = jobHalted - volume.scrubActiveJob.endTime = time.Now() - volume.scrubActiveJob = nil - } - volume.Unlock() - - ok, err = globals.volumeLLRB.DeleteByKey(volumeName) - if nil != err { - globals.Unlock() - err = fmt.Errorf("globals.volumeLLRB.DeleteByKey(%v) failed: %v", volumeName, err) - return - } - - globals.Unlock() - - return // return err from globals.volumeLLRB.DeleteByKey sufficient -} - -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} - -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - globals.confMap = confMap - globals.active = false - return nil -} - -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - globals.confMap = confMap - globals.active = true - return nil -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - var ( - numVolumes int - ) - - globals.Lock() - - numVolumes, err = globals.volumeLLRB.Len() - if nil != err { - globals.Unlock() - err = fmt.Errorf("httpserver.Down() couldn't read globals.volumeLLRB: %v", err) - return - } - if 0 != numVolumes { - globals.Unlock() - err = fmt.Errorf("httpserver.Down() called with 0 != globals.volumeLLRB.Len()") - return - } - - _ = globals.netListener.Close() - - globals.Unlock() - - globals.wg.Wait() - - err = nil - return -} diff --git a/httpserver/html_templates.go b/httpserver/html_templates.go deleted file mode 100644 index 9dd925e6..00000000 --- a/httpserver/html_templates.go +++ /dev/null @@ -1,1255 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package httpserver - -// To use: fmt.Sprintf(indexDotHTMLTemplate, proxyfsVersion, globals.ipAddrTCPPort) -const indexDotHTMLTemplate string = ` - - - - - - - ProxyFS Management - %[2]v - - - -
- -

- ProxyFS Management -

-
-
-
-
Configuration parameters
-

Diplays a JSON representation of the active configuration.

-
- -
-
-
-
-
StatsD/Prometheus
-

Displays current statistics.

-
- -
-
-
-
-
-
-
-
Triggers
-

Manage triggers for simulating failures.

-
- -
-
-
-
-
Volumes
-

Examine volumes currently active on this ProxyFS node.

-
- -
-
-
- - - - - -` - -// To use: fmt.Sprintf(configTemplate, proxyfsVersion, globals.ipAddrTCPPort, confMapJSONString) -const configTemplate string = ` - - - - - - - Config - %[2]v - - - -
- -

- Config -

-

-    
- - - - - - - -` - -// To use: fmt.Sprintf(metricsTemplate, proxyfsVersion, globals.ipAddrTCPPort, metricsJSONString) -const metricsTemplate string = ` - - - - - - - Metrics - %[2]v - - - -
- -

StatsD/Prometheus

-
-
-
-
- - -
-
- - - - - - -` - -// To use: fmt.Sprintf(volumeListTopTemplate, proxyfsVersion, globals.ipAddrTCPPort) -const volumeListTopTemplate string = ` - - - - - - - Volumes - %[2]v - - - -
- - -

Volumes

- - - - - - - - - - - - -` - -// To use: fmt.Sprintf(volumeListPerVolumeTemplate, volumeName) -const volumeListPerVolumeTemplate string = ` - - - - - - - -` - -const volumeListBottom string = ` -
Volume Name     
%[1]vSnapShotsFSCK jobsSCRUB jobsLayout ReportExtent Map
-
- - - - - -` - -// To use: fmt.Sprintf(snapShotsTopTemplate, proxyfsVersion, globals.ipAddrTCPPort, volumeName) -const snapShotsTopTemplate string = ` - - - - - - - - %[2]v SnapShots - %[2]v - - - -
- -
-

- SnapShots - %[3]v -

-
-
- -
- -
-
-
- - - - - - - - - - -` - -// To use: fmt.Sprintf(snapShotsPerSnapShotTemplate, id, timeStamp.Format(time.RFC3339), name) -const snapShotsPerSnapShotTemplate string = ` - - - - - -` - -// To use: fmt.Sprintf(snapShotsBottomTemplate, volumeName) -const snapShotsBottomTemplate string = ` -
IDTimeName 
%[1]v%[2]v%[3]v
-
-
- - - - - - -` - -// To use: fmt.Sprintf(jobsTopTemplate, proxyfsVersion, globals.ipAddrTCPPort, volumeName, {"FSCK"|"SCRUB"}) -const jobsTopTemplate string = ` - - - - - - - %[4]v Jobs %[3]v - %[2]v - - - -
- -

- %[4]v Jobs - %[3]v -

- - - - - - - - - - - -` - -// To use: fmt.Sprintf(jobsPerRunningJobTemplate, jobID, job.startTime.Format(time.RFC3339), volumeName, {"fsck"|"scrub"}) -const jobsPerRunningJobTemplate string = ` - - - - - - -` - -// To use: fmt.Sprintf(jobsPerHaltedJobTemplate, jobID, job.startTime.Format(time.RFC3339), job.endTime.Format(time.RFC3339), volumeName, {"fsck"|"scrub"}) -const jobsPerHaltedJobTemplate string = ` - - - - - - -` - -// To use: fmt.Sprintf(jobsPerSuccessfulJobTemplate, jobID, job.startTime.Format(time.RFC3339), job.endTime.Format(time.RFC3339), volumeName, {"fsck"|"scrub"}) -const jobsPerSuccessfulJobTemplate string = ` - - - - - - -` - -// To use: fmt.Sprintf(jobsPerFailedJobTemplate, jobID, job.startTime.Format(time.RFC3339), job.endTime.Format(time.RFC3339), volumeName, {"fsck"|"scrub"}) -const jobsPerFailedJobTemplate string = ` - - - - - - -` - -const jobsListBottom string = ` -
Job IDStart TimeEnd TimeStatus 
%[1]v%[2]vRunningView
%[1]v%[2]v%[3]vHaltedView
%[1]v%[2]v%[3]vSuccessfulView
%[1]v%[2]v%[3]vFailedView
-
-` - -// To use: fmt.Sprintf(jobsStartJobButtonTemplate, volumeName, {"fsck"|"scrub"}) -const jobsStartJobButtonTemplate string = `
- -
-` - -const jobsBottom string = ` - - - - -` - -// To use: fmt.Sprintf(jobTemplate, proxyfsVersion, globals.ipAddrTCPPort, volumeName, {"FSCK"|"SCRUB"}, {"fsck"|"scrub"}, jobID, jobStatusJSONString) -const jobTemplate string = ` - - - - - - - %[6]v %[4]v Job - %[2]v - - - -
- -

- %[4]v Job - %[6]v -

-
-
-
- - - - - - -` - -// To use: fmt.Sprintf(layoutReportTopTemplate, proxyfsVersion, globals.ipAddrTCPPort, volumeName) -const layoutReportTopTemplate string = ` - - - - - - - Layout Report %[3]v - %[2]v - - - -
- -

- Layout Report - %[3]v -

- Show validated version

-` - -// To use: fmt.Sprintf(layoutReportTableTopTemplate, TreeName, NumDiscrepencies, {"success"|"danger"}) -const layoutReportTableTopTemplate string = `
-
-

%[1]v

%[2]v discrepancies

-
- - - - - - - - -` - -// To use: fmt.Sprintf(layoutReportTableRowTemplate, ObjectName, ObjectBytes) -const layoutReportTableRowTemplate string = ` - - - -` - -const layoutReportTableBottom string = ` -
ObjectNameObjectBytes
%016[1]X
%[2]v
-` - -const layoutReportBottom string = `
- - - - - - -` - -// To use: fmt.Sprintf(extentMapTemplate, proxyfsVersion, globals.ipAddrTCPPort, volumeName, extentMapJSONString, pathDoubleQuotedString, serverErrorBoolString) -const extentMapTemplate string = ` - - - - - - - Extent Map %[3]v - %[2]v - - - -
- - -

- Extent Map - %[3]v -

- - - -
-
- -
- -
-
-
- -
- - - - - - - - - - -
File OffsetContainer/ObjectObject OffsetLength
-
- - - - - - -` - -// To use: fmt.Sprintf(triggerTopTemplate, proxyfsVersion, globals.ipAddrTCPPort) -const triggerTopTemplate string = ` - - - - - - - Triggers - %[2]v - - - -
- -

Triggers

-` - -const triggerAllActive string = `
-
- All - Armed - Disarmed -
-
-` - -const triggerArmedActive string = `
-
- All - Armed - Disarmed -
-
-` - -const triggerDisarmedActive string = `
-
- All - Armed - Disarmed -
-
-` - -const triggerTableTop string = `
- - - - - - - - -` - -// To use: fmt.Sprintf(triggerTableRowTemplate, haltTriggerString, haltTriggerCount) -const triggerTableRowTemplate string = ` - - - -` - -const triggerBottom string = ` -
Halt LabelHalt After Count
%[1]v -
- -
- New value successfully saved. -
-
- There was an error saving the new value. -
-
-
-
- - - - - - -` diff --git a/httpserver/request_handler.go b/httpserver/request_handler.go deleted file mode 100644 index b8eed5c9..00000000 --- a/httpserver/request_handler.go +++ /dev/null @@ -1,3082 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package httpserver - -import ( - "bufio" - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "math" - "net/http" - "net/http/pprof" - "net/url" - "runtime" - "strconv" - "strings" - "time" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/halter" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/jrpcfs" - "github.com/NVIDIA/proxyfs/liveness" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/utils" - "github.com/NVIDIA/proxyfs/version" -) - -type httpRequestHandler struct{} - -type requestStateStruct struct { - pathSplit []string - numPathParts int - formatResponseAsJSON bool - formatResponseCompactly bool - performValidation bool - percentRange string - startNonce uint64 - volume *volumeStruct -} - -func serveHTTP() { - _ = http.Serve(globals.netListener, httpRequestHandler{}) - globals.wg.Done() -} - -func (h httpRequestHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - globals.Lock() - if globals.active { - switch request.Method { - case http.MethodDelete: - doDelete(responseWriter, request) - case http.MethodGet: - doGet(responseWriter, request) - case http.MethodPost: - doPost(responseWriter, request) - case http.MethodPut: - doPut(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } - } else { - responseWriter.WriteHeader(http.StatusServiceUnavailable) - } - globals.Unlock() -} - -func doDelete(responseWriter http.ResponseWriter, request *http.Request) { - switch { - case strings.HasPrefix(request.URL.Path, "/volume"): - doDeleteOfVolume(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func doDeleteOfVolume(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - numPathParts int - ok bool - pathSplit []string - snapShotID uint64 - volume *volumeStruct - volumeAsValue sortedmap.Value - volumeName string - ) - - pathSplit = strings.Split(request.URL.Path, "/") // leading "/" places "" in pathSplit[0] - // pathSplit[1] should be "volume" based on how we got here - // trailing "/" places "" in pathSplit[len(pathSplit)-1] - numPathParts = len(pathSplit) - 1 - if "" == pathSplit[numPathParts] { - numPathParts-- - } - - if "volume" != pathSplit[1] { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - if 4 != numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - volumeName = pathSplit[2] - - volumeAsValue, ok, err = globals.volumeLLRB.GetByKey(volumeName) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - volume = volumeAsValue.(*volumeStruct) - - if "snapshot" != pathSplit[3] { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - // Form: /volume//snapshot/ - - snapShotID, err = strconv.ParseUint(pathSplit[4], 10, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - err = volume.inodeVolumeHandle.SnapShotDelete(snapShotID) - if nil == err { - responseWriter.WriteHeader(http.StatusNoContent) - } else { - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func doGet(responseWriter http.ResponseWriter, request *http.Request) { - path := strings.TrimRight(request.URL.Path, "/") - - switch { - case "" == path: - doGetOfIndexDotHTML(responseWriter, request) - case "/bootstrap.min.css" == path: - responseWriter.Header().Set("Content-Type", bootstrapDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(bootstrapDotCSSContent)) - case "/bootstrap.min.js" == path: - responseWriter.Header().Set("Content-Type", bootstrapDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(bootstrapDotJSContent)) - case "/config" == path: - doGetOfConfig(responseWriter, request) - case "/index.html" == path: - doGetOfIndexDotHTML(responseWriter, request) - case "/jquery.min.js" == path: - responseWriter.Header().Set("Content-Type", jqueryDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(jqueryDotJSContent)) - case "/jsontree.js" == path: - responseWriter.Header().Set("Content-Type", jsontreeDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(jsontreeDotJSContent)) - case "/liveness" == path: - doGetOfLiveness(responseWriter, request) - case "/metrics" == path: - doGetOfMetrics(responseWriter, request) - case "/stats" == path: - doGetOfStats(responseWriter, request) - case "/debug/pprof/cmdline" == path: - pprof.Cmdline(responseWriter, request) - case "/debug/pprof/profile" == path: - pprof.Profile(responseWriter, request) - case "/debug/pprof/symbol" == path: - pprof.Symbol(responseWriter, request) - case "/debug/pprof/trace" == path: - pprof.Trace(responseWriter, request) - case strings.HasPrefix(request.URL.Path, "/debug/pprof"): - pprof.Index(responseWriter, request) - case "/open-iconic/font/css/open-iconic-bootstrap.min.css" == path: - responseWriter.Header().Set("Content-Type", openIconicBootstrapDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(openIconicBootstrapDotCSSContent)) - case "/open-iconic/font/fonts/open-iconic.eot" == path: - responseWriter.Header().Set("Content-Type", openIconicDotEOTContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotEOTContent) - case "/open-iconic/font/fonts/open-iconic.otf" == path: - responseWriter.Header().Set("Content-Type", openIconicDotOTFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotOTFContent) - case "/open-iconic/font/fonts/open-iconic.svg" == path: - responseWriter.Header().Set("Content-Type", openIconicDotSVGContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(openIconicDotSVGContent)) - case "/open-iconic/font/fonts/open-iconic.ttf" == path: - responseWriter.Header().Set("Content-Type", openIconicDotTTFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotTTFContent) - case "/open-iconic/font/fonts/open-iconic.woff" == path: - responseWriter.Header().Set("Content-Type", openIconicDotWOFFContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(openIconicDotWOFFContent) - case "/popper.min.js" == path: - responseWriter.Header().Set("Content-Type", popperDotJSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(popperDotJSContent) - case "/styles.css" == path: - responseWriter.Header().Set("Content-Type", stylesDotCSSContentType) - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(stylesDotCSSContent)) - case "/version" == path: - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(version.ProxyFSVersion)) - case strings.HasPrefix(request.URL.Path, "/trigger"): - doGetOfTrigger(responseWriter, request) - case strings.HasPrefix(request.URL.Path, "/volume"): - doGetOfVolume(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func doGetOfIndexDotHTML(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort))) -} - -func doGetOfConfig(responseWriter http.ResponseWriter, request *http.Request) { - var ( - acceptHeader string - confMapJSON bytes.Buffer - confMapJSONPacked []byte - formatResponseAsJSON bool - ok bool - paramList []string - sendPackedConfig bool - ) - - paramList, ok = request.URL.Query()["compact"] - if ok { - if 0 == len(paramList) { - sendPackedConfig = false - } else { - sendPackedConfig = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - sendPackedConfig = false - } - - acceptHeader = request.Header.Get("Accept") - - if strings.Contains(acceptHeader, "application/json") { - formatResponseAsJSON = true - } else if strings.Contains(acceptHeader, "text/html") { - formatResponseAsJSON = false - } else if strings.Contains(acceptHeader, "*/*") { - formatResponseAsJSON = true - } else if strings.Contains(acceptHeader, "") { - formatResponseAsJSON = true - } else { - responseWriter.WriteHeader(http.StatusNotAcceptable) - return - } - - confMapJSONPacked, _ = json.Marshal(globals.confMap) - - if formatResponseAsJSON { - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if sendPackedConfig { - _, _ = responseWriter.Write(confMapJSONPacked) - } else { - json.Indent(&confMapJSON, confMapJSONPacked, "", "\t") - _, _ = responseWriter.Write(confMapJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(configTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, utils.ByteSliceToString(confMapJSONPacked)))) - } -} - -func doGetOfLiveness(responseWriter http.ResponseWriter, request *http.Request) { - var ( - livenessReportAsJSON bytes.Buffer - livenessReportAsJSONPacked []byte - livenessReportAsStruct *liveness.LivenessReportStruct - ok bool - paramList []string - sendPackedReport bool - ) - - // TODO: For now, assume JSON reponse requested - - livenessReportAsStruct = liveness.FetchLivenessReport() - - if nil == livenessReportAsStruct { - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - livenessReportAsJSONPacked, _ = json.Marshal(livenessReportAsStruct) - - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - paramList, ok = request.URL.Query()["compact"] - if ok { - if 0 == len(paramList) { - sendPackedReport = false - } else { - sendPackedReport = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - sendPackedReport = false - } - - if sendPackedReport { - _, _ = responseWriter.Write(livenessReportAsJSONPacked) - } else { - json.Indent(&livenessReportAsJSON, livenessReportAsJSONPacked, "", "\t") - _, _ = responseWriter.Write(livenessReportAsJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } -} - -func doGetOfMetrics(responseWriter http.ResponseWriter, request *http.Request) { - var ( - acceptHeader string - err error - formatResponseAsHTML bool - formatResponseAsJSON bool - i int - memStats runtime.MemStats - metricKey string - metricValueAsString string - metricValueAsUint64 uint64 - metricsJSON bytes.Buffer - metricsJSONPacked []byte - metricsLLRB sortedmap.LLRBTree - metricsMap map[string]uint64 - ok bool - paramList []string - pauseNsAccumulator uint64 - sendPackedMetrics bool - statKey string - statValue uint64 - statsMap map[string]uint64 - ) - - runtime.ReadMemStats(&memStats) - - metricsMap = make(map[string]uint64) - - // General statistics. - metricsMap["go_runtime_MemStats_Alloc"] = memStats.Alloc - metricsMap["go_runtime_MemStats_TotalAlloc"] = memStats.TotalAlloc - metricsMap["go_runtime_MemStats_Sys"] = memStats.Sys - metricsMap["go_runtime_MemStats_Lookups"] = memStats.Lookups - metricsMap["go_runtime_MemStats_Mallocs"] = memStats.Mallocs - metricsMap["go_runtime_MemStats_Frees"] = memStats.Frees - - // Main allocation heap statistics. - metricsMap["go_runtime_MemStats_HeapAlloc"] = memStats.HeapAlloc - metricsMap["go_runtime_MemStats_HeapSys"] = memStats.HeapSys - metricsMap["go_runtime_MemStats_HeapIdle"] = memStats.HeapIdle - metricsMap["go_runtime_MemStats_HeapInuse"] = memStats.HeapInuse - metricsMap["go_runtime_MemStats_HeapReleased"] = memStats.HeapReleased - metricsMap["go_runtime_MemStats_HeapObjects"] = memStats.HeapObjects - - // Low-level fixed-size structure allocator statistics. - // Inuse is bytes used now. - // Sys is bytes obtained from system. - metricsMap["go_runtime_MemStats_StackInuse"] = memStats.StackInuse - metricsMap["go_runtime_MemStats_StackSys"] = memStats.StackSys - metricsMap["go_runtime_MemStats_MSpanInuse"] = memStats.MSpanInuse - metricsMap["go_runtime_MemStats_MSpanSys"] = memStats.MSpanSys - metricsMap["go_runtime_MemStats_MCacheInuse"] = memStats.MCacheInuse - metricsMap["go_runtime_MemStats_MCacheSys"] = memStats.MCacheSys - metricsMap["go_runtime_MemStats_BuckHashSys"] = memStats.BuckHashSys - metricsMap["go_runtime_MemStats_GCSys"] = memStats.GCSys - metricsMap["go_runtime_MemStats_OtherSys"] = memStats.OtherSys - - // Garbage collector statistics (fixed portion). - metricsMap["go_runtime_MemStats_LastGC"] = memStats.LastGC - metricsMap["go_runtime_MemStats_PauseTotalNs"] = memStats.PauseTotalNs - metricsMap["go_runtime_MemStats_NumGC"] = uint64(memStats.NumGC) - metricsMap["go_runtime_MemStats_GCCPUPercentage"] = uint64(100.0 * memStats.GCCPUFraction) - - // Garbage collector statistics (go_runtime_MemStats_PauseAverageNs). - if 0 == memStats.NumGC { - metricsMap["go_runtime_MemStats_PauseAverageNs"] = 0 - } else { - pauseNsAccumulator = 0 - if memStats.NumGC < 255 { - for i = 0; i < int(memStats.NumGC); i++ { - pauseNsAccumulator += memStats.PauseNs[i] - } - metricsMap["go_runtime_MemStats_PauseAverageNs"] = pauseNsAccumulator / uint64(memStats.NumGC) - } else { - for i = 0; i < 256; i++ { - pauseNsAccumulator += memStats.PauseNs[i] - } - metricsMap["go_runtime_MemStats_PauseAverageNs"] = pauseNsAccumulator / 256 - } - } - - statsMap = stats.Dump() - - for statKey, statValue = range statsMap { - metricKey = strings.Replace(statKey, ".", "_", -1) - metricKey = strings.Replace(metricKey, "-", "_", -1) - metricsMap[metricKey] = statValue - } - - acceptHeader = request.Header.Get("Accept") - - if strings.Contains(acceptHeader, "application/json") { - formatResponseAsHTML = false - formatResponseAsJSON = true - } else if strings.Contains(acceptHeader, "text/html") { - formatResponseAsHTML = true - formatResponseAsJSON = true - } else if strings.Contains(acceptHeader, "text/plain") { - formatResponseAsHTML = false - formatResponseAsJSON = false - } else if strings.Contains(acceptHeader, "*/*") { - formatResponseAsHTML = false - formatResponseAsJSON = true - } else if strings.Contains(acceptHeader, "") { - formatResponseAsHTML = false - formatResponseAsJSON = true - } else { - responseWriter.WriteHeader(http.StatusNotAcceptable) - return - } - - if formatResponseAsJSON { - metricsJSONPacked, _ = json.Marshal(metricsMap) - if formatResponseAsHTML { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(metricsTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, utils.ByteSliceToString(metricsJSONPacked)))) - } else { - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - paramList, ok = request.URL.Query()["compact"] - if ok { - if 0 == len(paramList) { - sendPackedMetrics = false - } else { - sendPackedMetrics = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - sendPackedMetrics = false - } - - if sendPackedMetrics { - _, _ = responseWriter.Write(metricsJSONPacked) - } else { - json.Indent(&metricsJSON, metricsJSONPacked, "", "\t") - _, _ = responseWriter.Write(metricsJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } - } - } else { - metricsLLRB = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - for metricKey, metricValueAsUint64 = range metricsMap { - metricValueAsString = fmt.Sprintf("%v", metricValueAsUint64) - ok, err = metricsLLRB.Put(metricKey, metricValueAsString) - if nil != err { - err = fmt.Errorf("metricsLLRB.Put(%v, %v) failed: %v", metricKey, metricValueAsString, err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - err = fmt.Errorf("metricsLLRB.Put(%v, %v) returned ok == false", metricKey, metricValueAsString) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - } - - sortedTwoColumnResponseWriter(metricsLLRB, responseWriter) - } -} - -func doGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { - - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*"))) -} - -func doGetOfArmDisarmTrigger(responseWriter http.ResponseWriter, request *http.Request) { - var ( - availableTriggers []string - err error - haltTriggerString string - ok bool - triggersLLRB sortedmap.LLRBTree - ) - - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte("\n")) - _, _ = responseWriter.Write([]byte("\n")) - _, _ = responseWriter.Write([]byte(" \n")) - _, _ = responseWriter.Write([]byte(fmt.Sprintf(" Trigger Arm/Disarm Page\n"))) - _, _ = responseWriter.Write([]byte(" \n")) - _, _ = responseWriter.Write([]byte(" \n")) - _, _ = responseWriter.Write([]byte("
\n")) - _, _ = responseWriter.Write([]byte(" \n")) - _, _ = responseWriter.Write([]byte(" \n")) - _, _ = responseWriter.Write([]byte(" \n")) - _, _ = responseWriter.Write([]byte("
\n")) - _, _ = responseWriter.Write([]byte(" \n")) - _, _ = responseWriter.Write([]byte("\n")) -} - -func doGetOfTrigger(responseWriter http.ResponseWriter, request *http.Request) { - var ( - triggerAllArmedOrDisarmedActiveString string - armedTriggers map[string]uint32 - availableTriggers []string - err error - haltTriggerArmedStateAsBool bool - haltTriggerArmedStateAsString string - haltTriggerCount uint32 - haltTriggerString string - i int - lenTriggersLLRB int - numPathParts int - key sortedmap.Key - ok bool - pathSplit []string - triggersLLRB sortedmap.LLRBTree - value sortedmap.Value - ) - - pathSplit = strings.Split(request.URL.Path, "/") // leading "/" places "" in pathSplit[0] - // pathSplit[1] should be "trigger" based on how we got here - // trailing "/" places "" in pathSplit[len(pathSplit)-1] - - numPathParts = len(pathSplit) - 1 - if "" == pathSplit[numPathParts] { - numPathParts-- - } - - if "trigger" != pathSplit[1] { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - switch numPathParts { - case 1: - // Form: /trigger[?armed={true|false}] - - haltTriggerArmedStateAsString = request.FormValue("armed") - - if "" == haltTriggerArmedStateAsString { - triggerAllArmedOrDisarmedActiveString = triggerAllActive - armedTriggers = halter.Dump() - availableTriggers = halter.List() - - triggersLLRB = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - for _, haltTriggerString = range availableTriggers { - haltTriggerCount, ok = armedTriggers[haltTriggerString] - if !ok { - haltTriggerCount = 0 - } - ok, err = triggersLLRB.Put(haltTriggerString, haltTriggerCount) - if nil != err { - err = fmt.Errorf("triggersLLRB.Put(%v, %v) failed: %v", haltTriggerString, haltTriggerCount, err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - err = fmt.Errorf("triggersLLRB.Put(%v, %v) returned ok == false", haltTriggerString, haltTriggerCount) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - } - } else { - haltTriggerArmedStateAsBool, err = strconv.ParseBool(haltTriggerArmedStateAsString) - if nil == err { - triggersLLRB = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - if haltTriggerArmedStateAsBool { - triggerAllArmedOrDisarmedActiveString = triggerArmedActive - armedTriggers = halter.Dump() - for haltTriggerString, haltTriggerCount = range armedTriggers { - ok, err = triggersLLRB.Put(haltTriggerString, haltTriggerCount) - if nil != err { - err = fmt.Errorf("triggersLLRB.Put(%v, %v) failed: %v", haltTriggerString, haltTriggerCount, err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - err = fmt.Errorf("triggersLLRB.Put(%v, %v) returned ok == false", haltTriggerString, haltTriggerCount) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - } - } else { - triggerAllArmedOrDisarmedActiveString = triggerDisarmedActive - armedTriggers = halter.Dump() - availableTriggers = halter.List() - - for _, haltTriggerString = range availableTriggers { - _, ok = armedTriggers[haltTriggerString] - if !ok { - ok, err = triggersLLRB.Put(haltTriggerString, uint32(0)) - if nil != err { - err = fmt.Errorf("triggersLLRB.Put(%v, %v) failed: %v", haltTriggerString, 0, err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - err = fmt.Errorf("triggersLLRB.Put(%v, %v) returned ok == false", haltTriggerString, 0) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - } - } - } - } else { - responseWriter.WriteHeader(http.StatusBadRequest) - } - } - - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(triggerTopTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort))) - _, _ = responseWriter.Write([]byte(triggerAllArmedOrDisarmedActiveString)) - _, _ = responseWriter.Write([]byte(triggerTableTop)) - - lenTriggersLLRB, err = triggersLLRB.Len() - if nil != err { - err = fmt.Errorf("triggersLLRB.Len()) failed: %v", err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - for i = 0; i < lenTriggersLLRB; i++ { - key, value, ok, err = triggersLLRB.GetByIndex(i) - if nil != err { - err = fmt.Errorf("triggersLLRB.GetByIndex(%v) failed: %v", i, err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - err = fmt.Errorf("triggersLLRB.GetByIndex(%v) returned ok == false", i) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - haltTriggerString = key.(string) - haltTriggerCount = value.(uint32) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(triggerTableRowTemplate, haltTriggerString, haltTriggerCount))) - } - - _, _ = responseWriter.Write([]byte(triggerBottom)) - case 2: - // Form: /trigger/ - - haltTriggerString = pathSplit[2] - - haltTriggerCount, err = halter.Stat(haltTriggerString) - if nil == err { - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("%v\n", haltTriggerCount))) - } else { - responseWriter.WriteHeader(http.StatusNotFound) - } - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func doGetOfVolume(responseWriter http.ResponseWriter, request *http.Request) { - var ( - acceptHeader string - err error - formatResponseAsJSON bool - formatResponseCompactly bool - numPathParts int - ok bool - paramList []string - pathSplit []string - percentRange string - performValidation bool - requestState *requestStateStruct - startNonceAsString string - startNonceAsUint64 uint64 - volumeAsValue sortedmap.Value - volumeList []string - volumeListIndex int - volumeListJSON bytes.Buffer - volumeListJSONPacked []byte - volumeListLen int - volumeName string - volumeNameAsKey sortedmap.Key - ) - - pathSplit = strings.Split(request.URL.Path, "/") // leading "/" places "" in pathSplit[0] - // pathSplit[1] should be "volume" based on how we got here - // trailing "/" places "" in pathSplit[len(pathSplit)-1] - - numPathParts = len(pathSplit) - 1 - if "" == pathSplit[numPathParts] { - numPathParts-- - } - - if "volume" != pathSplit[1] { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - switch numPathParts { - case 1: - // Form: /volume - case 2: - responseWriter.WriteHeader(http.StatusNotFound) - return - case 3: - // Form: /volume//extent-map - // Form: /volume//fsck-job - // Form: /volume//layout-report - // Form: /volume//lease-report - // Form: /volume//meta-defrag - // Form: /volume//scrub-job - // Form: /volume//snapshot - case 4: - // Form: /volume//defrag/ - // Form: /volume//extent-map/ - // Form: /volume//fetch-ondisk-inode/ - // Form: /volume//fetch-ondisk-metadata-object/ - // Form: /volume//find-subdir-inodes/ - // Form: /volume//fsck-job/ - // Form: /volume//meta-defrag/ - // Form: /volume//scrub-job/ - default: - // Form: /volume//defrag//.../ - // Form: /volume//extent-map//.../ - // Form: /volume//find-dir-inode//.../ - } - - acceptHeader = request.Header.Get("Accept") - - if strings.Contains(acceptHeader, "application/json") { - formatResponseAsJSON = true - } else if strings.Contains(acceptHeader, "text/html") { - formatResponseAsJSON = false - } else if strings.Contains(acceptHeader, "*/*") { - formatResponseAsJSON = true - } else if strings.Contains(acceptHeader, "") { - formatResponseAsJSON = true - } else { - responseWriter.WriteHeader(http.StatusNotAcceptable) - return - } - - if formatResponseAsJSON { - paramList, ok = request.URL.Query()["compact"] - if ok { - if 0 == len(paramList) { - formatResponseCompactly = false - } else { - formatResponseCompactly = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - formatResponseCompactly = false - } - } - - paramList, ok = request.URL.Query()["validate"] - if ok { - if 0 == len(paramList) { - performValidation = false - } else { - performValidation = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - performValidation = false - } - - paramList, ok = request.URL.Query()["range"] - if ok { - if 0 == len(paramList) { - percentRange = "0-100" - } else { - percentRange = paramList[0] - if "" == percentRange { - percentRange = "0-100" - } - } - } else { - percentRange = "0-100" - } - - paramList, ok = request.URL.Query()["start"] - if ok { - if 0 == len(paramList) { - startNonceAsUint64 = uint64(0) - } else { - startNonceAsString = paramList[0] - startNonceAsUint64, err = strconv.ParseUint(startNonceAsString, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - } - } else { - startNonceAsUint64 = uint64(0) - } - - if 1 == numPathParts { - volumeListLen, err = globals.volumeLLRB.Len() - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - volumeList = make([]string, 0, volumeListLen) - for volumeListIndex = 0; volumeListIndex < volumeListLen; volumeListIndex++ { - // GetByIndex(index int) (key Key, value Value, ok bool, err error) - volumeNameAsKey, _, ok, err = globals.volumeLLRB.GetByIndex(volumeListIndex) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - err = fmt.Errorf("httpserver.doGetOfVolume() indexing globals.volumeLLRB failed") - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - volumeName = volumeNameAsKey.(string) - volumeList = append(volumeList, volumeName) - } - - if formatResponseAsJSON { - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - volumeListJSONPacked, err = json.Marshal(volumeList) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - if formatResponseCompactly { - _, _ = responseWriter.Write(volumeListJSONPacked) - } else { - json.Indent(&volumeListJSON, volumeListJSONPacked, "", "\t") - _, _ = responseWriter.Write(volumeListJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(volumeListTopTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort))) - - for volumeListIndex, volumeName = range volumeList { - _, _ = responseWriter.Write([]byte(fmt.Sprintf(volumeListPerVolumeTemplate, volumeName))) - } - - _, _ = responseWriter.Write([]byte(volumeListBottom)) - } - - return - } - - // If we reach here, numPathParts is at least 3 - - volumeName = pathSplit[2] - - volumeAsValue, ok, err = globals.volumeLLRB.GetByKey(volumeName) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - requestState = &requestStateStruct{ - pathSplit: pathSplit, - numPathParts: numPathParts, - formatResponseAsJSON: formatResponseAsJSON, - formatResponseCompactly: formatResponseCompactly, - performValidation: performValidation, - percentRange: percentRange, - startNonce: startNonceAsUint64, - volume: volumeAsValue.(*volumeStruct), - } - - requestState.pathSplit = pathSplit - requestState.numPathParts = numPathParts - requestState.formatResponseAsJSON = formatResponseAsJSON - requestState.formatResponseCompactly = formatResponseCompactly - - switch pathSplit[3] { - case "defrag": - doDefrag(responseWriter, request, requestState) - - case "fetch-ondisk-inode": - doFetchOnDiskInode(responseWriter, request, requestState) - - case "fetch-ondisk-metadata-object": - doFetchOnDiskMetaDataObject(responseWriter, request, requestState) - - case "find-dir-inode": - doFindDirInode(responseWriter, request, requestState) - - case "find-subdir-inodes": - doFindSubDirInodes(responseWriter, request, requestState) - - case "extent-map": - doExtentMap(responseWriter, request, requestState) - - case "fsck-job": - doJob(fsckJobType, responseWriter, request, requestState) - - case "layout-report": - doLayoutReport(responseWriter, request, requestState) - - case "lease-report": - doLeaseReport(responseWriter, request, requestState) - - case "meta-defrag": - doMetaDefrag(responseWriter, request, requestState) - - case "scrub-job": - doJob(scrubJobType, responseWriter, request, requestState) - - case "snapshot": - doGetOfSnapShot(responseWriter, request, requestState) - - default: - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - return -} - -func doFetchOnDiskInode(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - corruptionDetected inode.CorruptionDetected - err error - inodeNumber inode.InodeNumber - inodeNumberAsUint64 uint64 - onDiskInode []byte - version inode.Version - ) - - if 4 != requestState.numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - inodeNumberAsUint64, err = strconv.ParseUint(requestState.pathSplit[4], 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - inodeNumber = inode.InodeNumber(inodeNumberAsUint64) - - corruptionDetected, version, onDiskInode, err = requestState.volume.inodeVolumeHandle.FetchOnDiskInode(inodeNumber) - - responseWriter.Header().Set("Content-Type", "text/plain") - - if nil == err { - responseWriter.WriteHeader(http.StatusOK) - } else { - responseWriter.WriteHeader(http.StatusInternalServerError) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(" err: %v\n", err))) - _, _ = responseWriter.Write([]byte(fmt.Sprintf("\n"))) - } - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("corruptionDetected: %v\n", corruptionDetected))) - _, _ = responseWriter.Write([]byte(fmt.Sprintf(" version: %v\n", version))) - _, _ = responseWriter.Write([]byte(fmt.Sprintf("\n"))) - _, _ = responseWriter.Write([]byte(fmt.Sprintf(" onDiskInode: %s\n", string(onDiskInode[:])))) -} - -func doFetchOnDiskMetaDataObject(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - bytesThisLine uint64 - err error - objectOffset uint64 - objectNumber uint64 - onDiskMetaDataObject []byte - ) - - if 4 != requestState.numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - objectNumber, err = strconv.ParseUint(requestState.pathSplit[4], 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - onDiskMetaDataObject, err = requestState.volume.headhunterVolumeHandle.GetBPlusTreeObject(objectNumber) - - responseWriter.Header().Set("Content-Type", "text/plain") - - if nil == err { - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("len(onDiskMetaDataObject): 0x%08X\n", len(onDiskMetaDataObject)))) - _, _ = responseWriter.Write([]byte(fmt.Sprintf("\n"))) - _, _ = responseWriter.Write([]byte(fmt.Sprintf("onDiskMetaDataObject:\n"))) - _, _ = responseWriter.Write([]byte(fmt.Sprintf("\n"))) - - objectOffset = 0 - - for objectOffset < uint64(len(onDiskMetaDataObject)) { - _, _ = responseWriter.Write([]byte(fmt.Sprintf("%08X: ", objectOffset))) - - bytesThisLine = 0 - - for (bytesThisLine < 16) && (objectOffset < uint64(len(onDiskMetaDataObject))) { - _, _ = responseWriter.Write([]byte(fmt.Sprintf(" %02X", onDiskMetaDataObject[objectOffset]))) - bytesThisLine++ - objectOffset++ - } - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("\n"))) - } - } else { - responseWriter.WriteHeader(http.StatusInternalServerError) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("%v\n", err))) - } -} - -func doFindDirInode(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - parentDirEntryBasename string - dirInodeNumber inode.InodeNumber - err error - parentDirInodeNumber inode.InodeNumber - pathSplitIndex int - pathSplitIndexMax int - ) - - if 3 > requestState.numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - if "" == requestState.pathSplit[len(requestState.pathSplit)-1] { - pathSplitIndexMax = len(requestState.pathSplit) - 2 - } else { - pathSplitIndexMax = len(requestState.pathSplit) - 1 - } - - parentDirInodeNumber = inode.RootDirInodeNumber - parentDirEntryBasename = "." - dirInodeNumber = inode.RootDirInodeNumber - - for pathSplitIndex = 4; pathSplitIndex <= pathSplitIndexMax; pathSplitIndex++ { - parentDirInodeNumber = dirInodeNumber - parentDirEntryBasename = requestState.pathSplit[pathSplitIndex] - - dirInodeNumber, err = requestState.volume.fsVolumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInodeNumber, parentDirEntryBasename) - if nil != err { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - } - - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("(%016X) %s => %016X\n", parentDirInodeNumber, parentDirEntryBasename, dirInodeNumber))) -} - -func doFindSubDirInodes(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - dirInodeNumber inode.InodeNumber - dirInodeNumberAsUint64 uint64 - err error - lastInodeNumberAsUint64 uint64 - nextInodeNumberAsUint64 uint64 - ok bool - subDirInodeNumber inode.InodeNumber - subDirInodeNumbers []inode.InodeNumber - subDirParentInodeNumber inode.InodeNumber - ) - - if 4 != requestState.numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - dirInodeNumberAsUint64, err = strconv.ParseUint(requestState.pathSplit[4], 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - dirInodeNumber = inode.InodeNumber(dirInodeNumberAsUint64) - - globals.Unlock() - - subDirInodeNumbers = make([]inode.InodeNumber, 0) - - lastInodeNumberAsUint64 = requestState.startNonce - - for { - nextInodeNumberAsUint64, ok, err = requestState.volume.headhunterVolumeHandle.NextInodeNumber(lastInodeNumberAsUint64) - if nil != err { - logger.FatalfWithError(err, "Unexpected failure walking Inode Table") - } - - if !ok { - break - } - - subDirInodeNumber = inode.InodeNumber(nextInodeNumberAsUint64) - - subDirParentInodeNumber, err = requestState.volume.fsVolumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, subDirInodeNumber, "..") - if (nil == err) && (subDirParentInodeNumber == dirInodeNumber) { - subDirInodeNumbers = append(subDirInodeNumbers, subDirInodeNumber) - } - - lastInodeNumberAsUint64 = nextInodeNumberAsUint64 - } - - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - for _, subDirInodeNumber = range subDirInodeNumbers { - _, _ = responseWriter.Write([]byte(fmt.Sprintf("%016X\n", subDirInodeNumber))) - } - - globals.Lock() -} - -func doMetaDefrag(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - bPlusTreeType string - bPlusTreeTypeSlice []string - err error - percentRangeSplit []string - percentRangeStartAsFloat64 float64 - percentRangeStopAsFloat64 float64 - ) - - if 3 == requestState.numPathParts { - bPlusTreeTypeSlice = []string{"InodeRecBPlusTree", "LogSegmentRecBPlusTree", "BPlusTreeObjectBPlusTree", "CreatedObjectsBPlusTree", "DeletedObjectsBPlusTree"} - } else if 4 == requestState.numPathParts { - bPlusTreeTypeSlice = []string{requestState.pathSplit[4]} - } else { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - percentRangeSplit = strings.Split(requestState.percentRange, "-") - - if 2 != len(percentRangeSplit) { - responseWriter.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return - } - - percentRangeStartAsFloat64, err = strconv.ParseFloat(percentRangeSplit[0], 64) - if (nil != err) || (0 > percentRangeStartAsFloat64) || (100 <= percentRangeStartAsFloat64) { - responseWriter.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return - } - - percentRangeStopAsFloat64, err = strconv.ParseFloat(percentRangeSplit[1], 64) - if (nil != err) || (percentRangeStartAsFloat64 >= percentRangeStopAsFloat64) || (100 < percentRangeStopAsFloat64) { - responseWriter.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return - } - - for _, bPlusTreeType = range bPlusTreeTypeSlice { - switch bPlusTreeType { - case "InodeRecBPlusTree": - err = requestState.volume.headhunterVolumeHandle.DefragmentMetadata( - headhunter.InodeRecBPlusTree, - percentRangeStartAsFloat64, - percentRangeStopAsFloat64) - case "LogSegmentRecBPlusTree": - err = requestState.volume.headhunterVolumeHandle.DefragmentMetadata( - headhunter.LogSegmentRecBPlusTree, - percentRangeStartAsFloat64, - percentRangeStopAsFloat64) - case "BPlusTreeObjectBPlusTree": - err = requestState.volume.headhunterVolumeHandle.DefragmentMetadata( - headhunter.BPlusTreeObjectBPlusTree, - percentRangeStartAsFloat64, - percentRangeStopAsFloat64) - case "CreatedObjectsBPlusTree": - err = requestState.volume.headhunterVolumeHandle.DefragmentMetadata( - headhunter.CreatedObjectsBPlusTree, - percentRangeStartAsFloat64, - percentRangeStopAsFloat64) - case "DeletedObjectsBPlusTree": - err = requestState.volume.headhunterVolumeHandle.DefragmentMetadata( - headhunter.DeletedObjectsBPlusTree, - percentRangeStartAsFloat64, - percentRangeStopAsFloat64) - default: - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - if nil != err { - logger.Fatalf("Call to %s.headhunterVolumeHandle.DefragmentMetadata(%s,,) failed: %v", requestState.volume.name, bPlusTreeType, err) - responseWriter.WriteHeader(http.StatusInternalServerError) - return - } - } - - responseWriter.WriteHeader(http.StatusNoContent) -} - -func doDefrag(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - alreadyInActiveDefragInodeNumberSet bool - dirEntryInodeNumber inode.InodeNumber - dirInodeNumber inode.InodeNumber - err error - pathPartIndex int - ) - - if 3 > requestState.numPathParts { - err = fmt.Errorf("doDefrag() not passed enough requestState.numPathParts (%d)", requestState.numPathParts) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - dirEntryInodeNumber = inode.RootDirInodeNumber - pathPartIndex = 3 - - for ; pathPartIndex < requestState.numPathParts; pathPartIndex++ { - dirInodeNumber = dirEntryInodeNumber - - dirEntryInodeNumber, err = requestState.volume.fsVolumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirInodeNumber, requestState.pathSplit[pathPartIndex+1]) - if nil != err { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - } - - _, alreadyInActiveDefragInodeNumberSet = requestState.volume.activeDefragInodeNumberSet[dirEntryInodeNumber] - if alreadyInActiveDefragInodeNumberSet { - responseWriter.WriteHeader(http.StatusConflict) - return - } - - requestState.volume.activeDefragInodeNumberSet[dirEntryInodeNumber] = struct{}{} - - globals.Unlock() - - err = requestState.volume.fsVolumeHandle.DefragmentFile(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirEntryInodeNumber) - - if nil == err { - responseWriter.WriteHeader(http.StatusOK) - } else { - responseWriter.WriteHeader(http.StatusConflict) - } - - globals.Lock() - - delete(requestState.volume.activeDefragInodeNumberSet, dirEntryInodeNumber) -} - -func doExtentMap(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - dirEntryInodeNumber inode.InodeNumber - dirInodeNumber inode.InodeNumber - err error - extentMap []ExtentMapElementStruct - extentMapChunk *inode.ExtentMapChunkStruct - extentMapEntry inode.ExtentMapEntryStruct - extentMapJSONBuffer bytes.Buffer - extentMapJSONPacked []byte - path string - pathDoubleQuoted string - pathPartIndex int - ) - - if 3 > requestState.numPathParts { - err = fmt.Errorf("doExtentMap() not passed enough requestState.numPathParts (%d)", requestState.numPathParts) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - if (3 == requestState.numPathParts) && requestState.formatResponseAsJSON { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - if 3 == requestState.numPathParts { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(extentMapTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, requestState.volume.name, "null", "null", "false"))) - return - } - - path = strings.Join(requestState.pathSplit[4:], "/") - pathDoubleQuoted = "\"" + path + "\"" - - dirEntryInodeNumber = inode.RootDirInodeNumber - pathPartIndex = 3 - - for ; pathPartIndex < requestState.numPathParts; pathPartIndex++ { - dirInodeNumber = dirEntryInodeNumber - dirEntryInodeNumber, err = requestState.volume.fsVolumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirInodeNumber, requestState.pathSplit[pathPartIndex+1]) - if nil != err { - if requestState.formatResponseAsJSON { - responseWriter.WriteHeader(http.StatusNotFound) - return - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(extentMapTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, requestState.volume.name, "null", pathDoubleQuoted, "true"))) - return - } - } - } - - extentMapChunk, err = requestState.volume.fsVolumeHandle.FetchExtentMapChunk(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirEntryInodeNumber, uint64(0), math.MaxInt64, int64(0)) - if nil != err { - if requestState.formatResponseAsJSON { - responseWriter.WriteHeader(http.StatusNotFound) - return - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(extentMapTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, requestState.volume.name, "null", pathDoubleQuoted, "true"))) - return - } - } - - extentMap = make([]ExtentMapElementStruct, 0, len(extentMapChunk.ExtentMapEntry)) - - for _, extentMapEntry = range extentMapChunk.ExtentMapEntry { - extentMap = append(extentMap, ExtentMapElementStruct{ - FileOffset: extentMapEntry.FileOffset, - ContainerName: extentMapEntry.ContainerName, - ObjectName: extentMapEntry.ObjectName, - ObjectOffset: extentMapEntry.LogSegmentOffset, - Length: extentMapEntry.Length, - }) - } - - extentMapJSONPacked, err = json.Marshal(extentMap) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - if requestState.formatResponseAsJSON { - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if requestState.formatResponseCompactly { - _, _ = responseWriter.Write(extentMapJSONPacked) - } else { - json.Indent(&extentMapJSONBuffer, extentMapJSONPacked, "", "\t") - _, _ = responseWriter.Write(extentMapJSONBuffer.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(extentMapTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, requestState.volume.name, utils.ByteSliceToString(extentMapJSONPacked), pathDoubleQuoted, "false"))) - } -} - -func doJob(jobType jobTypeType, responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - err error - formatResponseAsJSON bool - formatResponseCompactly bool - inactive bool - job *jobStruct - jobAsValue sortedmap.Value - jobErrorList []string - jobID uint64 - jobIDAsKey sortedmap.Key - jobPerJobTemplate string - jobsIDListJSONBuffer bytes.Buffer - jobsIDListJSONPacked []byte - jobStatusJSONBuffer bytes.Buffer - jobStatusJSONPacked []byte - jobStatusJSONStruct *JobStatusJSONPackedStruct - jobsCount int - jobsIDList []uint64 - jobsIndex int - ok bool - numPathParts int - pathSplit []string - volume *volumeStruct - volumeName string - ) - - if limitJobType <= jobType { - err = fmt.Errorf("httpserver.doJob(jobtype==%v,,,) called for invalid jobType", jobType) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - volume = requestState.volume - pathSplit = requestState.pathSplit - numPathParts = requestState.numPathParts - formatResponseAsJSON = requestState.formatResponseAsJSON - formatResponseCompactly = requestState.formatResponseCompactly - - volumeName = volume.name - volume.Lock() - - markJobsCompletedIfNoLongerActiveWhileLocked(volume) - - if 3 == numPathParts { - switch jobType { - case fsckJobType: - jobsCount, err = volume.fsckJobs.Len() - case scrubJobType: - jobsCount, err = volume.scrubJobs.Len() - } - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - inactive = (nil == volume.fsckActiveJob) && (nil == volume.scrubActiveJob) - - volume.Unlock() - - if formatResponseAsJSON { - jobsIDList = make([]uint64, 0, jobsCount) - - for jobsIndex = jobsCount - 1; jobsIndex >= 0; jobsIndex-- { - switch jobType { - case fsckJobType: - jobIDAsKey, _, ok, err = volume.fsckJobs.GetByIndex(jobsIndex) - case scrubJobType: - jobIDAsKey, _, ok, err = volume.scrubJobs.GetByIndex(jobsIndex) - } - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - switch jobType { - case fsckJobType: - err = fmt.Errorf("httpserver.doGetOfVolume() indexing volume.fsckJobs failed") - case scrubJobType: - err = fmt.Errorf("httpserver.doGetOfVolume() indexing volume.scrubJobs failed") - } - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - jobID = jobIDAsKey.(uint64) - - jobsIDList = append(jobsIDList, jobID) - } - - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - jobsIDListJSONPacked, err = json.Marshal(jobsIDList) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - if formatResponseCompactly { - _, _ = responseWriter.Write(jobsIDListJSONPacked) - } else { - json.Indent(&jobsIDListJSONBuffer, jobsIDListJSONPacked, "", "\t") - _, _ = responseWriter.Write(jobsIDListJSONBuffer.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - switch jobType { - case fsckJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobsTopTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, volumeName, "FSCK"))) - case scrubJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobsTopTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, volumeName, "SCRUB"))) - } - - for jobsIndex = jobsCount - 1; jobsIndex >= 0; jobsIndex-- { - switch jobType { - case fsckJobType: - jobIDAsKey, jobAsValue, ok, err = volume.fsckJobs.GetByIndex(jobsIndex) - case scrubJobType: - jobIDAsKey, jobAsValue, ok, err = volume.scrubJobs.GetByIndex(jobsIndex) - } - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - switch jobType { - case fsckJobType: - err = fmt.Errorf("httpserver.doGetOfVolume() indexing volume.fsckJobs failed") - case scrubJobType: - err = fmt.Errorf("httpserver.doGetOfVolume() indexing volume.scrubJobs failed") - } - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - jobID = jobIDAsKey.(uint64) - job = jobAsValue.(*jobStruct) - - if jobRunning == job.state { - switch jobType { - case fsckJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobsPerRunningJobTemplate, jobID, job.startTime.Format(time.RFC3339), volumeName, "fsck"))) - case scrubJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobsPerRunningJobTemplate, jobID, job.startTime.Format(time.RFC3339), volumeName, "scrub"))) - } - } else { - switch job.state { - case jobHalted: - jobPerJobTemplate = jobsPerHaltedJobTemplate - case jobCompleted: - jobErrorList = job.jobHandle.Error() - if 0 == len(jobErrorList) { - jobPerJobTemplate = jobsPerSuccessfulJobTemplate - } else { - jobPerJobTemplate = jobsPerFailedJobTemplate - } - } - - switch jobType { - case fsckJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobPerJobTemplate, jobID, job.startTime.Format(time.RFC3339), job.endTime.Format(time.RFC3339), volumeName, "fsck"))) - case scrubJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobPerJobTemplate, jobID, job.startTime.Format(time.RFC3339), job.endTime.Format(time.RFC3339), volumeName, "scrub"))) - } - } - } - - _, _ = responseWriter.Write([]byte(jobsListBottom)) - - if inactive { - switch jobType { - case fsckJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobsStartJobButtonTemplate, volumeName, "fsck"))) - case scrubJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobsStartJobButtonTemplate, volumeName, "scrub"))) - } - } - - _, _ = responseWriter.Write([]byte(jobsBottom)) - } - - return - } - - // If we reach here, numPathParts is 4 - - jobID, err = strconv.ParseUint(pathSplit[4], 10, 64) - if nil != err { - volume.Unlock() - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - switch jobType { - case fsckJobType: - jobAsValue, ok, err = volume.fsckJobs.GetByKey(jobID) - case scrubJobType: - jobAsValue, ok, err = volume.scrubJobs.GetByKey(jobID) - } - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - volume.Unlock() - responseWriter.WriteHeader(http.StatusNotFound) - return - } - job = jobAsValue.(*jobStruct) - - jobStatusJSONStruct = &JobStatusJSONPackedStruct{ - StartTime: job.startTime.Format(time.RFC3339), - ErrorList: job.jobHandle.Error(), - InfoList: job.jobHandle.Info(), - } - - switch job.state { - case jobRunning: - // Nothing to add here - case jobHalted: - jobStatusJSONStruct.HaltTime = job.endTime.Format(time.RFC3339) - case jobCompleted: - jobStatusJSONStruct.DoneTime = job.endTime.Format(time.RFC3339) - } - - jobStatusJSONPacked, err = json.Marshal(jobStatusJSONStruct) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - if formatResponseAsJSON { - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if formatResponseCompactly { - _, _ = responseWriter.Write(jobStatusJSONPacked) - } else { - json.Indent(&jobStatusJSONBuffer, jobStatusJSONPacked, "", "\t") - _, _ = responseWriter.Write(jobStatusJSONBuffer.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - switch jobType { - case fsckJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, volumeName, "FSCK", "fsck", job.id, utils.ByteSliceToString(jobStatusJSONPacked)))) - case scrubJobType: - _, _ = responseWriter.Write([]byte(fmt.Sprintf(jobTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, volumeName, "SCRUB", "scrub", job.id, utils.ByteSliceToString(jobStatusJSONPacked)))) - } - } - - volume.Unlock() -} - -type layoutReportElementLayoutReportElementStruct struct { - ObjectNumber uint64 - ObjectBytes uint64 -} - -type layoutReportSetElementStruct struct { - TreeName string - Discrepencies uint64 - LayoutReport []layoutReportElementLayoutReportElementStruct -} - -type layoutReportSetElementWithoutDiscrepenciesStruct struct { - TreeName string - LayoutReport []layoutReportElementLayoutReportElementStruct -} - -func doLayoutReport(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - discrepencyFormatClass string - err error - layoutReportIndex int - layoutReportMap sortedmap.LayoutReport - layoutReportSet [6]*layoutReportSetElementStruct - layoutReportSetElement *layoutReportSetElementStruct - layoutReportSetJSON bytes.Buffer - layoutReportSetJSONPacked []byte - layoutReportSetWithoutDiscrepencies [6]*layoutReportSetElementWithoutDiscrepenciesStruct - objectBytes uint64 - objectNumber uint64 - treeTypeIndex int - ) - - layoutReportSet[headhunter.MergedBPlusTree] = &layoutReportSetElementStruct{ - TreeName: "Checkpoint (Trailer + B+Trees)", - } - layoutReportSet[headhunter.InodeRecBPlusTree] = &layoutReportSetElementStruct{ - TreeName: "Inode Record B+Tree", - } - layoutReportSet[headhunter.LogSegmentRecBPlusTree] = &layoutReportSetElementStruct{ - TreeName: "Log Segment Record B+Tree", - } - layoutReportSet[headhunter.BPlusTreeObjectBPlusTree] = &layoutReportSetElementStruct{ - TreeName: "B+Plus Tree Objects B+Tree", - } - layoutReportSet[headhunter.CreatedObjectsBPlusTree] = &layoutReportSetElementStruct{ - TreeName: "Created Objects B+Tree", - } - layoutReportSet[headhunter.DeletedObjectsBPlusTree] = &layoutReportSetElementStruct{ - TreeName: "Deleted Objects B+Tree", - } - - for treeTypeIndex, layoutReportSetElement = range layoutReportSet { - layoutReportMap, layoutReportSetElement.Discrepencies, err = requestState.volume.headhunterVolumeHandle.FetchLayoutReport(headhunter.BPlusTreeType(treeTypeIndex), requestState.performValidation) - if nil != err { - responseWriter.WriteHeader(http.StatusInternalServerError) - return - } - - layoutReportSetElement.LayoutReport = make([]layoutReportElementLayoutReportElementStruct, len(layoutReportMap)) - - layoutReportIndex = 0 - - for objectNumber, objectBytes = range layoutReportMap { - layoutReportSetElement.LayoutReport[layoutReportIndex] = layoutReportElementLayoutReportElementStruct{objectNumber, objectBytes} - layoutReportIndex++ - } - } - - if requestState.formatResponseAsJSON { - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if requestState.performValidation { - layoutReportSetJSONPacked, err = json.Marshal(layoutReportSet) - } else { - for treeTypeIndex, layoutReportSetElement = range layoutReportSet { - layoutReportSetWithoutDiscrepencies[treeTypeIndex] = &layoutReportSetElementWithoutDiscrepenciesStruct{ - TreeName: layoutReportSetElement.TreeName, - LayoutReport: layoutReportSetElement.LayoutReport, - } - } - layoutReportSetJSONPacked, err = json.Marshal(layoutReportSetWithoutDiscrepencies) - } - if nil != err { - responseWriter.WriteHeader(http.StatusInternalServerError) - return - } - - if requestState.formatResponseCompactly { - _, _ = responseWriter.Write(layoutReportSetJSONPacked) - } else { - json.Indent(&layoutReportSetJSON, layoutReportSetJSONPacked, "", "\t") - _, _ = responseWriter.Write(layoutReportSetJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(layoutReportTopTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, requestState.volume.name))) - - for _, layoutReportSetElement = range layoutReportSet { - if 0 == layoutReportSetElement.Discrepencies { - discrepencyFormatClass = "success" - } else { - discrepencyFormatClass = "danger" - } - _, _ = responseWriter.Write([]byte(fmt.Sprintf(layoutReportTableTopTemplate, layoutReportSetElement.TreeName, layoutReportSetElement.Discrepencies, discrepencyFormatClass))) - - for layoutReportIndex = 0; layoutReportIndex < len(layoutReportSetElement.LayoutReport); layoutReportIndex++ { - _, _ = responseWriter.Write([]byte(fmt.Sprintf(layoutReportTableRowTemplate, layoutReportSetElement.LayoutReport[layoutReportIndex].ObjectNumber, layoutReportSetElement.LayoutReport[layoutReportIndex].ObjectBytes))) - } - - _, _ = responseWriter.Write([]byte(layoutReportTableBottom)) - } - - _, _ = responseWriter.Write([]byte(layoutReportBottom)) - } -} - -func doLeaseReport(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - err error - leaseReport []*jrpcfs.LeaseReportElementStruct - leaseReportJSON bytes.Buffer - leaseReportJSONPacked []byte - ) - - leaseReport, err = jrpcfs.FetchLeaseReport(requestState.volume.name) - if nil != err { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - leaseReportJSONPacked, err = json.Marshal(leaseReport) - if nil != err { - responseWriter.WriteHeader(http.StatusInternalServerError) - return - } - - // TODO: Need to align this logic and cleanly format page... but for now... - - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if requestState.formatResponseCompactly { - _, _ = responseWriter.Write(leaseReportJSONPacked) - } else { - json.Indent(&leaseReportJSON, leaseReportJSONPacked, "", "\t") - _, _ = responseWriter.Write(leaseReportJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } -} - -func doGetOfSnapShot(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - directionStringCanonicalized string - directionStringSlice []string - err error - list []headhunter.SnapShotStruct - listJSON bytes.Buffer - listJSONPacked []byte - orderByStringCanonicalized string - orderByStringSlice []string - queryValues url.Values - reversed bool - snapShot headhunter.SnapShotStruct - ) - - queryValues = request.URL.Query() - - directionStringSlice = queryValues["direction"] - - if 0 == len(directionStringSlice) { - reversed = false - } else { - directionStringCanonicalized = strings.ToLower(directionStringSlice[0]) - if "desc" == directionStringCanonicalized { - reversed = true - } else { - reversed = false - } - } - - orderByStringSlice = queryValues["orderby"] - - if 0 == len(orderByStringSlice) { - list = requestState.volume.headhunterVolumeHandle.SnapShotListByTime(reversed) - } else { - orderByStringCanonicalized = strings.ToLower(orderByStringSlice[0]) - switch orderByStringCanonicalized { - case "id": - list = requestState.volume.headhunterVolumeHandle.SnapShotListByID(reversed) - case "name": - list = requestState.volume.headhunterVolumeHandle.SnapShotListByName(reversed) - default: // assume "time" - list = requestState.volume.headhunterVolumeHandle.SnapShotListByTime(reversed) - } - } - - if requestState.formatResponseAsJSON { - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - listJSONPacked, err = json.Marshal(list) - if nil != err { - responseWriter.WriteHeader(http.StatusInternalServerError) - return - } - - if requestState.formatResponseCompactly { - _, _ = responseWriter.Write(listJSONPacked) - } else { - json.Indent(&listJSON, listJSONPacked, "", "\t") - _, _ = responseWriter.Write(listJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } - } else { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(snapShotsTopTemplate, version.ProxyFSVersion, globals.ipAddrTCPPort, requestState.volume.name))) - - for _, snapShot = range list { - _, _ = responseWriter.Write([]byte(fmt.Sprintf(snapShotsPerSnapShotTemplate, snapShot.ID, snapShot.Time.Format(time.RFC3339), snapShot.Name))) - } - - _, _ = responseWriter.Write([]byte(fmt.Sprintf(snapShotsBottomTemplate, requestState.volume.name))) - } -} - -func doPost(responseWriter http.ResponseWriter, request *http.Request) { - switch { - case strings.HasPrefix(request.URL.Path, "/deletions"): - doPostOfDeletions(responseWriter, request) - case strings.HasPrefix(request.URL.Path, "/trigger"): - doPostOfTrigger(responseWriter, request) - case strings.HasPrefix(request.URL.Path, "/volume"): - doPostOfVolume(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func doPostOfDeletions(responseWriter http.ResponseWriter, request *http.Request) { - var ( - numPathParts int - pathSplit []string - ) - - pathSplit = strings.Split(request.URL.Path, "/") // leading "/" places "" in pathSplit[0] - // pathSplit[1] should be "deletions" based on how we got here - // trailing "/" places "" in pathSplit[len(pathSplit)-1] - - numPathParts = len(pathSplit) - 1 - if "" == pathSplit[numPathParts] { - numPathParts-- - } - - if "deletions" != pathSplit[1] { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - switch numPathParts { - case 2: - // Form: /deletions/{pause|resume} - - switch pathSplit[2] { - case "pause": - headhunter.DisableObjectDeletions() - responseWriter.WriteHeader(http.StatusNoContent) - case "resume": - headhunter.EnableObjectDeletions() - responseWriter.WriteHeader(http.StatusNoContent) - default: - responseWriter.WriteHeader(http.StatusNotFound) - } - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func doPostOfTrigger(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - haltTriggerCountAsString string - haltTriggerCountAsU32 uint32 - haltTriggerCountAsU64 uint64 - haltTriggerString string - numPathParts int - pathSplit []string - ) - - pathSplit = strings.Split(request.URL.Path, "/") // leading "/" places "" in pathSplit[0] - // pathSplit[1] should be "trigger" based on how we got here - // trailing "/" places "" in pathSplit[len(pathSplit)-1] - - numPathParts = len(pathSplit) - 1 - if "" == pathSplit[numPathParts] { - numPathParts-- - } - - if "trigger" != pathSplit[1] { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - switch numPathParts { - case 2: - // Form: /trigger/ - - haltTriggerString = pathSplit[2] - - _, err = halter.Stat(haltTriggerString) - if nil != err { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - haltTriggerCountAsString = request.FormValue("count") - - haltTriggerCountAsU64, err = strconv.ParseUint(haltTriggerCountAsString, 10, 32) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - haltTriggerCountAsU32 = uint32(haltTriggerCountAsU64) - - if 0 == haltTriggerCountAsU32 { - halter.Disarm(haltTriggerString) - } else { - halter.Arm(haltTriggerString, haltTriggerCountAsU32) - } - - responseWriter.WriteHeader(http.StatusNoContent) - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func doPostOfVolume(responseWriter http.ResponseWriter, request *http.Request) { - var ( - acceptHeader string - err error - job *jobStruct - jobAsValue sortedmap.Value - jobID uint64 - jobType jobTypeType - jobsCount int - numPathParts int - ok bool - pathSplit []string - volume *volumeStruct - volumeAsValue sortedmap.Value - volumeName string - ) - - pathSplit = strings.Split(request.URL.Path, "/") // leading "/" places "" in pathSplit[0] - // pathSplit[1] should be "volume" based on how we got here - // trailing "/" places "" in pathSplit[len(pathSplit)-1] - - numPathParts = len(pathSplit) - 1 - if "" == pathSplit[numPathParts] { - numPathParts-- - } - - if "volume" != pathSplit[1] { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - switch numPathParts { - case 3: - // Form: /volume//add-dir-entry - // Form: /volume//find-file-inodes-matching-lengths - // Form: /volume//fsck-job - // Form: /volume//patch-dir-inode - // Form: /volume//patch-file-inode - // Form: /volume//patch-symlink-inode - // Form: /volume//scrub-job - // Form: /volume//snapshot - case 4: - // Form: /volume//fsck-job/ - // Form: /volume//scrub-job/ - default: - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - volumeName = pathSplit[2] - - volumeAsValue, ok, err = globals.volumeLLRB.GetByKey(volumeName) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - volume = volumeAsValue.(*volumeStruct) - - switch pathSplit[3] { - case "add-dir-entry": - if 3 != numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - doPostOfAddDirEntry(responseWriter, request, volume) - return - case "fsck-job": - jobType = fsckJobType - case "find-file-inodes-matching-lengths": - if 3 != numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - doPostOfFindFileInodesMatchingLengths(responseWriter, request, volume) - return - case "patch-dir-inode": - if 3 != numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - doPostOfPatchDirInode(responseWriter, request, volume) - return - case "patch-file-inode": - if 3 != numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - doPostOfPatchFileInode(responseWriter, request, volume) - return - case "patch-symlink-inode": - if 3 != numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - doPostOfPatchSymlinkInode(responseWriter, request, volume) - return - case "scrub-job": - jobType = scrubJobType - case "snapshot": - if 3 != numPathParts { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - doPostOfSnapShot(responseWriter, request, volume) - return - default: - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - volume.Lock() - - if 3 == numPathParts { - markJobsCompletedIfNoLongerActiveWhileLocked(volume) - - if (nil != volume.fsckActiveJob) || (nil != volume.scrubActiveJob) { - // Cannot start an FSCK or SCRUB job while either is active - - volume.Unlock() - responseWriter.WriteHeader(http.StatusPreconditionFailed) - return - } - - for { - switch jobType { - case fsckJobType: - jobsCount, err = volume.fsckJobs.Len() - case scrubJobType: - jobsCount, err = volume.scrubJobs.Len() - } - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - if jobsCount < int(globals.jobHistoryMaxSize) { - break - } - - switch jobType { - case fsckJobType: - ok, err = volume.fsckJobs.DeleteByIndex(0) - case scrubJobType: - ok, err = volume.scrubJobs.DeleteByIndex(0) - } - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - switch jobType { - case fsckJobType: - err = fmt.Errorf("httpserver.doPostOfVolume() delete of oldest element of volume.fsckJobs failed") - case scrubJobType: - err = fmt.Errorf("httpserver.doPostOfVolume() delete of oldest element of volume.scrubJobs failed") - } - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - } - - job = &jobStruct{ - volume: volume, - state: jobRunning, - startTime: time.Now(), - } - - job.id = volume.headhunterVolumeHandle.FetchNonce() - - switch jobType { - case fsckJobType: - ok, err = volume.fsckJobs.Put(job.id, job) - case scrubJobType: - ok, err = volume.scrubJobs.Put(job.id, job) - } - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - switch jobType { - case fsckJobType: - err = fmt.Errorf("httpserver.doPostOfVolume() PUT to volume.fsckJobs failed") - case scrubJobType: - err = fmt.Errorf("httpserver.doPostOfVolume() PUT to volume.scrubJobs failed") - } - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - switch jobType { - case fsckJobType: - volume.fsckActiveJob = job - - job.jobHandle = fs.ValidateVolume(volumeName) - case scrubJobType: - volume.scrubActiveJob = job - - job.jobHandle = fs.ScrubVolume(volumeName) - } - - volume.Unlock() - - switch jobType { - case fsckJobType: - responseWriter.Header().Set("Location", fmt.Sprintf("/volume/%v/fsck-job/%v", volumeName, job.id)) - case scrubJobType: - responseWriter.Header().Set("Location", fmt.Sprintf("/volume/%v/scrub-job/%v", volumeName, job.id)) - } - - acceptHeader = request.Header.Get("Accept") - - if strings.Contains(acceptHeader, "text/html") { - responseWriter.WriteHeader(http.StatusSeeOther) - } else { - responseWriter.WriteHeader(http.StatusCreated) - } - - return - } - - // If we reach here, numPathParts is 4 - - jobID, err = strconv.ParseUint(pathSplit[4], 10, 64) - if nil != err { - volume.Unlock() - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - switch jobType { - case fsckJobType: - jobAsValue, ok, err = volume.fsckJobs.GetByKey(jobID) - case scrubJobType: - jobAsValue, ok, err = volume.scrubJobs.GetByKey(jobID) - } - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - volume.Unlock() - responseWriter.WriteHeader(http.StatusNotFound) - return - } - job = jobAsValue.(*jobStruct) - - switch jobType { - case fsckJobType: - if volume.fsckActiveJob != job { - volume.Unlock() - responseWriter.WriteHeader(http.StatusPreconditionFailed) - return - } - case scrubJobType: - if volume.scrubActiveJob != job { - volume.Unlock() - responseWriter.WriteHeader(http.StatusPreconditionFailed) - return - } - } - - job.jobHandle.Cancel() - - job.state = jobHalted - job.endTime = time.Now() - - switch jobType { - case fsckJobType: - volume.fsckActiveJob = nil - case scrubJobType: - volume.scrubActiveJob = nil - } - - volume.Unlock() - - responseWriter.WriteHeader(http.StatusNoContent) -} - -func doPostOfSnapShot(responseWriter http.ResponseWriter, request *http.Request, volume *volumeStruct) { - var ( - err error - snapShotID uint64 - ) - - snapShotID, err = volume.inodeVolumeHandle.SnapShotCreate(request.FormValue("name")) - if nil == err { - responseWriter.Header().Set("Location", fmt.Sprintf("/volume/%v/snapshot/%v", volume.name, snapShotID)) - - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusConflict) - } -} - -func doPostOfAddDirEntry(responseWriter http.ResponseWriter, request *http.Request, volume *volumeStruct) { - var ( - dirEntryInodeNumberAsString string - dirEntryInodeNumberAsUint64 uint64 - dirEntryName string - dirInodeNumberAsString string - dirInodeNumberAsUint64 uint64 - err error - skipDirEntryLinkCountIncrementOnNonSubDirEntry bool - skipDirEntryLinkCountIncrementOnNonSubDirEntryValue string - skipDirLinkCountIncrementOnSubDirEntry bool - skipDirLinkCountIncrementOnSubDirEntryValue string - skipSettingDotDotOnSubDirEntry bool - skipSettingDotDotOnSubDirEntryValue string - ) - - err = request.ParseForm() - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - dirInodeNumberAsString = request.Form.Get("dirInodeNumber") - if "" == dirInodeNumberAsString { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - dirInodeNumberAsUint64, err = strconv.ParseUint(dirInodeNumberAsString, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - dirEntryName = request.Form.Get("dirEntryName") - if "" == dirEntryName { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - dirEntryInodeNumberAsString = request.Form.Get("dirEntryInodeNumber") - if "" == dirEntryInodeNumberAsString { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - dirEntryInodeNumberAsUint64, err = strconv.ParseUint(dirEntryInodeNumberAsString, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - _, skipDirLinkCountIncrementOnSubDirEntry = request.Form["skipDirLinkCountIncrementOnSubDirEntry"] - if skipDirLinkCountIncrementOnSubDirEntry { - skipDirLinkCountIncrementOnSubDirEntryValue = request.Form.Get("skipDirLinkCountIncrementOnSubDirEntry") - switch skipDirLinkCountIncrementOnSubDirEntryValue { - case "0": - skipDirLinkCountIncrementOnSubDirEntry = false - case "false": - skipDirLinkCountIncrementOnSubDirEntry = false - } - } - - _, skipSettingDotDotOnSubDirEntry = request.Form["skipSettingDotDotOnSubDirEntry"] - if skipSettingDotDotOnSubDirEntry { - skipSettingDotDotOnSubDirEntryValue = request.Form.Get("skipSettingDotDotOnSubDirEntry") - switch skipSettingDotDotOnSubDirEntryValue { - case "0": - skipSettingDotDotOnSubDirEntry = false - case "false": - skipSettingDotDotOnSubDirEntry = false - } - } - - _, skipDirEntryLinkCountIncrementOnNonSubDirEntry = request.Form["skipDirEntryLinkCountIncrementOnNonSubDirEntry"] - if skipDirEntryLinkCountIncrementOnNonSubDirEntry { - skipDirEntryLinkCountIncrementOnNonSubDirEntryValue = request.Form.Get("skipDirEntryLinkCountIncrementOnNonSubDirEntry") - switch skipDirEntryLinkCountIncrementOnNonSubDirEntryValue { - case "0": - skipDirEntryLinkCountIncrementOnNonSubDirEntry = false - case "false": - skipDirEntryLinkCountIncrementOnNonSubDirEntry = false - } - } - - err = volume.inodeVolumeHandle.AddDirEntry( - inode.InodeNumber(dirInodeNumberAsUint64), - dirEntryName, - inode.InodeNumber(dirEntryInodeNumberAsUint64), - skipDirLinkCountIncrementOnSubDirEntry, - skipSettingDotDotOnSubDirEntry, - skipDirEntryLinkCountIncrementOnNonSubDirEntry) - - if nil == err { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusBadRequest) - } -} - -type fileLengthRangeStruct struct { - fileName string - minLength uint64 - maxLength uint64 - matchingInodeNumbers []uint64 -} - -func (dummy *fileLengthRangeStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - var ( - keyAsUint64 uint64 - ok bool - ) - - keyAsUint64, ok = key.(uint64) - - if ok { - keyAsString = fmt.Sprintf("%016X", keyAsUint64) - err = nil - } else { - err = fmt.Errorf("(*fileLengthRangeStruct).DumpKey() called with unexpected Key: %#v", key) - } - - return -} - -func (dummy *fileLengthRangeStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - valueAsString = "Not implemented - sortedmap.LLRBTree.Dump() doesn't display values" - err = nil - - return -} - -func doPostOfFindFileInodesMatchingLengths(responseWriter http.ResponseWriter, request *http.Request, volume *volumeStruct) { - var ( - byFileNameMap map[string]*fileLengthRangeStruct - byMinLengthSortedMap sortedmap.LLRBTree // key:*fileLengthRangeStruct.minLength value:*fileLengthRangeStruct - byMinLengthSortedMapIndex int - dummyFileLengthRangeStructForLLRBTreeCallbacks fileLengthRangeStruct - err error - exactLength uint64 - fileLengthRange *fileLengthRangeStruct - fileLengthRangeAsValue sortedmap.Value - fileName string - isPrefix bool - lastInodeNumberAsUint64 uint64 - lineAsByteSlice []byte - lineAsString string - matchingInodeNumberAsUint64 uint64 - matchingInodeNumberIndex int - maxLength uint64 - metadata *inode.MetadataStruct - minLength uint64 - nextInodeNumberAsUint64 uint64 - ok bool - paramList []string - reader *bufio.Reader - sscanfItemsParsed int - startNonceAsString string - startNonceAsUint64 uint64 - ) - - paramList, ok = request.URL.Query()["start"] - if ok { - if 0 == len(paramList) { - startNonceAsUint64 = uint64(0) - } else { - startNonceAsString = paramList[0] - startNonceAsUint64, err = strconv.ParseUint(startNonceAsString, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - } - } else { - startNonceAsUint64 = uint64(0) - } - - byFileNameMap = make(map[string]*fileLengthRangeStruct) - - byMinLengthSortedMap = sortedmap.NewLLRBTree(sortedmap.CompareUint64, &dummyFileLengthRangeStructForLLRBTreeCallbacks) - - reader = bufio.NewReader(request.Body) - - for { - lineAsByteSlice, isPrefix, err = reader.ReadLine() - if nil != err { - if io.EOF == err { - break - } - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - if isPrefix { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - lineAsString = string(lineAsByteSlice[:]) - - if (0 == len(lineAsString)) || ('#' == lineAsString[0]) { - continue - } - - sscanfItemsParsed, err = fmt.Sscanf(lineAsString, "%s %016X-%016X", &fileName, &minLength, &maxLength) - if (nil == err) && (3 == sscanfItemsParsed) { - fileLengthRange = &fileLengthRangeStruct{ - fileName: fileName, - minLength: minLength, - maxLength: maxLength, - matchingInodeNumbers: make([]uint64, 0), - } - - _, ok = byFileNameMap[fileName] - if ok { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - byFileNameMap[fileName] = fileLengthRange - - ok, err = byMinLengthSortedMap.Put(minLength, fileLengthRange) - if nil != err { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - if !ok { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - } else { - sscanfItemsParsed, err = fmt.Sscanf(lineAsString, "%s %016X", &fileName, &exactLength) - if (nil == err) && (2 == sscanfItemsParsed) { - fileLengthRange = &fileLengthRangeStruct{ - fileName: fileName, - minLength: exactLength, - maxLength: exactLength, - matchingInodeNumbers: make([]uint64, 0), - } - - _, ok = byFileNameMap[fileName] - if ok { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - byFileNameMap[fileName] = fileLengthRange - - ok, err = byMinLengthSortedMap.Put(exactLength, fileLengthRange) - if nil != err { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - if !ok { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - } else { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - } - } - - err = request.Body.Close() - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - globals.Unlock() - - lastInodeNumberAsUint64 = startNonceAsUint64 - - for { - nextInodeNumberAsUint64, ok, err = volume.headhunterVolumeHandle.NextInodeNumber(lastInodeNumberAsUint64) - if nil != err { - logger.FatalfWithError(err, "Unexpected failure walking Inode Table") - } - - if !ok { - break - } - - metadata, err = volume.inodeVolumeHandle.GetMetadata(inode.InodeNumber(nextInodeNumberAsUint64)) - if nil == err { - if inode.FileType == metadata.InodeType { - exactLength = metadata.Size - - byMinLengthSortedMapIndex = 0 - - for { - _, fileLengthRangeAsValue, ok, err = byMinLengthSortedMap.GetByIndex(byMinLengthSortedMapIndex) - if nil != err { - logger.FatalfWithError(err, "Unexpected failure bisecting byMinLengthSortedMap") - } - - if !ok { - break - } - - fileLengthRange = fileLengthRangeAsValue.(*fileLengthRangeStruct) - - if fileLengthRange.minLength > exactLength { - break - } - - if fileLengthRange.maxLength >= exactLength { - fileLengthRange.matchingInodeNumbers = append(fileLengthRange.matchingInodeNumbers, nextInodeNumberAsUint64) - } - - byMinLengthSortedMapIndex++ - } - } - } - - lastInodeNumberAsUint64 = nextInodeNumberAsUint64 - } - - globals.Lock() - - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - for fileName, fileLengthRange = range byFileNameMap { - _, _ = responseWriter.Write([]byte(fmt.Sprintf("%s [", fileName))) - for matchingInodeNumberIndex, matchingInodeNumberAsUint64 = range fileLengthRange.matchingInodeNumbers { - if 0 == matchingInodeNumberIndex { - _, _ = responseWriter.Write([]byte(fmt.Sprintf("%016X", matchingInodeNumberAsUint64))) - } else { - _, _ = responseWriter.Write([]byte(fmt.Sprintf(" %016X", matchingInodeNumberAsUint64))) - } - } - _, _ = responseWriter.Write([]byte(fmt.Sprintf("]\n"))) - } -} - -func doPostOfPatchDirInode(responseWriter http.ResponseWriter, request *http.Request, volume *volumeStruct) { - var ( - err error - inodeNumberAsString string - inodeNumberAsUint64 uint64 - parentInodeNumberAsString string - parentInodeNumberAsUint64 uint64 - ) - - err = request.ParseForm() - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - inodeNumberAsString = request.Form.Get("inodeNumber") - if "" == inodeNumberAsString { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAsString, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - parentInodeNumberAsString = request.Form.Get("inodeNumber") - if "" == parentInodeNumberAsString { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - parentInodeNumberAsUint64, err = strconv.ParseUint(parentInodeNumberAsString, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - err = volume.inodeVolumeHandle.PatchInode( - inode.InodeNumber(inodeNumberAsUint64), - inode.DirType, - uint64(2), - inode.PosixModePerm, - inode.InodeRootUserID, - inode.InodeGroupID(0), - inode.InodeNumber(parentInodeNumberAsUint64), - "") - - if nil == err { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusInternalServerError) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("err: %v\n", err))) - } -} - -func doPostOfPatchFileInode(responseWriter http.ResponseWriter, request *http.Request, volume *volumeStruct) { - var ( - err error - inodeNumberAsString string - inodeNumberAsUint64 uint64 - linkCountAsString string - linkCountAsUint64 uint64 - ) - - err = request.ParseForm() - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - inodeNumberAsString = request.Form.Get("inodeNumber") - if "" == inodeNumberAsString { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAsString, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - linkCountAsString = request.Form.Get("linkCount") - if "" == linkCountAsString { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - linkCountAsUint64, err = strconv.ParseUint(linkCountAsString, 10, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - err = volume.inodeVolumeHandle.PatchInode( - inode.InodeNumber(inodeNumberAsUint64), - inode.FileType, - linkCountAsUint64, - inode.PosixModePerm, - inode.InodeRootUserID, - inode.InodeGroupID(0), - inode.InodeNumber(0), - "") - - if nil == err { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusInternalServerError) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("err: %v\n", err))) - } -} - -func doPostOfPatchSymlinkInode(responseWriter http.ResponseWriter, request *http.Request, volume *volumeStruct) { - var ( - err error - inodeNumberAsString string - inodeNumberAsUint64 uint64 - linkCountAsString string - linkCountAsUint64 uint64 - symlinkTarget string - ) - - err = request.ParseForm() - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - inodeNumberAsString = request.Form.Get("inodeNumber") - if "" == inodeNumberAsString { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - inodeNumberAsUint64, err = strconv.ParseUint(inodeNumberAsString, 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - linkCountAsString = request.Form.Get("linkCount") - if "" == linkCountAsString { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - linkCountAsUint64, err = strconv.ParseUint(linkCountAsString, 10, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - symlinkTarget = request.Form.Get("symlinkTarget") - if "" == symlinkTarget { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - err = volume.inodeVolumeHandle.PatchInode( - inode.InodeNumber(inodeNumberAsUint64), - inode.SymlinkType, - linkCountAsUint64, - inode.PosixModePerm, - inode.InodeRootUserID, - inode.InodeGroupID(0), - inode.InodeNumber(0), - symlinkTarget) - - if nil == err { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusInternalServerError) - - _, _ = responseWriter.Write([]byte(fmt.Sprintf("err: %v\n", err))) - } -} - -func sortedTwoColumnResponseWriter(llrb sortedmap.LLRBTree, responseWriter http.ResponseWriter) { - var ( - err error - format string - i int - keyAsKey sortedmap.Key - keyAsString string - lenLLRB int - line string - longestKeyAsString int - longestValueAsString int - ok bool - valueAsString string - valueAsValue sortedmap.Value - ) - - lenLLRB, err = llrb.Len() - if nil != err { - err = fmt.Errorf("llrb.Len() failed: %v", err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - longestKeyAsString = 0 - longestValueAsString = 0 - - for i = 0; i < lenLLRB; i++ { - keyAsKey, valueAsValue, ok, err = llrb.GetByIndex(i) - if nil != err { - err = fmt.Errorf("llrb.GetByIndex(%v) failed: %v", i, err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - err = fmt.Errorf("llrb.GetByIndex(%v) returned ok == false", i) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - keyAsString = keyAsKey.(string) - valueAsString = valueAsValue.(string) - if len(keyAsString) > longestKeyAsString { - longestKeyAsString = len(keyAsString) - } - if len(valueAsString) > longestValueAsString { - longestValueAsString = len(valueAsString) - } - } - - format = fmt.Sprintf("%%-%vs %%%vs\n", longestKeyAsString, longestValueAsString) - - for i = 0; i < lenLLRB; i++ { - keyAsKey, valueAsValue, ok, err = llrb.GetByIndex(i) - if nil != err { - err = fmt.Errorf("llrb.GetByIndex(%v) failed: %v", i, err) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - err = fmt.Errorf("llrb.GetByIndex(%v) returned ok == false", i) - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - keyAsString = keyAsKey.(string) - valueAsString = valueAsValue.(string) - line = fmt.Sprintf(format, keyAsString, valueAsString) - _, _ = responseWriter.Write([]byte(line)) - } -} - -func markJobsCompletedIfNoLongerActiveWhileLocked(volume *volumeStruct) { - // First, mark as finished now any FSCK/SCRUB job - - if (nil != volume.fsckActiveJob) && !volume.fsckActiveJob.jobHandle.Active() { - // FSCK job finished at some point... make it look like it just finished now - - volume.fsckActiveJob.state = jobCompleted - volume.fsckActiveJob.endTime = time.Now() - volume.fsckActiveJob = nil - } - - if (nil != volume.scrubActiveJob) && !volume.scrubActiveJob.jobHandle.Active() { - // SCRUB job finished at some point... make it look like it just finished now - - volume.scrubActiveJob.state = jobCompleted - volume.scrubActiveJob.endTime = time.Now() - volume.scrubActiveJob = nil - } -} - -func doPut(responseWriter http.ResponseWriter, request *http.Request) { - switch { - case strings.HasPrefix(request.URL.Path, "/volume"): - doPutOfVolume(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func doPutOfVolume(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - numPathParts int - ok bool - pathSplit []string - requestState *requestStateStruct - volumeAsValue sortedmap.Value - volumeName string - ) - - pathSplit = strings.Split(request.URL.Path, "/") // leading "/" places "" in pathSplit[0] - // pathSplit[1] should be "volume" based on how we got here - // trailing "/" places "" in pathSplit[len(pathSplit)-1] - - numPathParts = len(pathSplit) - 1 - if "" == pathSplit[numPathParts] { - numPathParts-- - } - - if "volume" != pathSplit[1] { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - switch numPathParts { - case 3: - // Form: /volume//replace-dir-entries - default: - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - volumeName = pathSplit[2] - - volumeAsValue, ok, err = globals.volumeLLRB.GetByKey(volumeName) - if nil != err { - logger.Fatalf("HTTP Server Logic Error: %v", err) - } - if !ok { - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - requestState = &requestStateStruct{ - volume: volumeAsValue.(*volumeStruct), - } - - switch pathSplit[3] { - case "replace-dir-entries": - doReplaceDirEntries(responseWriter, request, requestState) - default: - responseWriter.WriteHeader(http.StatusNotFound) - return - } -} - -// doReplaceDirEntries expects the instructions for how to replace a DirInode's directory entries -// to be passed entirely in request.Body in the following form: -// -// () => -// -// -// ... -// -// -// where each InodeNumber is a 16 Hex Digit string. The list should not include "." nor "..". If -// successful, the DirInode's directory entries will be replaced by: -// -// "." => -// ".." => -// => -// => -// ... -// => -// -// In addition, the LinkCount for the DirInode will be properly set to 2 + the number of -// dirEntryInodeNumber's that refer to subdirectories. -// -func doReplaceDirEntries(responseWriter http.ResponseWriter, request *http.Request, requestState *requestStateStruct) { - var ( - dirEntryInodeNumber inode.InodeNumber - dirEntryInodeNumbers []inode.InodeNumber - dirInodeNumber inode.InodeNumber - err error - inodeNumberAsUint64 uint64 - maxNumberOfDirEntryInodeNumbers int - parentDirEntryBasename string - parentDirInodeNumber inode.InodeNumber - requestBody []byte - ) - - requestBody, err = ioutil.ReadAll(request.Body) - if nil != err { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - err = request.Body.Close() - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - // Parse parentDirInodeNumber first since it is in a fixed location - - if (24 > len(requestBody)) || ('(' != requestBody[0]) || (')' != requestBody[17]) || (' ' != requestBody[18]) { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - inodeNumberAsUint64, err = strconv.ParseUint(string(requestBody[1:17]), 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - parentDirInodeNumber = inode.InodeNumber(inodeNumberAsUint64) - - requestBody = requestBody[19:] - - // Parse backwards reading dirEntryInodeNumbers until hitting one preceeded by " => " - - maxNumberOfDirEntryInodeNumbers = (len(requestBody) + 15) / 16 - dirEntryInodeNumbers = make([]inode.InodeNumber, 0, maxNumberOfDirEntryInodeNumbers) - - for { - if 21 > len(requestBody) { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - if " => " == string(requestBody[len(requestBody)-20:len(requestBody)-16]) { - break - } - - inodeNumberAsUint64, err = strconv.ParseUint(string(requestBody[len(requestBody)-16:]), 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - dirEntryInodeNumber = inode.InodeNumber(inodeNumberAsUint64) - dirEntryInodeNumbers = append(dirEntryInodeNumbers, dirEntryInodeNumber) - - requestBody = requestBody[:len(requestBody)-16] - } - - // Parse dirInodeNumber - - inodeNumberAsUint64, err = strconv.ParseUint(string(requestBody[len(requestBody)-16:]), 16, 64) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - dirInodeNumber = inode.InodeNumber(inodeNumberAsUint64) - - requestBody = requestBody[:len(requestBody)-20] - - // What remains is parentDirEntryBasename - - parentDirEntryBasename = string(requestBody[:]) - - // Now we can finally proceed - - err = requestState.volume.inodeVolumeHandle.ReplaceDirEntries(parentDirInodeNumber, parentDirEntryBasename, dirInodeNumber, dirEntryInodeNumbers) - if nil == err { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusBadRequest) - } -} diff --git a/httpserver/request_handler_test.go b/httpserver/request_handler_test.go deleted file mode 100644 index 10d9c38a..00000000 --- a/httpserver/request_handler_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package httpserver - -import ( - "bytes" - "io/ioutil" - "net/http/httptest" - "testing" -) - -func TestConfigExpansion(t *testing.T) { - testSetup(t) - defer testTeardown(t) - - testConfigExpansion := func(url string, shouldBeExpanded bool) { - req := httptest.NewRequest("GET", "http://pfs.com"+url, nil) - w := httptest.NewRecorder() - - doGet(w, req) - resp := w.Result() - body, _ := ioutil.ReadAll(resp.Body) - - if resp.StatusCode != 200 { - t.Errorf("[%s]: Config response was %d; expected 200", url, resp.StatusCode) - return - } - - crs := bytes.Count(body, []byte("\n")) - if shouldBeExpanded { - if crs <= 2 { - t.Errorf("[%s]: Only found %d s, but config should be expanded.", url, crs) - } - } else { - if crs > 2 { - t.Errorf("[%s]: Found %d s, but config should be compact.", url, crs) - } - } - } - - // NOTE: These are really routing tests. If and when we isolate the routing code, we can - // move tests like this there. If and when we use an off-the-shelf router, we can delete - // these altogether and just call the internal function for a few canonical urls. - expandedUrls := []string{ - "/config", - "/config/?", - "/config/?foo=bar", - "/config/?compact=false", - } - - compactUrls := []string{ - "/config?compact=true", - "/config?compact=1", - "/config/?foo=bar&compact=1&baz=quux", - } - - for _, url := range expandedUrls { - testConfigExpansion(url, true) - } - - for _, url := range compactUrls { - testConfigExpansion(url, false) - } - - return -} diff --git a/httpserver/setup_teardown_test.go b/httpserver/setup_teardown_test.go deleted file mode 100644 index 5c138c41..00000000 --- a/httpserver/setup_teardown_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package httpserver - -import ( - "sync" - "syscall" - "testing" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/ramswift" - "github.com/NVIDIA/proxyfs/transitions" -) - -var ( - ramswiftDoneChan chan bool // our global ramswiftDoneChan used during testTeardown() to know ramswift is, indeed, down - testConfMap conf.ConfMap -) - -func testSetup(t *testing.T) { - var ( - err error - signalHandlerIsArmedWG sync.WaitGroup - testConfMapStrings []string - testConfUpdateStrings []string - ) - - testConfMapStrings = []string{ - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=35262", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=3", - "SwiftClient.RetryLimitObject=3", - "SwiftClient.RetryDelay=10ms", - "SwiftClient.RetryDelayObject=10ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=256", - "SwiftClient.NonChunkedConnectionPoolSize=64", - "Peer:Peer0.PublicIPAddr=127.0.0.1", - "Peer:Peer0.PrivateIPAddr=127.0.0.1", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - "Cluster.Peers=Peer0", - "Cluster.WhoAmI=Peer0", - "Cluster.PrivateClusterUDPPort=18123", - "Cluster.UDPPacketSendSize=1400", - "Cluster.UDPPacketRecvSize=1500", - "Cluster.UDPPacketCapPerMessage=5", - "Cluster.HeartBeatDuration=1s", - "Cluster.HeartBeatMissLimit=3", - "Cluster.MessageQueueDepthPerPeer=4", - "Cluster.MaxRequestDuration=1s", - "Cluster.LivenessCheckerEnable=true", - "Cluster.LivenessCheckRedundancy=2", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.DirEntryCacheEvictLowLimit=10000", - "FSGlobals.DirEntryCacheEvictHighLimit=10010", - "FSGlobals.FileExtentMapEvictLowLimit=10000", - "FSGlobals.FileExtentMapEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - "JSONRPCServer.TCPPort=12346", // 12346 instead of 12345 so that test can run if proxyfsd is already running - "JSONRPCServer.FastTCPPort=32346", // ...and similarly here... - "JSONRPCServer.DataPathLogging=false", - "JSONRPCServer.MinLeaseDuration=250ms", - "JSONRPCServer.LeaseInterruptInterval=250ms", - "JSONRPCServer.LeaseInterruptLimit=20", - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - - "HTTPServer.TCPPort=53461", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - err = testConfMap.UpdateFromStrings(testConfUpdateStrings) - if nil != err { - t.Fatalf("testConfMap.UpdateFromStrings(testConfUpdateStrings) failed: %v", err) - } - - signalHandlerIsArmedWG.Add(1) - ramswiftDoneChan = make(chan bool, 1) - go ramswift.Daemon("/dev/null", testConfMapStrings, &signalHandlerIsArmedWG, ramswiftDoneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - err = transitions.Up(testConfMap) - if nil != err { - t.Fatalf("transitions.Up() failed: %v", err) - } -} - -func testTeardown(t *testing.T) { - var ( - err error - ) - - err = transitions.Down(testConfMap) - if nil != err { - t.Fatalf("transitions.Down() failed: %v", err) - } - - _ = syscall.Kill(syscall.Getpid(), unix.SIGTERM) - _ = <-ramswiftDoneChan -} diff --git a/httpserver/static-content.go b/httpserver/static-content.go deleted file mode 100644 index 9a81bb18..00000000 --- a/httpserver/static-content.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package httpserver - -//go:generate make-static-content httpserver stylesDotCSS text/css s static-content/styles.css styles_dot_css_.go - -//go:generate make-static-content httpserver jsontreeDotJS application/javascript s static-content/jsontree.js jsontree_dot_js_.go - -//go:generate make-static-content httpserver bootstrapDotCSS text/css s static-content/bootstrap.min.css bootstrap_dot_min_dot_css_.go - -//go:generate make-static-content httpserver bootstrapDotJS application/javascript s static-content/bootstrap.min.js bootstrap_dot_min_dot_js_.go -//go:generate make-static-content httpserver jqueryDotJS application/javascript s static-content/jquery.min.js jquery_dot_min_dot_js_.go -//go:generate make-static-content httpserver popperDotJS application/javascript b static-content/popper.min.js popper_dot_min_dot_js_.go - -//go:generate make-static-content httpserver openIconicBootstrapDotCSS text/css s static-content/open-iconic/font/css/open-iconic-bootstrap.min.css open_iconic_bootstrap_dot_css_.go - -//go:generate make-static-content httpserver openIconicDotEOT application/vnd.ms-fontobject b static-content/open-iconic/font/fonts/open-iconic.eot open_iconic_dot_eot_.go -//go:generate make-static-content httpserver openIconicDotOTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.otf open_iconic_dot_otf_.go -//go:generate make-static-content httpserver openIconicDotSVG image/svg+xml s static-content/open-iconic/font/fonts/open-iconic.svg open_iconic_dot_svg_.go -//go:generate make-static-content httpserver openIconicDotTTF application/font-sfnt b static-content/open-iconic/font/fonts/open-iconic.ttf open_iconic_dot_ttf_.go -//go:generate make-static-content httpserver openIconicDotWOFF application/font-woff b static-content/open-iconic/font/fonts/open-iconic.woff open_iconic_dot_woff_.go diff --git a/httpserver/static-content/bootstrap.min.css b/httpserver/static-content/bootstrap.min.css deleted file mode 100644 index 7d2a868f..00000000 --- a/httpserver/static-content/bootstrap.min.css +++ /dev/null @@ -1,7 +0,0 @@ -/*! - * Bootstrap v4.5.0 (https://getbootstrap.com/) - * Copyright 2011-2020 The Bootstrap Authors - * Copyright 2011-2020 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */:root{--blue:#007bff;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#dc3545;--orange:#fd7e14;--yellow:#ffc107;--green:#28a745;--teal:#20c997;--cyan:#17a2b8;--white:#fff;--gray:#6c757d;--gray-dark:#343a40;--primary:#007bff;--secondary:#6c757d;--success:#28a745;--info:#17a2b8;--warning:#ffc107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:left;background-color:#fff}[tabindex="-1"]:focus:not(:focus-visible){outline:0!important}hr{box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem}p{margin-top:0;margin-bottom:1rem}abbr[data-original-title],abbr[title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#007bff;text-decoration:none;background-color:transparent}a:hover{color:#0056b3;text-decoration:underline}a:not([href]){color:inherit;text-decoration:none}a:not([href]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:.75rem;padding-bottom:.75rem;color:#6c757d;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{padding:0;border-style:none}input[type=checkbox],input[type=radio]{box-sizing:border-box;padding:0}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:none}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none!important}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-bottom:.5rem;font-weight:500;line-height:1.2}.h1,h1{font-size:2.5rem}.h2,h2{font-size:2rem}.h3,h3{font-size:1.75rem}.h4,h4{font-size:1.5rem}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,.1)}.small,small{font-size:80%;font-weight:400}.mark,mark{padding:.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#6c757d}.blockquote-footer::before{content:"\2014\00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:90%;color:#6c757d}code{font-size:87.5%;color:#e83e8c;word-wrap:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:87.5%;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;font-size:87.5%;color:#212529}pre code{font-size:inherit;color:inherit;word-break:normal}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container{max-width:540px}}@media (min-width:768px){.container{max-width:720px}}@media (min-width:992px){.container{max-width:960px}}@media (min-width:1200px){.container{max-width:1140px}}.container-fluid,.container-lg,.container-md,.container-sm,.container-xl{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width:576px){.container,.container-sm{max-width:540px}}@media (min-width:768px){.container,.container-md,.container-sm{max-width:720px}}@media (min-width:992px){.container,.container-lg,.container-md,.container-sm{max-width:960px}}@media (min-width:1200px){.container,.container-lg,.container-md,.container-sm,.container-xl{max-width:1140px}}.row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*=col-]{padding-right:0;padding-left:0}.col,.col-1,.col-10,.col-11,.col-12,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-auto,.col-lg,.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-auto,.col-md,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-auto,.col-sm,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-auto,.col-xl,.col-xl-1,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9,.col-xl-auto{position:relative;width:100%;padding-right:15px;padding-left:15px}.col{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-first{-ms-flex-order:-1;order:-1}.order-last{-ms-flex-order:13;order:13}.order-0{-ms-flex-order:0;order:0}.order-1{-ms-flex-order:1;order:1}.order-2{-ms-flex-order:2;order:2}.order-3{-ms-flex-order:3;order:3}.order-4{-ms-flex-order:4;order:4}.order-5{-ms-flex-order:5;order:5}.order-6{-ms-flex-order:6;order:6}.order-7{-ms-flex-order:7;order:7}.order-8{-ms-flex-order:8;order:8}.order-9{-ms-flex-order:9;order:9}.order-10{-ms-flex-order:10;order:10}.order-11{-ms-flex-order:11;order:11}.order-12{-ms-flex-order:12;order:12}.offset-1{margin-left:8.333333%}.offset-2{margin-left:16.666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.333333%}.offset-5{margin-left:41.666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.333333%}.offset-8{margin-left:66.666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.333333%}.offset-11{margin-left:91.666667%}@media (min-width:576px){.col-sm{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-sm-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-sm-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-sm-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-sm-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-sm-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-sm-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-sm-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-sm-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-sm-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-sm-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-sm-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-sm-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-sm-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-sm-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-sm-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-sm-first{-ms-flex-order:-1;order:-1}.order-sm-last{-ms-flex-order:13;order:13}.order-sm-0{-ms-flex-order:0;order:0}.order-sm-1{-ms-flex-order:1;order:1}.order-sm-2{-ms-flex-order:2;order:2}.order-sm-3{-ms-flex-order:3;order:3}.order-sm-4{-ms-flex-order:4;order:4}.order-sm-5{-ms-flex-order:5;order:5}.order-sm-6{-ms-flex-order:6;order:6}.order-sm-7{-ms-flex-order:7;order:7}.order-sm-8{-ms-flex-order:8;order:8}.order-sm-9{-ms-flex-order:9;order:9}.order-sm-10{-ms-flex-order:10;order:10}.order-sm-11{-ms-flex-order:11;order:11}.order-sm-12{-ms-flex-order:12;order:12}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.333333%}.offset-sm-2{margin-left:16.666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.333333%}.offset-sm-5{margin-left:41.666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.333333%}.offset-sm-8{margin-left:66.666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.333333%}.offset-sm-11{margin-left:91.666667%}}@media (min-width:768px){.col-md{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-md-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-md-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-md-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-md-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-md-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-md-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-md-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-md-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-md-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-md-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-md-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-md-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-md-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-md-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-md-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-md-first{-ms-flex-order:-1;order:-1}.order-md-last{-ms-flex-order:13;order:13}.order-md-0{-ms-flex-order:0;order:0}.order-md-1{-ms-flex-order:1;order:1}.order-md-2{-ms-flex-order:2;order:2}.order-md-3{-ms-flex-order:3;order:3}.order-md-4{-ms-flex-order:4;order:4}.order-md-5{-ms-flex-order:5;order:5}.order-md-6{-ms-flex-order:6;order:6}.order-md-7{-ms-flex-order:7;order:7}.order-md-8{-ms-flex-order:8;order:8}.order-md-9{-ms-flex-order:9;order:9}.order-md-10{-ms-flex-order:10;order:10}.order-md-11{-ms-flex-order:11;order:11}.order-md-12{-ms-flex-order:12;order:12}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.333333%}.offset-md-2{margin-left:16.666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.333333%}.offset-md-5{margin-left:41.666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.333333%}.offset-md-8{margin-left:66.666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.333333%}.offset-md-11{margin-left:91.666667%}}@media (min-width:992px){.col-lg{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-lg-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-lg-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-lg-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-lg-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-lg-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-lg-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-lg-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-lg-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-lg-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-lg-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-lg-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-lg-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-lg-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-lg-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-lg-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-lg-first{-ms-flex-order:-1;order:-1}.order-lg-last{-ms-flex-order:13;order:13}.order-lg-0{-ms-flex-order:0;order:0}.order-lg-1{-ms-flex-order:1;order:1}.order-lg-2{-ms-flex-order:2;order:2}.order-lg-3{-ms-flex-order:3;order:3}.order-lg-4{-ms-flex-order:4;order:4}.order-lg-5{-ms-flex-order:5;order:5}.order-lg-6{-ms-flex-order:6;order:6}.order-lg-7{-ms-flex-order:7;order:7}.order-lg-8{-ms-flex-order:8;order:8}.order-lg-9{-ms-flex-order:9;order:9}.order-lg-10{-ms-flex-order:10;order:10}.order-lg-11{-ms-flex-order:11;order:11}.order-lg-12{-ms-flex-order:12;order:12}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.333333%}.offset-lg-2{margin-left:16.666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.333333%}.offset-lg-5{margin-left:41.666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.333333%}.offset-lg-8{margin-left:66.666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.333333%}.offset-lg-11{margin-left:91.666667%}}@media (min-width:1200px){.col-xl{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;min-width:0;max-width:100%}.row-cols-xl-1>*{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.row-cols-xl-2>*{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.row-cols-xl-3>*{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.row-cols-xl-4>*{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.row-cols-xl-5>*{-ms-flex:0 0 20%;flex:0 0 20%;max-width:20%}.row-cols-xl-6>*{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-auto{-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:100%}.col-xl-1{-ms-flex:0 0 8.333333%;flex:0 0 8.333333%;max-width:8.333333%}.col-xl-2{-ms-flex:0 0 16.666667%;flex:0 0 16.666667%;max-width:16.666667%}.col-xl-3{-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-ms-flex:0 0 33.333333%;flex:0 0 33.333333%;max-width:33.333333%}.col-xl-5{-ms-flex:0 0 41.666667%;flex:0 0 41.666667%;max-width:41.666667%}.col-xl-6{-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-ms-flex:0 0 58.333333%;flex:0 0 58.333333%;max-width:58.333333%}.col-xl-8{-ms-flex:0 0 66.666667%;flex:0 0 66.666667%;max-width:66.666667%}.col-xl-9{-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-ms-flex:0 0 83.333333%;flex:0 0 83.333333%;max-width:83.333333%}.col-xl-11{-ms-flex:0 0 91.666667%;flex:0 0 91.666667%;max-width:91.666667%}.col-xl-12{-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-xl-first{-ms-flex-order:-1;order:-1}.order-xl-last{-ms-flex-order:13;order:13}.order-xl-0{-ms-flex-order:0;order:0}.order-xl-1{-ms-flex-order:1;order:1}.order-xl-2{-ms-flex-order:2;order:2}.order-xl-3{-ms-flex-order:3;order:3}.order-xl-4{-ms-flex-order:4;order:4}.order-xl-5{-ms-flex-order:5;order:5}.order-xl-6{-ms-flex-order:6;order:6}.order-xl-7{-ms-flex-order:7;order:7}.order-xl-8{-ms-flex-order:8;order:8}.order-xl-9{-ms-flex-order:9;order:9}.order-xl-10{-ms-flex-order:10;order:10}.order-xl-11{-ms-flex-order:11;order:11}.order-xl-12{-ms-flex-order:12;order:12}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.333333%}.offset-xl-2{margin-left:16.666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.333333%}.offset-xl-5{margin-left:41.666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.333333%}.offset-xl-8{margin-left:66.666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.333333%}.offset-xl-11{margin-left:91.666667%}}.table{width:100%;margin-bottom:1rem;color:#212529}.table td,.table th{padding:.75rem;vertical-align:top;border-top:1px solid #dee2e6}.table thead th{vertical-align:bottom;border-bottom:2px solid #dee2e6}.table tbody+tbody{border-top:2px solid #dee2e6}.table-sm td,.table-sm th{padding:.3rem}.table-bordered{border:1px solid #dee2e6}.table-bordered td,.table-bordered th{border:1px solid #dee2e6}.table-bordered thead td,.table-bordered thead th{border-bottom-width:2px}.table-borderless tbody+tbody,.table-borderless td,.table-borderless th,.table-borderless thead th{border:0}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,.05)}.table-hover tbody tr:hover{color:#212529;background-color:rgba(0,0,0,.075)}.table-primary,.table-primary>td,.table-primary>th{background-color:#b8daff}.table-primary tbody+tbody,.table-primary td,.table-primary th,.table-primary thead th{border-color:#7abaff}.table-hover .table-primary:hover{background-color:#9fcdff}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#9fcdff}.table-secondary,.table-secondary>td,.table-secondary>th{background-color:#d6d8db}.table-secondary tbody+tbody,.table-secondary td,.table-secondary th,.table-secondary thead th{border-color:#b3b7bb}.table-hover .table-secondary:hover{background-color:#c8cbcf}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#c8cbcf}.table-success,.table-success>td,.table-success>th{background-color:#c3e6cb}.table-success tbody+tbody,.table-success td,.table-success th,.table-success thead th{border-color:#8fd19e}.table-hover .table-success:hover{background-color:#b1dfbb}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#b1dfbb}.table-info,.table-info>td,.table-info>th{background-color:#bee5eb}.table-info tbody+tbody,.table-info td,.table-info th,.table-info thead th{border-color:#86cfda}.table-hover .table-info:hover{background-color:#abdde5}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#abdde5}.table-warning,.table-warning>td,.table-warning>th{background-color:#ffeeba}.table-warning tbody+tbody,.table-warning td,.table-warning th,.table-warning thead th{border-color:#ffdf7e}.table-hover .table-warning:hover{background-color:#ffe8a1}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#ffe8a1}.table-danger,.table-danger>td,.table-danger>th{background-color:#f5c6cb}.table-danger tbody+tbody,.table-danger td,.table-danger th,.table-danger thead th{border-color:#ed969e}.table-hover .table-danger:hover{background-color:#f1b0b7}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#f1b0b7}.table-light,.table-light>td,.table-light>th{background-color:#fdfdfe}.table-light tbody+tbody,.table-light td,.table-light th,.table-light thead th{border-color:#fbfcfc}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>td,.table-dark>th{background-color:#c6c8ca}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#95999c}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>td,.table-active>th{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,.075)}.table .thead-dark th{color:#fff;background-color:#343a40;border-color:#454d55}.table .thead-light th{color:#495057;background-color:#e9ecef;border-color:#dee2e6}.table-dark{color:#fff;background-color:#343a40}.table-dark td,.table-dark th,.table-dark thead th{border-color:#454d55}.table-dark.table-bordered{border:0}.table-dark.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,.05)}.table-dark.table-hover tbody tr:hover{color:#fff;background-color:rgba(255,255,255,.075)}@media (max-width:575.98px){.table-responsive-sm{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-sm>.table-bordered{border:0}}@media (max-width:767.98px){.table-responsive-md{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-md>.table-bordered{border:0}}@media (max-width:991.98px){.table-responsive-lg{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-lg>.table-bordered{border:0}}@media (max-width:1199.98px){.table-responsive-xl{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive-xl>.table-bordered{border:0}}.table-responsive{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch}.table-responsive>.table-bordered{border:0}.form-control{display:block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.form-control:focus{color:#495057;background-color:#fff;border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.form-control::-webkit-input-placeholder{color:#6c757d;opacity:1}.form-control::-moz-placeholder{color:#6c757d;opacity:1}.form-control:-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::-ms-input-placeholder{color:#6c757d;opacity:1}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}input[type=date].form-control,input[type=datetime-local].form-control,input[type=month].form-control,input[type=time].form-control{-webkit-appearance:none;-moz-appearance:none;appearance:none}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block;width:100%}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem;line-height:1.5}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem;line-height:1.5}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;font-size:1rem;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.form-control-lg{height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}select.form-control[multiple],select.form-control[size]{height:auto}textarea.form-control{height:auto}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:.25rem}.form-row{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*=col-]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;padding-left:1.25rem}.form-check-input{position:absolute;margin-top:.3rem;margin-left:-1.25rem}.form-check-input:disabled~.form-check-label,.form-check-input[disabled]~.form-check-label{color:#6c757d}.form-check-label{margin-bottom:0}.form-check-inline{display:-ms-inline-flexbox;display:inline-flex;-ms-flex-align:center;align-items:center;padding-left:0;margin-right:.75rem}.form-check-inline .form-check-input{position:static;margin-top:0;margin-right:.3125rem;margin-left:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#28a745}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(40,167,69,.9);border-radius:.25rem}.is-valid~.valid-feedback,.is-valid~.valid-tooltip,.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip{display:block}.form-control.is-valid,.was-validated .form-control:valid{border-color:#28a745;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-valid,.was-validated .custom-select:valid{border-color:#28a745;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%2328a745' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-valid:focus,.was-validated .custom-select:valid:focus{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:#28a745}.form-check-input.is-valid~.valid-feedback,.form-check-input.is-valid~.valid-tooltip,.was-validated .form-check-input:valid~.valid-feedback,.was-validated .form-check-input:valid~.valid-tooltip{display:block}.custom-control-input.is-valid~.custom-control-label,.was-validated .custom-control-input:valid~.custom-control-label{color:#28a745}.custom-control-input.is-valid~.custom-control-label::before,.was-validated .custom-control-input:valid~.custom-control-label::before{border-color:#28a745}.custom-control-input.is-valid:checked~.custom-control-label::before,.was-validated .custom-control-input:valid:checked~.custom-control-label::before{border-color:#34ce57;background-color:#34ce57}.custom-control-input.is-valid:focus~.custom-control-label::before,.was-validated .custom-control-input:valid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.custom-control-input.is-valid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:valid:focus:not(:checked)~.custom-control-label::before{border-color:#28a745}.custom-file-input.is-valid~.custom-file-label,.was-validated .custom-file-input:valid~.custom-file-label{border-color:#28a745}.custom-file-input.is-valid:focus~.custom-file-label,.was-validated .custom-file-input:valid:focus~.custom-file-label{border-color:#28a745;box-shadow:0 0 0 .2rem rgba(40,167,69,.25)}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:80%;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;line-height:1.5;color:#fff;background-color:rgba(220,53,69,.9);border-radius:.25rem}.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip,.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip{display:block}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:#dc3545;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.custom-select.is-invalid,.was-validated .custom-select:invalid{border-color:#dc3545;padding-right:calc(.75em + 2.3125rem);background:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px,url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' fill='none' stroke='%23dc3545' viewBox='0 0 12 12'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e") #fff no-repeat center right 1.75rem/calc(.75em + .375rem) calc(.75em + .375rem)}.custom-select.is-invalid:focus,.was-validated .custom-select:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:#dc3545}.form-check-input.is-invalid~.invalid-feedback,.form-check-input.is-invalid~.invalid-tooltip,.was-validated .form-check-input:invalid~.invalid-feedback,.was-validated .form-check-input:invalid~.invalid-tooltip{display:block}.custom-control-input.is-invalid~.custom-control-label,.was-validated .custom-control-input:invalid~.custom-control-label{color:#dc3545}.custom-control-input.is-invalid~.custom-control-label::before,.was-validated .custom-control-input:invalid~.custom-control-label::before{border-color:#dc3545}.custom-control-input.is-invalid:checked~.custom-control-label::before,.was-validated .custom-control-input:invalid:checked~.custom-control-label::before{border-color:#e4606d;background-color:#e4606d}.custom-control-input.is-invalid:focus~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.custom-control-input.is-invalid:focus:not(:checked)~.custom-control-label::before,.was-validated .custom-control-input:invalid:focus:not(:checked)~.custom-control-label::before{border-color:#dc3545}.custom-file-input.is-invalid~.custom-file-label,.was-validated .custom-file-input:invalid~.custom-file-label{border-color:#dc3545}.custom-file-input.is-invalid:focus~.custom-file-label,.was-validated .custom-file-input:invalid:focus~.custom-file-label{border-color:#dc3545;box-shadow:0 0 0 .2rem rgba(220,53,69,.25)}.form-inline{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width:576px){.form-inline label{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-ms-flexbox;display:flex;-ms-flex:0 0 auto;flex:0 0 auto;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .custom-select,.form-inline .input-group{width:auto}.form-inline .form-check{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:auto;padding-left:0}.form-inline .form-check-input{position:relative;-ms-flex-negative:0;flex-shrink:0;margin-top:0;margin-right:.25rem;margin-left:0}.form-inline .custom-control{-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center}.form-inline .custom-control-label{margin-bottom:0}}.btn{display:inline-block;font-weight:400;color:#212529;text-align:center;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;line-height:1.5;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:#212529;text-decoration:none}.btn.focus,.btn:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.btn.disabled,.btn:disabled{opacity:.65}.btn:not(:disabled):not(.disabled){cursor:pointer}a.btn.disabled,fieldset:disabled a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:hover{color:#fff;background-color:#0069d9;border-color:#0062cc}.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#0069d9;border-color:#0062cc;box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#007bff;border-color:#007bff}.btn-primary:not(:disabled):not(.disabled).active,.btn-primary:not(:disabled):not(.disabled):active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0062cc;border-color:#005cbf}.btn-primary:not(:disabled):not(.disabled).active:focus,.btn-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(38,143,255,.5)}.btn-secondary{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:hover{color:#fff;background-color:#5a6268;border-color:#545b62}.btn-secondary.focus,.btn-secondary:focus{color:#fff;background-color:#5a6268;border-color:#545b62;box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:not(:disabled):not(.disabled).active,.btn-secondary:not(:disabled):not(.disabled):active,.show>.btn-secondary.dropdown-toggle{color:#fff;background-color:#545b62;border-color:#4e555b}.btn-secondary:not(:disabled):not(.disabled).active:focus,.btn-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(130,138,145,.5)}.btn-success{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:hover{color:#fff;background-color:#218838;border-color:#1e7e34}.btn-success.focus,.btn-success:focus{color:#fff;background-color:#218838;border-color:#1e7e34;box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#28a745;border-color:#28a745}.btn-success:not(:disabled):not(.disabled).active,.btn-success:not(:disabled):not(.disabled):active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#1e7e34;border-color:#1c7430}.btn-success:not(:disabled):not(.disabled).active:focus,.btn-success:not(:disabled):not(.disabled):active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(72,180,97,.5)}.btn-info{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:hover{color:#fff;background-color:#138496;border-color:#117a8b}.btn-info.focus,.btn-info:focus{color:#fff;background-color:#138496;border-color:#117a8b;box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-info.disabled,.btn-info:disabled{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-info:not(:disabled):not(.disabled).active,.btn-info:not(:disabled):not(.disabled):active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#117a8b;border-color:#10707f}.btn-info:not(:disabled):not(.disabled).active:focus,.btn-info:not(:disabled):not(.disabled):active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(58,176,195,.5)}.btn-warning{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#212529;background-color:#e0a800;border-color:#d39e00}.btn-warning.focus,.btn-warning:focus{color:#212529;background-color:#e0a800;border-color:#d39e00;box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-warning.disabled,.btn-warning:disabled{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-warning:not(:disabled):not(.disabled).active,.btn-warning:not(:disabled):not(.disabled):active,.show>.btn-warning.dropdown-toggle{color:#212529;background-color:#d39e00;border-color:#c69500}.btn-warning:not(:disabled):not(.disabled).active:focus,.btn-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(222,170,12,.5)}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#c82333;border-color:#bd2130}.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c82333;border-color:#bd2130;box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:not(:disabled):not(.disabled).active,.btn-danger:not(:disabled):not(.disabled):active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#bd2130;border-color:#b21f2d}.btn-danger:not(:disabled):not(.disabled).active:focus,.btn-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(225,83,97,.5)}.btn-light{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#212529;background-color:#e2e6ea;border-color:#dae0e5}.btn-light.focus,.btn-light:focus{color:#212529;background-color:#e2e6ea;border-color:#dae0e5;box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-light.disabled,.btn-light:disabled{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:not(:disabled):not(.disabled).active,.btn-light:not(:disabled):not(.disabled):active,.show>.btn-light.dropdown-toggle{color:#212529;background-color:#dae0e5;border-color:#d3d9df}.btn-light:not(:disabled):not(.disabled).active:focus,.btn-light:not(:disabled):not(.disabled):active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(216,217,219,.5)}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark.focus,.btn-dark:focus{color:#fff;background-color:#23272b;border-color:#1d2124;box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:not(:disabled):not(.disabled).active,.btn-dark:not(:disabled):not(.disabled):active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1d2124;border-color:#171a1d}.btn-dark:not(:disabled):not(.disabled).active:focus,.btn-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(82,88,93,.5)}.btn-outline-primary{color:#007bff;border-color:#007bff}.btn-outline-primary:hover{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary.focus,.btn-outline-primary:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#007bff;background-color:transparent}.btn-outline-primary:not(:disabled):not(.disabled).active,.btn-outline-primary:not(:disabled):not(.disabled):active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#007bff;border-color:#007bff}.btn-outline-primary:not(:disabled):not(.disabled).active:focus,.btn-outline-primary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-primary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.btn-outline-secondary{color:#6c757d;border-color:#6c757d}.btn-outline-secondary:hover{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary.focus,.btn-outline-secondary:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#6c757d;background-color:transparent}.btn-outline-secondary:not(:disabled):not(.disabled).active,.btn-outline-secondary:not(:disabled):not(.disabled):active,.show>.btn-outline-secondary.dropdown-toggle{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-outline-secondary:not(:disabled):not(.disabled).active:focus,.btn-outline-secondary:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.btn-outline-success{color:#28a745;border-color:#28a745}.btn-outline-success:hover{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success.focus,.btn-outline-success:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#28a745;background-color:transparent}.btn-outline-success:not(:disabled):not(.disabled).active,.btn-outline-success:not(:disabled):not(.disabled):active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#28a745;border-color:#28a745}.btn-outline-success:not(:disabled):not(.disabled).active:focus,.btn-outline-success:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-success.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.btn-outline-info{color:#17a2b8;border-color:#17a2b8}.btn-outline-info:hover{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info.focus,.btn-outline-info:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#17a2b8;background-color:transparent}.btn-outline-info:not(:disabled):not(.disabled).active,.btn-outline-info:not(:disabled):not(.disabled):active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#17a2b8;border-color:#17a2b8}.btn-outline-info:not(:disabled):not(.disabled).active:focus,.btn-outline-info:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-info.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.btn-outline-warning{color:#ffc107;border-color:#ffc107}.btn-outline-warning:hover{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning.focus,.btn-outline-warning:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-warning:not(:disabled):not(.disabled).active,.btn-outline-warning:not(:disabled):not(.disabled):active,.show>.btn-outline-warning.dropdown-toggle{color:#212529;background-color:#ffc107;border-color:#ffc107}.btn-outline-warning:not(:disabled):not(.disabled).active:focus,.btn-outline-warning:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-warning.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.btn-outline-danger{color:#dc3545;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger.focus,.btn-outline-danger:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-danger:not(:disabled):not(.disabled).active,.btn-outline-danger:not(:disabled):not(.disabled):active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-outline-danger:not(:disabled):not(.disabled).active:focus,.btn-outline-danger:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-danger.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light.focus,.btn-outline-light:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light:not(:disabled):not(.disabled).active,.btn-outline-light:not(:disabled):not(.disabled):active,.show>.btn-outline-light.dropdown-toggle{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:not(:disabled):not(.disabled).active:focus,.btn-outline-light:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-light.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.btn-outline-dark{color:#343a40;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark.focus,.btn-outline-dark:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark:not(:disabled):not(.disabled).active,.btn-outline-dark:not(:disabled):not(.disabled):active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:not(:disabled):not(.disabled).active:focus,.btn-outline-dark:not(:disabled):not(.disabled):active:focus,.show>.btn-outline-dark.dropdown-toggle:focus{box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.btn-link{font-weight:400;color:#007bff;text-decoration:none}.btn-link:hover{color:#0056b3;text-decoration:underline}.btn-link.focus,.btn-link:focus{text-decoration:underline}.btn-link.disabled,.btn-link:disabled{color:#6c757d;pointer-events:none}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:.5rem}input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{position:relative;height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.dropdown,.dropleft,.dropright,.dropup{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:.5rem 0;margin:.125rem 0 0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-menu-left{right:auto;left:0}.dropdown-menu-right{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-left{right:auto;left:0}.dropdown-menu-sm-right{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-left{right:auto;left:0}.dropdown-menu-md-right{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-left{right:auto;left:0}.dropdown-menu-lg-right{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-left{right:auto;left:0}.dropdown-menu-xl-right{right:0;left:auto}}.dropup .dropdown-menu{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-menu{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropright .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropright .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-toggle::after{vertical-align:0}.dropleft .dropdown-menu{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropleft .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropleft .dropdown-toggle::after{display:none}.dropleft .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropleft .dropdown-toggle:empty::after{margin-left:0}.dropleft .dropdown-toggle::before{vertical-align:0}.dropdown-menu[x-placement^=bottom],.dropdown-menu[x-placement^=left],.dropdown-menu[x-placement^=right],.dropdown-menu[x-placement^=top]{right:auto;bottom:auto}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:.25rem 1.5rem;clear:both;font-weight:400;color:#212529;text-align:inherit;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#16181b;text-decoration:none;background-color:#f8f9fa}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#007bff}.dropdown-item.disabled,.dropdown-item:disabled{color:#6c757d;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1.5rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1.5rem;color:#212529}.btn-group,.btn-group-vertical{position:relative;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;-ms-flex:1 1 auto;flex:1 1 auto}.btn-group-vertical>.btn:hover,.btn-group>.btn:hover{z-index:1}.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus{z-index:1}.btn-toolbar{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn-group:not(:first-child),.btn-group>.btn:not(:first-child){margin-left:-1px}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropright .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropleft .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{-ms-flex-direction:column;flex-direction:column;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:center;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn:not(:first-child){border-top-left-radius:0;border-top-right-radius:0}.btn-group-toggle>.btn,.btn-group-toggle>.btn-group>.btn{margin-bottom:0}.btn-group-toggle>.btn input[type=checkbox],.btn-group-toggle>.btn input[type=radio],.btn-group-toggle>.btn-group>.btn input[type=checkbox],.btn-group-toggle>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.input-group{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:stretch;align-items:stretch;width:100%}.input-group>.custom-file,.input-group>.custom-select,.input-group>.form-control,.input-group>.form-control-plaintext{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;min-width:0;margin-bottom:0}.input-group>.custom-file+.custom-file,.input-group>.custom-file+.custom-select,.input-group>.custom-file+.form-control,.input-group>.custom-select+.custom-file,.input-group>.custom-select+.custom-select,.input-group>.custom-select+.form-control,.input-group>.form-control+.custom-file,.input-group>.form-control+.custom-select,.input-group>.form-control+.form-control,.input-group>.form-control-plaintext+.custom-file,.input-group>.form-control-plaintext+.custom-select,.input-group>.form-control-plaintext+.form-control{margin-left:-1px}.input-group>.custom-file .custom-file-input:focus~.custom-file-label,.input-group>.custom-select:focus,.input-group>.form-control:focus{z-index:3}.input-group>.custom-file .custom-file-input:focus{z-index:4}.input-group>.custom-select:not(:last-child),.input-group>.form-control:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-select:not(:first-child),.input-group>.form-control:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.custom-file{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center}.input-group>.custom-file:not(:last-child) .custom-file-label,.input-group>.custom-file:not(:last-child) .custom-file-label::after{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-file:not(:first-child) .custom-file-label{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-append,.input-group-prepend{display:-ms-flexbox;display:flex}.input-group-append .btn,.input-group-prepend .btn{position:relative;z-index:2}.input-group-append .btn:focus,.input-group-prepend .btn:focus{z-index:3}.input-group-append .btn+.btn,.input-group-append .btn+.input-group-text,.input-group-append .input-group-text+.btn,.input-group-append .input-group-text+.input-group-text,.input-group-prepend .btn+.btn,.input-group-prepend .btn+.input-group-text,.input-group-prepend .input-group-text+.btn,.input-group-prepend .input-group-text+.input-group-text{margin-left:-1px}.input-group-prepend{margin-right:-1px}.input-group-append{margin-left:-1px}.input-group-text{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.375rem .75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-text input[type=checkbox],.input-group-text input[type=radio]{margin-top:0}.input-group-lg>.custom-select,.input-group-lg>.form-control:not(textarea){height:calc(1.5em + 1rem + 2px)}.input-group-lg>.custom-select,.input-group-lg>.form-control,.input-group-lg>.input-group-append>.btn,.input-group-lg>.input-group-append>.input-group-text,.input-group-lg>.input-group-prepend>.btn,.input-group-lg>.input-group-prepend>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:.3rem}.input-group-sm>.custom-select,.input-group-sm>.form-control:not(textarea){height:calc(1.5em + .5rem + 2px)}.input-group-sm>.custom-select,.input-group-sm>.form-control,.input-group-sm>.input-group-append>.btn,.input-group-sm>.input-group-append>.input-group-text,.input-group-sm>.input-group-prepend>.btn,.input-group-sm>.input-group-prepend>.input-group-text{padding:.25rem .5rem;font-size:.875rem;line-height:1.5;border-radius:.2rem}.input-group-lg>.custom-select,.input-group-sm>.custom-select{padding-right:1.75rem}.input-group>.input-group-append:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group>.input-group-append:last-child>.input-group-text:not(:last-child),.input-group>.input-group-append:not(:last-child)>.btn,.input-group>.input-group-append:not(:last-child)>.input-group-text,.input-group>.input-group-prepend>.btn,.input-group>.input-group-prepend>.input-group-text{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.input-group-append>.btn,.input-group>.input-group-append>.input-group-text,.input-group>.input-group-prepend:first-child>.btn:not(:first-child),.input-group>.input-group-prepend:first-child>.input-group-text:not(:first-child),.input-group>.input-group-prepend:not(:first-child)>.btn,.input-group>.input-group-prepend:not(:first-child)>.input-group-text{border-top-left-radius:0;border-bottom-left-radius:0}.custom-control{position:relative;display:block;min-height:1.5rem;padding-left:1.5rem}.custom-control-inline{display:-ms-inline-flexbox;display:inline-flex;margin-right:1rem}.custom-control-input{position:absolute;left:0;z-index:-1;width:1rem;height:1.25rem;opacity:0}.custom-control-input:checked~.custom-control-label::before{color:#fff;border-color:#007bff;background-color:#007bff}.custom-control-input:focus~.custom-control-label::before{box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-control-input:focus:not(:checked)~.custom-control-label::before{border-color:#80bdff}.custom-control-input:not(:disabled):active~.custom-control-label::before{color:#fff;background-color:#b3d7ff;border-color:#b3d7ff}.custom-control-input:disabled~.custom-control-label,.custom-control-input[disabled]~.custom-control-label{color:#6c757d}.custom-control-input:disabled~.custom-control-label::before,.custom-control-input[disabled]~.custom-control-label::before{background-color:#e9ecef}.custom-control-label{position:relative;margin-bottom:0;vertical-align:top}.custom-control-label::before{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;pointer-events:none;content:"";background-color:#fff;border:#adb5bd solid 1px}.custom-control-label::after{position:absolute;top:.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;content:"";background:no-repeat 50%/50% 50%}.custom-checkbox .custom-control-label::before{border-radius:.25rem}.custom-checkbox .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26l2.974 2.99L8 2.193z'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::before{border-color:#007bff;background-color:#007bff}.custom-checkbox .custom-control-input:indeterminate~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='4' viewBox='0 0 4 4'%3e%3cpath stroke='%23fff' d='M0 2h4'/%3e%3c/svg%3e")}.custom-checkbox .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-checkbox .custom-control-input:disabled:indeterminate~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-radio .custom-control-label::before{border-radius:50%}.custom-radio .custom-control-input:checked~.custom-control-label::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.custom-radio .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-switch{padding-left:2.25rem}.custom-switch .custom-control-label::before{left:-2.25rem;width:1.75rem;pointer-events:all;border-radius:.5rem}.custom-switch .custom-control-label::after{top:calc(.25rem + 2px);left:calc(-2.25rem + 2px);width:calc(1rem - 4px);height:calc(1rem - 4px);background-color:#adb5bd;border-radius:.5rem;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:transform .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-transform .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-switch .custom-control-label::after{transition:none}}.custom-switch .custom-control-input:checked~.custom-control-label::after{background-color:#fff;-webkit-transform:translateX(.75rem);transform:translateX(.75rem)}.custom-switch .custom-control-input:disabled:checked~.custom-control-label::before{background-color:rgba(0,123,255,.5)}.custom-select{display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);padding:.375rem 1.75rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;vertical-align:middle;background:#fff url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='4' height='5' viewBox='0 0 4 5'%3e%3cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3e%3c/svg%3e") no-repeat right .75rem center/8px 10px;border:1px solid #ced4da;border-radius:.25rem;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-select:focus{border-color:#80bdff;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select[multiple],.custom-select[size]:not([size="1"]){height:auto;padding-right:.75rem;background-image:none}.custom-select:disabled{color:#6c757d;background-color:#e9ecef}.custom-select::-ms-expand{display:none}.custom-select:-moz-focusring{color:transparent;text-shadow:0 0 0 #495057}.custom-select-sm{height:calc(1.5em + .5rem + 2px);padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem}.custom-select-lg{height:calc(1.5em + 1rem + 2px);padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}.custom-file{position:relative;display:inline-block;width:100%;height:calc(1.5em + .75rem + 2px);margin-bottom:0}.custom-file-input{position:relative;z-index:2;width:100%;height:calc(1.5em + .75rem + 2px);margin:0;opacity:0}.custom-file-input:focus~.custom-file-label{border-color:#80bdff;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.custom-file-input:disabled~.custom-file-label,.custom-file-input[disabled]~.custom-file-label{background-color:#e9ecef}.custom-file-input:lang(en)~.custom-file-label::after{content:"Browse"}.custom-file-input~.custom-file-label[data-browse]::after{content:attr(data-browse)}.custom-file-label{position:absolute;top:0;right:0;left:0;z-index:1;height:calc(1.5em + .75rem + 2px);padding:.375rem .75rem;font-weight:400;line-height:1.5;color:#495057;background-color:#fff;border:1px solid #ced4da;border-radius:.25rem}.custom-file-label::after{position:absolute;top:0;right:0;bottom:0;z-index:3;display:block;height:calc(1.5em + .75rem);padding:.375rem .75rem;line-height:1.5;color:#495057;content:"Browse";background-color:#e9ecef;border-left:inherit;border-radius:0 .25rem .25rem 0}.custom-range{width:100%;height:1.4rem;padding:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-range:focus{outline:0}.custom-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range:focus::-ms-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .2rem rgba(0,123,255,.25)}.custom-range::-moz-focus-outer{border:0}.custom-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#007bff;border:0;border-radius:1rem;-webkit-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.custom-range::-webkit-slider-thumb:active{background-color:#b3d7ff}.custom-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#007bff;border:0;border-radius:1rem;-moz-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-moz-range-thumb{-moz-transition:none;transition:none}}.custom-range::-moz-range-thumb:active{background-color:#b3d7ff}.custom-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-ms-thumb{width:1rem;height:1rem;margin-top:0;margin-right:.2rem;margin-left:.2rem;background-color:#007bff;border:0;border-radius:1rem;-ms-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;appearance:none}@media (prefers-reduced-motion:reduce){.custom-range::-ms-thumb{-ms-transition:none;transition:none}}.custom-range::-ms-thumb:active{background-color:#b3d7ff}.custom-range::-ms-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:transparent;border-color:transparent;border-width:.5rem}.custom-range::-ms-fill-lower{background-color:#dee2e6;border-radius:1rem}.custom-range::-ms-fill-upper{margin-right:15px;background-color:#dee2e6;border-radius:1rem}.custom-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.custom-range:disabled::-webkit-slider-runnable-track{cursor:default}.custom-range:disabled::-moz-range-thumb{background-color:#adb5bd}.custom-range:disabled::-moz-range-track{cursor:default}.custom-range:disabled::-ms-thumb{background-color:#adb5bd}.custom-control-label::before,.custom-file-label,.custom-select{transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.custom-control-label::before,.custom-file-label,.custom-select{transition:none}}.nav{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem}.nav-link:focus,.nav-link:hover{text-decoration:none}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #dee2e6}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#007bff}.nav-fill .nav-item{-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-ms-flex-preferred-size:0;flex-basis:0;-ms-flex-positive:1;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between;padding:.5rem 1rem}.navbar .container,.navbar .container-fluid,.navbar .container-lg,.navbar .container-md,.navbar .container-sm,.navbar .container-xl{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:justify;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:focus,.navbar-brand:hover{text-decoration:none}.navbar-nav{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{-ms-flex-preferred-size:100%;flex-basis:100%;-ms-flex-positive:1;flex-grow:1;-ms-flex-align:center;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem}.navbar-toggler:focus,.navbar-toggler:hover{text-decoration:none}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width:575.98px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{padding-right:0;padding-left:0}}@media (min-width:576px){.navbar-expand-sm{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-sm .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid,.navbar-expand-sm>.container-lg,.navbar-expand-sm>.container-md,.navbar-expand-sm>.container-sm,.navbar-expand-sm>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width:767.98px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{padding-right:0;padding-left:0}}@media (min-width:768px){.navbar-expand-md{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-md .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid,.navbar-expand-md>.container-lg,.navbar-expand-md>.container-md,.navbar-expand-md>.container-sm,.navbar-expand-md>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width:991.98px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{padding-right:0;padding-left:0}}@media (min-width:992px){.navbar-expand-lg{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-lg .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid,.navbar-expand-lg>.container-lg,.navbar-expand-lg>.container-md,.navbar-expand-lg>.container-sm,.navbar-expand-lg>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width:1199.98px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{padding-right:0;padding-left:0}}@media (min-width:1200px){.navbar-expand-xl{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-xl .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid,.navbar-expand-xl>.container-lg,.navbar-expand-xl>.container-md,.navbar-expand-xl>.container-sm,.navbar-expand-xl>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{-ms-flex-flow:row nowrap;flex-flow:row nowrap;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{-ms-flex-direction:row;flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid,.navbar-expand>.container-lg,.navbar-expand>.container-md,.navbar-expand>.container-sm,.navbar-expand>.container-xl{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:-ms-flexbox!important;display:flex!important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.5)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.5);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%280, 0, 0, 0.5%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,.5)}.navbar-light .navbar-text a{color:rgba(0,0,0,.9)}.navbar-light .navbar-text a:focus,.navbar-light .navbar-text a:hover{color:rgba(0,0,0,.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.5)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.5);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='30' height='30' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.5%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,.5)}.navbar-dark .navbar-text a{color:#fff}.navbar-dark .navbar-text a:focus,.navbar-dark .navbar-text a:hover{color:#fff}.card{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card>.list-group:last-child{border-bottom-width:0;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-body{-ms-flex:1 1 auto;flex:1 1 auto;min-height:1px;padding:1.25rem}.card-title{margin-bottom:.75rem}.card-subtitle{margin-top:-.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card-header{padding:.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-header+.list-group .list-group-item:first-child{border-top:0}.card-footer{padding:.75rem 1.25rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.625rem;margin-bottom:-.75rem;margin-left:-.625rem;border-bottom:0}.card-header-pills{margin-right:-.625rem;margin-left:-.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img,.card-img-bottom,.card-img-top{-ms-flex-negative:0;flex-shrink:0;width:100%}.card-img,.card-img-top{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-deck .card{margin-bottom:15px}@media (min-width:576px){.card-deck{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{-ms-flex:1 0 0%;flex:1 0 0%;margin-right:15px;margin-bottom:0;margin-left:15px}}.card-group>.card{margin-bottom:15px}@media (min-width:576px){.card-group{display:-ms-flexbox;display:flex;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group>.card{-ms-flex:1 0 0%;flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.card-columns .card{margin-bottom:.75rem}@media (min-width:576px){.card-columns{-webkit-column-count:3;-moz-column-count:3;column-count:3;-webkit-column-gap:1.25rem;-moz-column-gap:1.25rem;column-gap:1.25rem;orphans:1;widows:1}.card-columns .card{display:inline-block;width:100%}}.accordion>.card{overflow:hidden}.accordion>.card:not(:last-of-type){border-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.accordion>.card:not(:first-of-type){border-top-left-radius:0;border-top-right-radius:0}.accordion>.card>.card-header{border-radius:0;margin-bottom:-1px}.breadcrumb{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding:.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:.25rem}.breadcrumb-item{display:-ms-flexbox;display:flex}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:.5rem;color:#6c757d;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#6c757d}.pagination{display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:.25rem}.page-link{position:relative;display:block;padding:.5rem .75rem;margin-left:-1px;line-height:1.25;color:#007bff;background-color:#fff;border:1px solid #dee2e6}.page-link:hover{z-index:2;color:#0056b3;text-decoration:none;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:3;outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.25)}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.page-item.active .page-link{z-index:3;color:#fff;background-color:#007bff;border-color:#007bff}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;cursor:auto;background-color:#fff;border-color:#dee2e6}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.25em .4em;font-size:75%;font-weight:700;line-height:1;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.badge{transition:none}}a.badge:focus,a.badge:hover{text-decoration:none}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:.6em;padding-left:.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#007bff}a.badge-primary:focus,a.badge-primary:hover{color:#fff;background-color:#0062cc}a.badge-primary.focus,a.badge-primary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(0,123,255,.5)}.badge-secondary{color:#fff;background-color:#6c757d}a.badge-secondary:focus,a.badge-secondary:hover{color:#fff;background-color:#545b62}a.badge-secondary.focus,a.badge-secondary:focus{outline:0;box-shadow:0 0 0 .2rem rgba(108,117,125,.5)}.badge-success{color:#fff;background-color:#28a745}a.badge-success:focus,a.badge-success:hover{color:#fff;background-color:#1e7e34}a.badge-success.focus,a.badge-success:focus{outline:0;box-shadow:0 0 0 .2rem rgba(40,167,69,.5)}.badge-info{color:#fff;background-color:#17a2b8}a.badge-info:focus,a.badge-info:hover{color:#fff;background-color:#117a8b}a.badge-info.focus,a.badge-info:focus{outline:0;box-shadow:0 0 0 .2rem rgba(23,162,184,.5)}.badge-warning{color:#212529;background-color:#ffc107}a.badge-warning:focus,a.badge-warning:hover{color:#212529;background-color:#d39e00}a.badge-warning.focus,a.badge-warning:focus{outline:0;box-shadow:0 0 0 .2rem rgba(255,193,7,.5)}.badge-danger{color:#fff;background-color:#dc3545}a.badge-danger:focus,a.badge-danger:hover{color:#fff;background-color:#bd2130}a.badge-danger.focus,a.badge-danger:focus{outline:0;box-shadow:0 0 0 .2rem rgba(220,53,69,.5)}.badge-light{color:#212529;background-color:#f8f9fa}a.badge-light:focus,a.badge-light:hover{color:#212529;background-color:#dae0e5}a.badge-light.focus,a.badge-light:focus{outline:0;box-shadow:0 0 0 .2rem rgba(248,249,250,.5)}.badge-dark{color:#fff;background-color:#343a40}a.badge-dark:focus,a.badge-dark:hover{color:#fff;background-color:#1d2124}a.badge-dark.focus,a.badge-dark:focus{outline:0;box-shadow:0 0 0 .2rem rgba(52,58,64,.5)}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:.3rem}@media (min-width:576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{position:relative;padding:.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:4rem}.alert-dismissible .close{position:absolute;top:0;right:0;padding:.75rem 1.25rem;color:inherit}.alert-primary{color:#004085;background-color:#cce5ff;border-color:#b8daff}.alert-primary hr{border-top-color:#9fcdff}.alert-primary .alert-link{color:#002752}.alert-secondary{color:#383d41;background-color:#e2e3e5;border-color:#d6d8db}.alert-secondary hr{border-top-color:#c8cbcf}.alert-secondary .alert-link{color:#202326}.alert-success{color:#155724;background-color:#d4edda;border-color:#c3e6cb}.alert-success hr{border-top-color:#b1dfbb}.alert-success .alert-link{color:#0b2e13}.alert-info{color:#0c5460;background-color:#d1ecf1;border-color:#bee5eb}.alert-info hr{border-top-color:#abdde5}.alert-info .alert-link{color:#062c33}.alert-warning{color:#856404;background-color:#fff3cd;border-color:#ffeeba}.alert-warning hr{border-top-color:#ffe8a1}.alert-warning .alert-link{color:#533f03}.alert-danger{color:#721c24;background-color:#f8d7da;border-color:#f5c6cb}.alert-danger hr{border-top-color:#f1b0b7}.alert-danger .alert-link{color:#491217}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-ms-flexbox;display:flex;height:1rem;overflow:hidden;line-height:0;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;overflow:hidden;color:#fff;text-align:center;white-space:nowrap;background-color:#007bff;transition:width .6s ease}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}@media (prefers-reduced-motion:reduce){.progress-bar-animated{-webkit-animation:none;animation:none}}.media{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start}.media-body{-ms-flex:1;flex:1}.list-group{display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;border-radius:.25rem}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.75rem 1.25rem;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.list-group-item:last-child{border-bottom-right-radius:inherit;border-bottom-left-radius:inherit}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#007bff;border-color:#007bff}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:-1px;border-top-width:1px}.list-group-horizontal{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}@media (min-width:576px){.list-group-horizontal-sm{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-sm>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:768px){.list-group-horizontal-md{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-md>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:992px){.list-group-horizontal-lg{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-lg>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1200px){.list-group-horizontal-xl{-ms-flex-direction:row;flex-direction:row}.list-group-horizontal-xl>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}.list-group-flush{border-radius:0}.list-group-flush>.list-group-item{border-width:0 0 1px}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{color:#004085;background-color:#b8daff}.list-group-item-primary.list-group-item-action:focus,.list-group-item-primary.list-group-item-action:hover{color:#004085;background-color:#9fcdff}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#004085;border-color:#004085}.list-group-item-secondary{color:#383d41;background-color:#d6d8db}.list-group-item-secondary.list-group-item-action:focus,.list-group-item-secondary.list-group-item-action:hover{color:#383d41;background-color:#c8cbcf}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#383d41;border-color:#383d41}.list-group-item-success{color:#155724;background-color:#c3e6cb}.list-group-item-success.list-group-item-action:focus,.list-group-item-success.list-group-item-action:hover{color:#155724;background-color:#b1dfbb}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#155724;border-color:#155724}.list-group-item-info{color:#0c5460;background-color:#bee5eb}.list-group-item-info.list-group-item-action:focus,.list-group-item-info.list-group-item-action:hover{color:#0c5460;background-color:#abdde5}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#0c5460;border-color:#0c5460}.list-group-item-warning{color:#856404;background-color:#ffeeba}.list-group-item-warning.list-group-item-action:focus,.list-group-item-warning.list-group-item-action:hover{color:#856404;background-color:#ffe8a1}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#856404;border-color:#856404}.list-group-item-danger{color:#721c24;background-color:#f5c6cb}.list-group-item-danger.list-group-item-action:focus,.list-group-item-danger.list-group-item-action:hover{color:#721c24;background-color:#f1b0b7}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#721c24;border-color:#721c24}.list-group-item-light{color:#818182;background-color:#fdfdfe}.list-group-item-light.list-group-item-action:focus,.list-group-item-light.list-group-item-action:hover{color:#818182;background-color:#ececf6}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}.list-group-item-dark.list-group-item-action:focus,.list-group-item-dark.list-group-item-action:hover{color:#1b1e21;background-color:#b9bbbe}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:hover{color:#000;text-decoration:none}.close:not(:disabled):not(.disabled):focus,.close:not(:disabled):not(.disabled):hover{opacity:.75}button.close{padding:0;background-color:transparent;border:0}a.close.disabled{pointer-events:none}.toast{max-width:350px;overflow:hidden;font-size:.875rem;background-color:rgba(255,255,255,.85);background-clip:padding-box;border:1px solid rgba(0,0,0,.1);box-shadow:0 .25rem .75rem rgba(0,0,0,.1);-webkit-backdrop-filter:blur(10px);backdrop-filter:blur(10px);opacity:0;border-radius:.25rem}.toast:not(:last-child){margin-bottom:.75rem}.toast.showing{opacity:1}.toast.show{display:block;opacity:1}.toast.hide{display:none}.toast-header{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;padding:.25rem .75rem;color:#6c757d;background-color:rgba(255,255,255,.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,.05)}.toast-body{padding:.75rem}.modal-open{overflow:hidden}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal{position:fixed;top:0;left:0;z-index:1050;display:none;width:100%;height:100%;overflow:hidden;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:-webkit-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out;-webkit-transform:translate(0,-50px);transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{-webkit-transform:none;transform:none}.modal.modal-static .modal-dialog{-webkit-transform:scale(1.02);transform:scale(1.02)}.modal-dialog-scrollable{display:-ms-flexbox;display:flex;max-height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 1rem);overflow:hidden}.modal-dialog-scrollable .modal-footer,.modal-dialog-scrollable .modal-header{-ms-flex-negative:0;flex-shrink:0}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;min-height:calc(100% - 1rem)}.modal-dialog-centered::before{display:block;height:calc(100vh - 1rem);height:-webkit-min-content;height:-moz-min-content;height:min-content;content:""}.modal-dialog-centered.modal-dialog-scrollable{-ms-flex-direction:column;flex-direction:column;-ms-flex-pack:center;justify-content:center;height:100%}.modal-dialog-centered.modal-dialog-scrollable .modal-content{max-height:none}.modal-dialog-centered.modal-dialog-scrollable::before{content:none}.modal-content{position:relative;display:-ms-flexbox;display:flex;-ms-flex-direction:column;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:-ms-flexbox;display:flex;-ms-flex-align:start;align-items:flex-start;-ms-flex-pack:justify;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.modal-header .close{padding:1rem 1rem;margin:-1rem -1rem -1rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-ms-flex:1 1 auto;flex:1 1 auto;padding:1rem}.modal-footer{display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-ms-flex-align:center;align-items:center;-ms-flex-pack:end;justify-content:flex-end;padding:.75rem;border-top:1px solid #dee2e6;border-bottom-right-radius:calc(.3rem - 1px);border-bottom-left-radius:calc(.3rem - 1px)}.modal-footer>*{margin:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width:576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{max-height:calc(100% - 3.5rem)}.modal-dialog-scrollable .modal-content{max-height:calc(100vh - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-dialog-centered::before{height:calc(100vh - 3.5rem);height:-webkit-min-content;height:-moz-min-content;height:min-content}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width:1200px){.modal-xl{max-width:1140px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[x-placement^=top],.bs-tooltip-top{padding:.4rem 0}.bs-tooltip-auto[x-placement^=top] .arrow,.bs-tooltip-top .arrow{bottom:0}.bs-tooltip-auto[x-placement^=top] .arrow::before,.bs-tooltip-top .arrow::before{top:0;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-auto[x-placement^=right],.bs-tooltip-right{padding:0 .4rem}.bs-tooltip-auto[x-placement^=right] .arrow,.bs-tooltip-right .arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=right] .arrow::before,.bs-tooltip-right .arrow::before{right:0;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-auto[x-placement^=bottom],.bs-tooltip-bottom{padding:.4rem 0}.bs-tooltip-auto[x-placement^=bottom] .arrow,.bs-tooltip-bottom .arrow{top:0}.bs-tooltip-auto[x-placement^=bottom] .arrow::before,.bs-tooltip-bottom .arrow::before{bottom:0;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-auto[x-placement^=left],.bs-tooltip-left{padding:0 .4rem}.bs-tooltip-auto[x-placement^=left] .arrow,.bs-tooltip-left .arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-auto[x-placement^=left] .arrow::before,.bs-tooltip-left .arrow::before{left:0;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .arrow{position:absolute;display:block;width:1rem;height:.5rem;margin:0 .3rem}.popover .arrow::after,.popover .arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-auto[x-placement^=top],.bs-popover-top{margin-bottom:.5rem}.bs-popover-auto[x-placement^=top]>.arrow,.bs-popover-top>.arrow{bottom:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=top]>.arrow::before,.bs-popover-top>.arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=top]>.arrow::after,.bs-popover-top>.arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-auto[x-placement^=right],.bs-popover-right{margin-left:.5rem}.bs-popover-auto[x-placement^=right]>.arrow,.bs-popover-right>.arrow{left:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=right]>.arrow::before,.bs-popover-right>.arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=right]>.arrow::after,.bs-popover-right>.arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-auto[x-placement^=bottom],.bs-popover-bottom{margin-top:.5rem}.bs-popover-auto[x-placement^=bottom]>.arrow,.bs-popover-bottom>.arrow{top:calc(-.5rem - 1px)}.bs-popover-auto[x-placement^=bottom]>.arrow::before,.bs-popover-bottom>.arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=bottom]>.arrow::after,.bs-popover-bottom>.arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-auto[x-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f7f7f7}.bs-popover-auto[x-placement^=left],.bs-popover-left{margin-right:.5rem}.bs-popover-auto[x-placement^=left]>.arrow,.bs-popover-left>.arrow{right:calc(-.5rem - 1px);width:.5rem;height:1rem;margin:.3rem 0}.bs-popover-auto[x-placement^=left]>.arrow::before,.bs-popover-left>.arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,.25)}.bs-popover-auto[x-placement^=left]>.arrow::after,.bs-popover-left>.arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem .75rem;margin-bottom:0;font-size:1rem;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:.5rem .75rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{-ms-touch-action:pan-y;touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:-webkit-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-right,.carousel-item-next:not(.carousel-item-left){-webkit-transform:translateX(100%);transform:translateX(100%)}.active.carousel-item-left,.carousel-item-prev:not(.carousel-item-right){-webkit-transform:translateX(-100%);transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;-webkit-transform:none;transform:none}.carousel-fade .carousel-item-next.carousel-item-left,.carousel-fade .carousel-item-prev.carousel-item-right,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:-ms-flexbox;display:flex;-ms-flex-align:center;align-items:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:20px;height:20px;background:no-repeat 50%/100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M5.25 0l-4 4 4 4 1.5-1.5L4.25 4l2.5-2.5L5.25 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' width='8' height='8' viewBox='0 0 8 8'%3e%3cpath d='M2.75 0l-1.5 1.5L3.75 4l-2.5 2.5L2.75 8l4-4-4-4z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:15;display:-ms-flexbox;display:flex;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{box-sizing:content-box;-ms-flex:0 1 auto;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators li{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}@-webkit-keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes spinner-border{to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;-webkit-animation:spinner-border .75s linear infinite;animation:spinner-border .75s linear infinite}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@-webkit-keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1;-webkit-transform:none;transform:none}}@keyframes spinner-grow{0%{-webkit-transform:scale(0);transform:scale(0)}50%{opacity:1;-webkit-transform:none;transform:none}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:text-bottom;background-color:currentColor;border-radius:50%;opacity:0;-webkit-animation:spinner-grow .75s linear infinite;animation:spinner-grow .75s linear infinite}.spinner-grow-sm{width:1rem;height:1rem}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.bg-primary{background-color:#007bff!important}a.bg-primary:focus,a.bg-primary:hover,button.bg-primary:focus,button.bg-primary:hover{background-color:#0062cc!important}.bg-secondary{background-color:#6c757d!important}a.bg-secondary:focus,a.bg-secondary:hover,button.bg-secondary:focus,button.bg-secondary:hover{background-color:#545b62!important}.bg-success{background-color:#28a745!important}a.bg-success:focus,a.bg-success:hover,button.bg-success:focus,button.bg-success:hover{background-color:#1e7e34!important}.bg-info{background-color:#17a2b8!important}a.bg-info:focus,a.bg-info:hover,button.bg-info:focus,button.bg-info:hover{background-color:#117a8b!important}.bg-warning{background-color:#ffc107!important}a.bg-warning:focus,a.bg-warning:hover,button.bg-warning:focus,button.bg-warning:hover{background-color:#d39e00!important}.bg-danger{background-color:#dc3545!important}a.bg-danger:focus,a.bg-danger:hover,button.bg-danger:focus,button.bg-danger:hover{background-color:#bd2130!important}.bg-light{background-color:#f8f9fa!important}a.bg-light:focus,a.bg-light:hover,button.bg-light:focus,button.bg-light:hover{background-color:#dae0e5!important}.bg-dark{background-color:#343a40!important}a.bg-dark:focus,a.bg-dark:hover,button.bg-dark:focus,button.bg-dark:hover{background-color:#1d2124!important}.bg-white{background-color:#fff!important}.bg-transparent{background-color:transparent!important}.border{border:1px solid #dee2e6!important}.border-top{border-top:1px solid #dee2e6!important}.border-right{border-right:1px solid #dee2e6!important}.border-bottom{border-bottom:1px solid #dee2e6!important}.border-left{border-left:1px solid #dee2e6!important}.border-0{border:0!important}.border-top-0{border-top:0!important}.border-right-0{border-right:0!important}.border-bottom-0{border-bottom:0!important}.border-left-0{border-left:0!important}.border-primary{border-color:#007bff!important}.border-secondary{border-color:#6c757d!important}.border-success{border-color:#28a745!important}.border-info{border-color:#17a2b8!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#343a40!important}.border-white{border-color:#fff!important}.rounded-sm{border-radius:.2rem!important}.rounded{border-radius:.25rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-right{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-left{border-top-left-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-lg{border-radius:.3rem!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:50rem!important}.rounded-0{border-radius:0!important}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:-ms-flexbox!important;display:flex!important}.d-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}@media (min-width:576px){.d-sm-none{display:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:-ms-flexbox!important;display:flex!important}.d-sm-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:768px){.d-md-none{display:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:-ms-flexbox!important;display:flex!important}.d-md-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:992px){.d-lg-none{display:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:-ms-flexbox!important;display:flex!important}.d-lg-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media (min-width:1200px){.d-xl-none{display:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:-ms-flexbox!important;display:flex!important}.d-xl-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}@media print{.d-print-none{display:none!important}.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:-ms-flexbox!important;display:flex!important}.d-print-inline-flex{display:-ms-inline-flexbox!important;display:inline-flex!important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.857143%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-center{-ms-flex-align:center!important;align-items:center!important}.align-items-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}@media (min-width:576px){.flex-sm-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-sm-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-sm-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-sm-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-sm-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-sm-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-sm-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-sm-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-sm-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-sm-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-sm-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-sm-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-sm-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-sm-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-sm-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-sm-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-sm-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-sm-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-sm-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-sm-center{-ms-flex-align:center!important;align-items:center!important}.align-items-sm-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-sm-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-sm-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-sm-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-sm-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-sm-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-sm-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-sm-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-sm-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-sm-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-sm-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-sm-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-sm-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-sm-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:768px){.flex-md-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-md-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-md-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-md-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-md-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-md-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-md-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-md-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-md-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-md-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-md-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-md-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-md-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-md-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-md-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-md-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-md-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-md-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-md-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-md-center{-ms-flex-align:center!important;align-items:center!important}.align-items-md-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-md-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-md-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-md-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-md-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-md-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-md-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-md-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-md-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-md-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-md-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-md-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-md-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-md-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:992px){.flex-lg-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-lg-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-lg-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-lg-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-lg-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-lg-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-lg-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-lg-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-lg-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-lg-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-lg-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-lg-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-lg-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-lg-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-lg-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-lg-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-lg-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-lg-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-lg-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-lg-center{-ms-flex-align:center!important;align-items:center!important}.align-items-lg-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-lg-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-lg-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-lg-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-lg-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-lg-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-lg-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-lg-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-lg-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-lg-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-lg-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-lg-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-lg-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-lg-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}@media (min-width:1200px){.flex-xl-row{-ms-flex-direction:row!important;flex-direction:row!important}.flex-xl-column{-ms-flex-direction:column!important;flex-direction:column!important}.flex-xl-row-reverse{-ms-flex-direction:row-reverse!important;flex-direction:row-reverse!important}.flex-xl-column-reverse{-ms-flex-direction:column-reverse!important;flex-direction:column-reverse!important}.flex-xl-wrap{-ms-flex-wrap:wrap!important;flex-wrap:wrap!important}.flex-xl-nowrap{-ms-flex-wrap:nowrap!important;flex-wrap:nowrap!important}.flex-xl-wrap-reverse{-ms-flex-wrap:wrap-reverse!important;flex-wrap:wrap-reverse!important}.flex-xl-fill{-ms-flex:1 1 auto!important;flex:1 1 auto!important}.flex-xl-grow-0{-ms-flex-positive:0!important;flex-grow:0!important}.flex-xl-grow-1{-ms-flex-positive:1!important;flex-grow:1!important}.flex-xl-shrink-0{-ms-flex-negative:0!important;flex-shrink:0!important}.flex-xl-shrink-1{-ms-flex-negative:1!important;flex-shrink:1!important}.justify-content-xl-start{-ms-flex-pack:start!important;justify-content:flex-start!important}.justify-content-xl-end{-ms-flex-pack:end!important;justify-content:flex-end!important}.justify-content-xl-center{-ms-flex-pack:center!important;justify-content:center!important}.justify-content-xl-between{-ms-flex-pack:justify!important;justify-content:space-between!important}.justify-content-xl-around{-ms-flex-pack:distribute!important;justify-content:space-around!important}.align-items-xl-start{-ms-flex-align:start!important;align-items:flex-start!important}.align-items-xl-end{-ms-flex-align:end!important;align-items:flex-end!important}.align-items-xl-center{-ms-flex-align:center!important;align-items:center!important}.align-items-xl-baseline{-ms-flex-align:baseline!important;align-items:baseline!important}.align-items-xl-stretch{-ms-flex-align:stretch!important;align-items:stretch!important}.align-content-xl-start{-ms-flex-line-pack:start!important;align-content:flex-start!important}.align-content-xl-end{-ms-flex-line-pack:end!important;align-content:flex-end!important}.align-content-xl-center{-ms-flex-line-pack:center!important;align-content:center!important}.align-content-xl-between{-ms-flex-line-pack:justify!important;align-content:space-between!important}.align-content-xl-around{-ms-flex-line-pack:distribute!important;align-content:space-around!important}.align-content-xl-stretch{-ms-flex-line-pack:stretch!important;align-content:stretch!important}.align-self-xl-auto{-ms-flex-item-align:auto!important;align-self:auto!important}.align-self-xl-start{-ms-flex-item-align:start!important;align-self:flex-start!important}.align-self-xl-end{-ms-flex-item-align:end!important;align-self:flex-end!important}.align-self-xl-center{-ms-flex-item-align:center!important;align-self:center!important}.align-self-xl-baseline{-ms-flex-item-align:baseline!important;align-self:baseline!important}.align-self-xl-stretch{-ms-flex-item-align:stretch!important;align-self:stretch!important}}.float-left{float:left!important}.float-right{float:right!important}.float-none{float:none!important}@media (min-width:576px){.float-sm-left{float:left!important}.float-sm-right{float:right!important}.float-sm-none{float:none!important}}@media (min-width:768px){.float-md-left{float:left!important}.float-md-right{float:right!important}.float-md-none{float:none!important}}@media (min-width:992px){.float-lg-left{float:left!important}.float-lg-right{float:right!important}.float-lg-none{float:none!important}}@media (min-width:1200px){.float-xl-left{float:left!important}.float-xl-right{float:right!important}.float-xl-none{float:none!important}}.user-select-all{-webkit-user-select:all!important;-moz-user-select:all!important;-ms-user-select:all!important;user-select:all!important}.user-select-auto{-webkit-user-select:auto!important;-moz-user-select:auto!important;-ms-user-select:auto!important;user-select:auto!important}.user-select-none{-webkit-user-select:none!important;-moz-user-select:none!important;-ms-user-select:none!important;user-select:none!important}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports ((position:-webkit-sticky) or (position:sticky)){.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075)!important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15)!important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175)!important}.shadow-none{box-shadow:none!important}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mw-100{max-width:100%!important}.mh-100{max-height:100%!important}.min-vw-100{min-width:100vw!important}.min-vh-100{min-height:100vh!important}.vw-100{width:100vw!important}.vh-100{height:100vh!important}.m-0{margin:0!important}.mt-0,.my-0{margin-top:0!important}.mr-0,.mx-0{margin-right:0!important}.mb-0,.my-0{margin-bottom:0!important}.ml-0,.mx-0{margin-left:0!important}.m-1{margin:.25rem!important}.mt-1,.my-1{margin-top:.25rem!important}.mr-1,.mx-1{margin-right:.25rem!important}.mb-1,.my-1{margin-bottom:.25rem!important}.ml-1,.mx-1{margin-left:.25rem!important}.m-2{margin:.5rem!important}.mt-2,.my-2{margin-top:.5rem!important}.mr-2,.mx-2{margin-right:.5rem!important}.mb-2,.my-2{margin-bottom:.5rem!important}.ml-2,.mx-2{margin-left:.5rem!important}.m-3{margin:1rem!important}.mt-3,.my-3{margin-top:1rem!important}.mr-3,.mx-3{margin-right:1rem!important}.mb-3,.my-3{margin-bottom:1rem!important}.ml-3,.mx-3{margin-left:1rem!important}.m-4{margin:1.5rem!important}.mt-4,.my-4{margin-top:1.5rem!important}.mr-4,.mx-4{margin-right:1.5rem!important}.mb-4,.my-4{margin-bottom:1.5rem!important}.ml-4,.mx-4{margin-left:1.5rem!important}.m-5{margin:3rem!important}.mt-5,.my-5{margin-top:3rem!important}.mr-5,.mx-5{margin-right:3rem!important}.mb-5,.my-5{margin-bottom:3rem!important}.ml-5,.mx-5{margin-left:3rem!important}.p-0{padding:0!important}.pt-0,.py-0{padding-top:0!important}.pr-0,.px-0{padding-right:0!important}.pb-0,.py-0{padding-bottom:0!important}.pl-0,.px-0{padding-left:0!important}.p-1{padding:.25rem!important}.pt-1,.py-1{padding-top:.25rem!important}.pr-1,.px-1{padding-right:.25rem!important}.pb-1,.py-1{padding-bottom:.25rem!important}.pl-1,.px-1{padding-left:.25rem!important}.p-2{padding:.5rem!important}.pt-2,.py-2{padding-top:.5rem!important}.pr-2,.px-2{padding-right:.5rem!important}.pb-2,.py-2{padding-bottom:.5rem!important}.pl-2,.px-2{padding-left:.5rem!important}.p-3{padding:1rem!important}.pt-3,.py-3{padding-top:1rem!important}.pr-3,.px-3{padding-right:1rem!important}.pb-3,.py-3{padding-bottom:1rem!important}.pl-3,.px-3{padding-left:1rem!important}.p-4{padding:1.5rem!important}.pt-4,.py-4{padding-top:1.5rem!important}.pr-4,.px-4{padding-right:1.5rem!important}.pb-4,.py-4{padding-bottom:1.5rem!important}.pl-4,.px-4{padding-left:1.5rem!important}.p-5{padding:3rem!important}.pt-5,.py-5{padding-top:3rem!important}.pr-5,.px-5{padding-right:3rem!important}.pb-5,.py-5{padding-bottom:3rem!important}.pl-5,.px-5{padding-left:3rem!important}.m-n1{margin:-.25rem!important}.mt-n1,.my-n1{margin-top:-.25rem!important}.mr-n1,.mx-n1{margin-right:-.25rem!important}.mb-n1,.my-n1{margin-bottom:-.25rem!important}.ml-n1,.mx-n1{margin-left:-.25rem!important}.m-n2{margin:-.5rem!important}.mt-n2,.my-n2{margin-top:-.5rem!important}.mr-n2,.mx-n2{margin-right:-.5rem!important}.mb-n2,.my-n2{margin-bottom:-.5rem!important}.ml-n2,.mx-n2{margin-left:-.5rem!important}.m-n3{margin:-1rem!important}.mt-n3,.my-n3{margin-top:-1rem!important}.mr-n3,.mx-n3{margin-right:-1rem!important}.mb-n3,.my-n3{margin-bottom:-1rem!important}.ml-n3,.mx-n3{margin-left:-1rem!important}.m-n4{margin:-1.5rem!important}.mt-n4,.my-n4{margin-top:-1.5rem!important}.mr-n4,.mx-n4{margin-right:-1.5rem!important}.mb-n4,.my-n4{margin-bottom:-1.5rem!important}.ml-n4,.mx-n4{margin-left:-1.5rem!important}.m-n5{margin:-3rem!important}.mt-n5,.my-n5{margin-top:-3rem!important}.mr-n5,.mx-n5{margin-right:-3rem!important}.mb-n5,.my-n5{margin-bottom:-3rem!important}.ml-n5,.mx-n5{margin-left:-3rem!important}.m-auto{margin:auto!important}.mt-auto,.my-auto{margin-top:auto!important}.mr-auto,.mx-auto{margin-right:auto!important}.mb-auto,.my-auto{margin-bottom:auto!important}.ml-auto,.mx-auto{margin-left:auto!important}@media (min-width:576px){.m-sm-0{margin:0!important}.mt-sm-0,.my-sm-0{margin-top:0!important}.mr-sm-0,.mx-sm-0{margin-right:0!important}.mb-sm-0,.my-sm-0{margin-bottom:0!important}.ml-sm-0,.mx-sm-0{margin-left:0!important}.m-sm-1{margin:.25rem!important}.mt-sm-1,.my-sm-1{margin-top:.25rem!important}.mr-sm-1,.mx-sm-1{margin-right:.25rem!important}.mb-sm-1,.my-sm-1{margin-bottom:.25rem!important}.ml-sm-1,.mx-sm-1{margin-left:.25rem!important}.m-sm-2{margin:.5rem!important}.mt-sm-2,.my-sm-2{margin-top:.5rem!important}.mr-sm-2,.mx-sm-2{margin-right:.5rem!important}.mb-sm-2,.my-sm-2{margin-bottom:.5rem!important}.ml-sm-2,.mx-sm-2{margin-left:.5rem!important}.m-sm-3{margin:1rem!important}.mt-sm-3,.my-sm-3{margin-top:1rem!important}.mr-sm-3,.mx-sm-3{margin-right:1rem!important}.mb-sm-3,.my-sm-3{margin-bottom:1rem!important}.ml-sm-3,.mx-sm-3{margin-left:1rem!important}.m-sm-4{margin:1.5rem!important}.mt-sm-4,.my-sm-4{margin-top:1.5rem!important}.mr-sm-4,.mx-sm-4{margin-right:1.5rem!important}.mb-sm-4,.my-sm-4{margin-bottom:1.5rem!important}.ml-sm-4,.mx-sm-4{margin-left:1.5rem!important}.m-sm-5{margin:3rem!important}.mt-sm-5,.my-sm-5{margin-top:3rem!important}.mr-sm-5,.mx-sm-5{margin-right:3rem!important}.mb-sm-5,.my-sm-5{margin-bottom:3rem!important}.ml-sm-5,.mx-sm-5{margin-left:3rem!important}.p-sm-0{padding:0!important}.pt-sm-0,.py-sm-0{padding-top:0!important}.pr-sm-0,.px-sm-0{padding-right:0!important}.pb-sm-0,.py-sm-0{padding-bottom:0!important}.pl-sm-0,.px-sm-0{padding-left:0!important}.p-sm-1{padding:.25rem!important}.pt-sm-1,.py-sm-1{padding-top:.25rem!important}.pr-sm-1,.px-sm-1{padding-right:.25rem!important}.pb-sm-1,.py-sm-1{padding-bottom:.25rem!important}.pl-sm-1,.px-sm-1{padding-left:.25rem!important}.p-sm-2{padding:.5rem!important}.pt-sm-2,.py-sm-2{padding-top:.5rem!important}.pr-sm-2,.px-sm-2{padding-right:.5rem!important}.pb-sm-2,.py-sm-2{padding-bottom:.5rem!important}.pl-sm-2,.px-sm-2{padding-left:.5rem!important}.p-sm-3{padding:1rem!important}.pt-sm-3,.py-sm-3{padding-top:1rem!important}.pr-sm-3,.px-sm-3{padding-right:1rem!important}.pb-sm-3,.py-sm-3{padding-bottom:1rem!important}.pl-sm-3,.px-sm-3{padding-left:1rem!important}.p-sm-4{padding:1.5rem!important}.pt-sm-4,.py-sm-4{padding-top:1.5rem!important}.pr-sm-4,.px-sm-4{padding-right:1.5rem!important}.pb-sm-4,.py-sm-4{padding-bottom:1.5rem!important}.pl-sm-4,.px-sm-4{padding-left:1.5rem!important}.p-sm-5{padding:3rem!important}.pt-sm-5,.py-sm-5{padding-top:3rem!important}.pr-sm-5,.px-sm-5{padding-right:3rem!important}.pb-sm-5,.py-sm-5{padding-bottom:3rem!important}.pl-sm-5,.px-sm-5{padding-left:3rem!important}.m-sm-n1{margin:-.25rem!important}.mt-sm-n1,.my-sm-n1{margin-top:-.25rem!important}.mr-sm-n1,.mx-sm-n1{margin-right:-.25rem!important}.mb-sm-n1,.my-sm-n1{margin-bottom:-.25rem!important}.ml-sm-n1,.mx-sm-n1{margin-left:-.25rem!important}.m-sm-n2{margin:-.5rem!important}.mt-sm-n2,.my-sm-n2{margin-top:-.5rem!important}.mr-sm-n2,.mx-sm-n2{margin-right:-.5rem!important}.mb-sm-n2,.my-sm-n2{margin-bottom:-.5rem!important}.ml-sm-n2,.mx-sm-n2{margin-left:-.5rem!important}.m-sm-n3{margin:-1rem!important}.mt-sm-n3,.my-sm-n3{margin-top:-1rem!important}.mr-sm-n3,.mx-sm-n3{margin-right:-1rem!important}.mb-sm-n3,.my-sm-n3{margin-bottom:-1rem!important}.ml-sm-n3,.mx-sm-n3{margin-left:-1rem!important}.m-sm-n4{margin:-1.5rem!important}.mt-sm-n4,.my-sm-n4{margin-top:-1.5rem!important}.mr-sm-n4,.mx-sm-n4{margin-right:-1.5rem!important}.mb-sm-n4,.my-sm-n4{margin-bottom:-1.5rem!important}.ml-sm-n4,.mx-sm-n4{margin-left:-1.5rem!important}.m-sm-n5{margin:-3rem!important}.mt-sm-n5,.my-sm-n5{margin-top:-3rem!important}.mr-sm-n5,.mx-sm-n5{margin-right:-3rem!important}.mb-sm-n5,.my-sm-n5{margin-bottom:-3rem!important}.ml-sm-n5,.mx-sm-n5{margin-left:-3rem!important}.m-sm-auto{margin:auto!important}.mt-sm-auto,.my-sm-auto{margin-top:auto!important}.mr-sm-auto,.mx-sm-auto{margin-right:auto!important}.mb-sm-auto,.my-sm-auto{margin-bottom:auto!important}.ml-sm-auto,.mx-sm-auto{margin-left:auto!important}}@media (min-width:768px){.m-md-0{margin:0!important}.mt-md-0,.my-md-0{margin-top:0!important}.mr-md-0,.mx-md-0{margin-right:0!important}.mb-md-0,.my-md-0{margin-bottom:0!important}.ml-md-0,.mx-md-0{margin-left:0!important}.m-md-1{margin:.25rem!important}.mt-md-1,.my-md-1{margin-top:.25rem!important}.mr-md-1,.mx-md-1{margin-right:.25rem!important}.mb-md-1,.my-md-1{margin-bottom:.25rem!important}.ml-md-1,.mx-md-1{margin-left:.25rem!important}.m-md-2{margin:.5rem!important}.mt-md-2,.my-md-2{margin-top:.5rem!important}.mr-md-2,.mx-md-2{margin-right:.5rem!important}.mb-md-2,.my-md-2{margin-bottom:.5rem!important}.ml-md-2,.mx-md-2{margin-left:.5rem!important}.m-md-3{margin:1rem!important}.mt-md-3,.my-md-3{margin-top:1rem!important}.mr-md-3,.mx-md-3{margin-right:1rem!important}.mb-md-3,.my-md-3{margin-bottom:1rem!important}.ml-md-3,.mx-md-3{margin-left:1rem!important}.m-md-4{margin:1.5rem!important}.mt-md-4,.my-md-4{margin-top:1.5rem!important}.mr-md-4,.mx-md-4{margin-right:1.5rem!important}.mb-md-4,.my-md-4{margin-bottom:1.5rem!important}.ml-md-4,.mx-md-4{margin-left:1.5rem!important}.m-md-5{margin:3rem!important}.mt-md-5,.my-md-5{margin-top:3rem!important}.mr-md-5,.mx-md-5{margin-right:3rem!important}.mb-md-5,.my-md-5{margin-bottom:3rem!important}.ml-md-5,.mx-md-5{margin-left:3rem!important}.p-md-0{padding:0!important}.pt-md-0,.py-md-0{padding-top:0!important}.pr-md-0,.px-md-0{padding-right:0!important}.pb-md-0,.py-md-0{padding-bottom:0!important}.pl-md-0,.px-md-0{padding-left:0!important}.p-md-1{padding:.25rem!important}.pt-md-1,.py-md-1{padding-top:.25rem!important}.pr-md-1,.px-md-1{padding-right:.25rem!important}.pb-md-1,.py-md-1{padding-bottom:.25rem!important}.pl-md-1,.px-md-1{padding-left:.25rem!important}.p-md-2{padding:.5rem!important}.pt-md-2,.py-md-2{padding-top:.5rem!important}.pr-md-2,.px-md-2{padding-right:.5rem!important}.pb-md-2,.py-md-2{padding-bottom:.5rem!important}.pl-md-2,.px-md-2{padding-left:.5rem!important}.p-md-3{padding:1rem!important}.pt-md-3,.py-md-3{padding-top:1rem!important}.pr-md-3,.px-md-3{padding-right:1rem!important}.pb-md-3,.py-md-3{padding-bottom:1rem!important}.pl-md-3,.px-md-3{padding-left:1rem!important}.p-md-4{padding:1.5rem!important}.pt-md-4,.py-md-4{padding-top:1.5rem!important}.pr-md-4,.px-md-4{padding-right:1.5rem!important}.pb-md-4,.py-md-4{padding-bottom:1.5rem!important}.pl-md-4,.px-md-4{padding-left:1.5rem!important}.p-md-5{padding:3rem!important}.pt-md-5,.py-md-5{padding-top:3rem!important}.pr-md-5,.px-md-5{padding-right:3rem!important}.pb-md-5,.py-md-5{padding-bottom:3rem!important}.pl-md-5,.px-md-5{padding-left:3rem!important}.m-md-n1{margin:-.25rem!important}.mt-md-n1,.my-md-n1{margin-top:-.25rem!important}.mr-md-n1,.mx-md-n1{margin-right:-.25rem!important}.mb-md-n1,.my-md-n1{margin-bottom:-.25rem!important}.ml-md-n1,.mx-md-n1{margin-left:-.25rem!important}.m-md-n2{margin:-.5rem!important}.mt-md-n2,.my-md-n2{margin-top:-.5rem!important}.mr-md-n2,.mx-md-n2{margin-right:-.5rem!important}.mb-md-n2,.my-md-n2{margin-bottom:-.5rem!important}.ml-md-n2,.mx-md-n2{margin-left:-.5rem!important}.m-md-n3{margin:-1rem!important}.mt-md-n3,.my-md-n3{margin-top:-1rem!important}.mr-md-n3,.mx-md-n3{margin-right:-1rem!important}.mb-md-n3,.my-md-n3{margin-bottom:-1rem!important}.ml-md-n3,.mx-md-n3{margin-left:-1rem!important}.m-md-n4{margin:-1.5rem!important}.mt-md-n4,.my-md-n4{margin-top:-1.5rem!important}.mr-md-n4,.mx-md-n4{margin-right:-1.5rem!important}.mb-md-n4,.my-md-n4{margin-bottom:-1.5rem!important}.ml-md-n4,.mx-md-n4{margin-left:-1.5rem!important}.m-md-n5{margin:-3rem!important}.mt-md-n5,.my-md-n5{margin-top:-3rem!important}.mr-md-n5,.mx-md-n5{margin-right:-3rem!important}.mb-md-n5,.my-md-n5{margin-bottom:-3rem!important}.ml-md-n5,.mx-md-n5{margin-left:-3rem!important}.m-md-auto{margin:auto!important}.mt-md-auto,.my-md-auto{margin-top:auto!important}.mr-md-auto,.mx-md-auto{margin-right:auto!important}.mb-md-auto,.my-md-auto{margin-bottom:auto!important}.ml-md-auto,.mx-md-auto{margin-left:auto!important}}@media (min-width:992px){.m-lg-0{margin:0!important}.mt-lg-0,.my-lg-0{margin-top:0!important}.mr-lg-0,.mx-lg-0{margin-right:0!important}.mb-lg-0,.my-lg-0{margin-bottom:0!important}.ml-lg-0,.mx-lg-0{margin-left:0!important}.m-lg-1{margin:.25rem!important}.mt-lg-1,.my-lg-1{margin-top:.25rem!important}.mr-lg-1,.mx-lg-1{margin-right:.25rem!important}.mb-lg-1,.my-lg-1{margin-bottom:.25rem!important}.ml-lg-1,.mx-lg-1{margin-left:.25rem!important}.m-lg-2{margin:.5rem!important}.mt-lg-2,.my-lg-2{margin-top:.5rem!important}.mr-lg-2,.mx-lg-2{margin-right:.5rem!important}.mb-lg-2,.my-lg-2{margin-bottom:.5rem!important}.ml-lg-2,.mx-lg-2{margin-left:.5rem!important}.m-lg-3{margin:1rem!important}.mt-lg-3,.my-lg-3{margin-top:1rem!important}.mr-lg-3,.mx-lg-3{margin-right:1rem!important}.mb-lg-3,.my-lg-3{margin-bottom:1rem!important}.ml-lg-3,.mx-lg-3{margin-left:1rem!important}.m-lg-4{margin:1.5rem!important}.mt-lg-4,.my-lg-4{margin-top:1.5rem!important}.mr-lg-4,.mx-lg-4{margin-right:1.5rem!important}.mb-lg-4,.my-lg-4{margin-bottom:1.5rem!important}.ml-lg-4,.mx-lg-4{margin-left:1.5rem!important}.m-lg-5{margin:3rem!important}.mt-lg-5,.my-lg-5{margin-top:3rem!important}.mr-lg-5,.mx-lg-5{margin-right:3rem!important}.mb-lg-5,.my-lg-5{margin-bottom:3rem!important}.ml-lg-5,.mx-lg-5{margin-left:3rem!important}.p-lg-0{padding:0!important}.pt-lg-0,.py-lg-0{padding-top:0!important}.pr-lg-0,.px-lg-0{padding-right:0!important}.pb-lg-0,.py-lg-0{padding-bottom:0!important}.pl-lg-0,.px-lg-0{padding-left:0!important}.p-lg-1{padding:.25rem!important}.pt-lg-1,.py-lg-1{padding-top:.25rem!important}.pr-lg-1,.px-lg-1{padding-right:.25rem!important}.pb-lg-1,.py-lg-1{padding-bottom:.25rem!important}.pl-lg-1,.px-lg-1{padding-left:.25rem!important}.p-lg-2{padding:.5rem!important}.pt-lg-2,.py-lg-2{padding-top:.5rem!important}.pr-lg-2,.px-lg-2{padding-right:.5rem!important}.pb-lg-2,.py-lg-2{padding-bottom:.5rem!important}.pl-lg-2,.px-lg-2{padding-left:.5rem!important}.p-lg-3{padding:1rem!important}.pt-lg-3,.py-lg-3{padding-top:1rem!important}.pr-lg-3,.px-lg-3{padding-right:1rem!important}.pb-lg-3,.py-lg-3{padding-bottom:1rem!important}.pl-lg-3,.px-lg-3{padding-left:1rem!important}.p-lg-4{padding:1.5rem!important}.pt-lg-4,.py-lg-4{padding-top:1.5rem!important}.pr-lg-4,.px-lg-4{padding-right:1.5rem!important}.pb-lg-4,.py-lg-4{padding-bottom:1.5rem!important}.pl-lg-4,.px-lg-4{padding-left:1.5rem!important}.p-lg-5{padding:3rem!important}.pt-lg-5,.py-lg-5{padding-top:3rem!important}.pr-lg-5,.px-lg-5{padding-right:3rem!important}.pb-lg-5,.py-lg-5{padding-bottom:3rem!important}.pl-lg-5,.px-lg-5{padding-left:3rem!important}.m-lg-n1{margin:-.25rem!important}.mt-lg-n1,.my-lg-n1{margin-top:-.25rem!important}.mr-lg-n1,.mx-lg-n1{margin-right:-.25rem!important}.mb-lg-n1,.my-lg-n1{margin-bottom:-.25rem!important}.ml-lg-n1,.mx-lg-n1{margin-left:-.25rem!important}.m-lg-n2{margin:-.5rem!important}.mt-lg-n2,.my-lg-n2{margin-top:-.5rem!important}.mr-lg-n2,.mx-lg-n2{margin-right:-.5rem!important}.mb-lg-n2,.my-lg-n2{margin-bottom:-.5rem!important}.ml-lg-n2,.mx-lg-n2{margin-left:-.5rem!important}.m-lg-n3{margin:-1rem!important}.mt-lg-n3,.my-lg-n3{margin-top:-1rem!important}.mr-lg-n3,.mx-lg-n3{margin-right:-1rem!important}.mb-lg-n3,.my-lg-n3{margin-bottom:-1rem!important}.ml-lg-n3,.mx-lg-n3{margin-left:-1rem!important}.m-lg-n4{margin:-1.5rem!important}.mt-lg-n4,.my-lg-n4{margin-top:-1.5rem!important}.mr-lg-n4,.mx-lg-n4{margin-right:-1.5rem!important}.mb-lg-n4,.my-lg-n4{margin-bottom:-1.5rem!important}.ml-lg-n4,.mx-lg-n4{margin-left:-1.5rem!important}.m-lg-n5{margin:-3rem!important}.mt-lg-n5,.my-lg-n5{margin-top:-3rem!important}.mr-lg-n5,.mx-lg-n5{margin-right:-3rem!important}.mb-lg-n5,.my-lg-n5{margin-bottom:-3rem!important}.ml-lg-n5,.mx-lg-n5{margin-left:-3rem!important}.m-lg-auto{margin:auto!important}.mt-lg-auto,.my-lg-auto{margin-top:auto!important}.mr-lg-auto,.mx-lg-auto{margin-right:auto!important}.mb-lg-auto,.my-lg-auto{margin-bottom:auto!important}.ml-lg-auto,.mx-lg-auto{margin-left:auto!important}}@media (min-width:1200px){.m-xl-0{margin:0!important}.mt-xl-0,.my-xl-0{margin-top:0!important}.mr-xl-0,.mx-xl-0{margin-right:0!important}.mb-xl-0,.my-xl-0{margin-bottom:0!important}.ml-xl-0,.mx-xl-0{margin-left:0!important}.m-xl-1{margin:.25rem!important}.mt-xl-1,.my-xl-1{margin-top:.25rem!important}.mr-xl-1,.mx-xl-1{margin-right:.25rem!important}.mb-xl-1,.my-xl-1{margin-bottom:.25rem!important}.ml-xl-1,.mx-xl-1{margin-left:.25rem!important}.m-xl-2{margin:.5rem!important}.mt-xl-2,.my-xl-2{margin-top:.5rem!important}.mr-xl-2,.mx-xl-2{margin-right:.5rem!important}.mb-xl-2,.my-xl-2{margin-bottom:.5rem!important}.ml-xl-2,.mx-xl-2{margin-left:.5rem!important}.m-xl-3{margin:1rem!important}.mt-xl-3,.my-xl-3{margin-top:1rem!important}.mr-xl-3,.mx-xl-3{margin-right:1rem!important}.mb-xl-3,.my-xl-3{margin-bottom:1rem!important}.ml-xl-3,.mx-xl-3{margin-left:1rem!important}.m-xl-4{margin:1.5rem!important}.mt-xl-4,.my-xl-4{margin-top:1.5rem!important}.mr-xl-4,.mx-xl-4{margin-right:1.5rem!important}.mb-xl-4,.my-xl-4{margin-bottom:1.5rem!important}.ml-xl-4,.mx-xl-4{margin-left:1.5rem!important}.m-xl-5{margin:3rem!important}.mt-xl-5,.my-xl-5{margin-top:3rem!important}.mr-xl-5,.mx-xl-5{margin-right:3rem!important}.mb-xl-5,.my-xl-5{margin-bottom:3rem!important}.ml-xl-5,.mx-xl-5{margin-left:3rem!important}.p-xl-0{padding:0!important}.pt-xl-0,.py-xl-0{padding-top:0!important}.pr-xl-0,.px-xl-0{padding-right:0!important}.pb-xl-0,.py-xl-0{padding-bottom:0!important}.pl-xl-0,.px-xl-0{padding-left:0!important}.p-xl-1{padding:.25rem!important}.pt-xl-1,.py-xl-1{padding-top:.25rem!important}.pr-xl-1,.px-xl-1{padding-right:.25rem!important}.pb-xl-1,.py-xl-1{padding-bottom:.25rem!important}.pl-xl-1,.px-xl-1{padding-left:.25rem!important}.p-xl-2{padding:.5rem!important}.pt-xl-2,.py-xl-2{padding-top:.5rem!important}.pr-xl-2,.px-xl-2{padding-right:.5rem!important}.pb-xl-2,.py-xl-2{padding-bottom:.5rem!important}.pl-xl-2,.px-xl-2{padding-left:.5rem!important}.p-xl-3{padding:1rem!important}.pt-xl-3,.py-xl-3{padding-top:1rem!important}.pr-xl-3,.px-xl-3{padding-right:1rem!important}.pb-xl-3,.py-xl-3{padding-bottom:1rem!important}.pl-xl-3,.px-xl-3{padding-left:1rem!important}.p-xl-4{padding:1.5rem!important}.pt-xl-4,.py-xl-4{padding-top:1.5rem!important}.pr-xl-4,.px-xl-4{padding-right:1.5rem!important}.pb-xl-4,.py-xl-4{padding-bottom:1.5rem!important}.pl-xl-4,.px-xl-4{padding-left:1.5rem!important}.p-xl-5{padding:3rem!important}.pt-xl-5,.py-xl-5{padding-top:3rem!important}.pr-xl-5,.px-xl-5{padding-right:3rem!important}.pb-xl-5,.py-xl-5{padding-bottom:3rem!important}.pl-xl-5,.px-xl-5{padding-left:3rem!important}.m-xl-n1{margin:-.25rem!important}.mt-xl-n1,.my-xl-n1{margin-top:-.25rem!important}.mr-xl-n1,.mx-xl-n1{margin-right:-.25rem!important}.mb-xl-n1,.my-xl-n1{margin-bottom:-.25rem!important}.ml-xl-n1,.mx-xl-n1{margin-left:-.25rem!important}.m-xl-n2{margin:-.5rem!important}.mt-xl-n2,.my-xl-n2{margin-top:-.5rem!important}.mr-xl-n2,.mx-xl-n2{margin-right:-.5rem!important}.mb-xl-n2,.my-xl-n2{margin-bottom:-.5rem!important}.ml-xl-n2,.mx-xl-n2{margin-left:-.5rem!important}.m-xl-n3{margin:-1rem!important}.mt-xl-n3,.my-xl-n3{margin-top:-1rem!important}.mr-xl-n3,.mx-xl-n3{margin-right:-1rem!important}.mb-xl-n3,.my-xl-n3{margin-bottom:-1rem!important}.ml-xl-n3,.mx-xl-n3{margin-left:-1rem!important}.m-xl-n4{margin:-1.5rem!important}.mt-xl-n4,.my-xl-n4{margin-top:-1.5rem!important}.mr-xl-n4,.mx-xl-n4{margin-right:-1.5rem!important}.mb-xl-n4,.my-xl-n4{margin-bottom:-1.5rem!important}.ml-xl-n4,.mx-xl-n4{margin-left:-1.5rem!important}.m-xl-n5{margin:-3rem!important}.mt-xl-n5,.my-xl-n5{margin-top:-3rem!important}.mr-xl-n5,.mx-xl-n5{margin-right:-3rem!important}.mb-xl-n5,.my-xl-n5{margin-bottom:-3rem!important}.ml-xl-n5,.mx-xl-n5{margin-left:-3rem!important}.m-xl-auto{margin:auto!important}.mt-xl-auto,.my-xl-auto{margin-top:auto!important}.mr-xl-auto,.mx-xl-auto{margin-right:auto!important}.mb-xl-auto,.my-xl-auto{margin-bottom:auto!important}.ml-xl-auto,.mx-xl-auto{margin-left:auto!important}}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;pointer-events:auto;content:"";background-color:rgba(0,0,0,0)}.text-monospace{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace!important}.text-justify{text-align:justify!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left!important}.text-right{text-align:right!important}.text-center{text-align:center!important}@media (min-width:576px){.text-sm-left{text-align:left!important}.text-sm-right{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.text-md-left{text-align:left!important}.text-md-right{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.text-lg-left{text-align:left!important}.text-lg-right{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.text-xl-left{text-align:left!important}.text-xl-right{text-align:right!important}.text-xl-center{text-align:center!important}}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.font-weight-light{font-weight:300!important}.font-weight-lighter{font-weight:lighter!important}.font-weight-normal{font-weight:400!important}.font-weight-bold{font-weight:700!important}.font-weight-bolder{font-weight:bolder!important}.font-italic{font-style:italic!important}.text-white{color:#fff!important}.text-primary{color:#007bff!important}a.text-primary:focus,a.text-primary:hover{color:#0056b3!important}.text-secondary{color:#6c757d!important}a.text-secondary:focus,a.text-secondary:hover{color:#494f54!important}.text-success{color:#28a745!important}a.text-success:focus,a.text-success:hover{color:#19692c!important}.text-info{color:#17a2b8!important}a.text-info:focus,a.text-info:hover{color:#0f6674!important}.text-warning{color:#ffc107!important}a.text-warning:focus,a.text-warning:hover{color:#ba8b00!important}.text-danger{color:#dc3545!important}a.text-danger:focus,a.text-danger:hover{color:#a71d2a!important}.text-light{color:#f8f9fa!important}a.text-light:focus,a.text-light:hover{color:#cbd3da!important}.text-dark{color:#343a40!important}a.text-dark:focus,a.text-dark:hover{color:#121416!important}.text-body{color:#212529!important}.text-muted{color:#6c757d!important}.text-black-50{color:rgba(0,0,0,.5)!important}.text-white-50{color:rgba(255,255,255,.5)!important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.text-decoration-none{text-decoration:none!important}.text-break{word-wrap:break-word!important}.text-reset{color:inherit!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}@media print{*,::after,::before{text-shadow:none!important;box-shadow:none!important}a:not(.btn){text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap!important}blockquote,pre{border:1px solid #adb5bd;page-break-inside:avoid}thead{display:table-header-group}img,tr{page-break-inside:avoid}h2,h3,p{orphans:3;widows:3}h2,h3{page-break-after:avoid}@page{size:a3}body{min-width:992px!important}.container{min-width:992px!important}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse!important}.table td,.table th{background-color:#fff!important}.table-bordered td,.table-bordered th{border:1px solid #dee2e6!important}.table-dark{color:inherit}.table-dark tbody+tbody,.table-dark td,.table-dark th,.table-dark thead th{border-color:#dee2e6}.table .thead-dark th{color:inherit;border-color:#dee2e6}} -/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/httpserver/static-content/bootstrap.min.js b/httpserver/static-content/bootstrap.min.js deleted file mode 100644 index 3ecf55f2..00000000 --- a/httpserver/static-content/bootstrap.min.js +++ /dev/null @@ -1,7 +0,0 @@ -/*! - * Bootstrap v4.5.0 (https://getbootstrap.com/) - * Copyright 2011-2020 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */ -!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports,require("jquery"),require("popper.js")):"function"==typeof define&&define.amd?define(["exports","jquery","popper.js"],e):e((t=t||self).bootstrap={},t.jQuery,t.Popper)}(this,(function(t,e,n){"use strict";function i(t,e){for(var n=0;n=4)throw new Error("Bootstrap's JavaScript requires at least jQuery v1.9.1 but less than v4.0.0")}};c.jQueryDetection(),e.fn.emulateTransitionEnd=l,e.event.special[c.TRANSITION_END]={bindType:"transitionend",delegateType:"transitionend",handle:function(t){if(e(t.target).is(this))return t.handleObj.handler.apply(this,arguments)}};var h="alert",u=e.fn[h],d=function(){function t(t){this._element=t}var n=t.prototype;return n.close=function(t){var e=this._element;t&&(e=this._getRootElement(t)),this._triggerCloseEvent(e).isDefaultPrevented()||this._removeElement(e)},n.dispose=function(){e.removeData(this._element,"bs.alert"),this._element=null},n._getRootElement=function(t){var n=c.getSelectorFromElement(t),i=!1;return n&&(i=document.querySelector(n)),i||(i=e(t).closest(".alert")[0]),i},n._triggerCloseEvent=function(t){var n=e.Event("close.bs.alert");return e(t).trigger(n),n},n._removeElement=function(t){var n=this;if(e(t).removeClass("show"),e(t).hasClass("fade")){var i=c.getTransitionDurationFromElement(t);e(t).one(c.TRANSITION_END,(function(e){return n._destroyElement(t,e)})).emulateTransitionEnd(i)}else this._destroyElement(t)},n._destroyElement=function(t){e(t).detach().trigger("closed.bs.alert").remove()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.alert");o||(o=new t(this),i.data("bs.alert",o)),"close"===n&&o[n](this)}))},t._handleDismiss=function(t){return function(e){e&&e.preventDefault(),t.close(this)}},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.alert.data-api",'[data-dismiss="alert"]',d._handleDismiss(new d)),e.fn[h]=d._jQueryInterface,e.fn[h].Constructor=d,e.fn[h].noConflict=function(){return e.fn[h]=u,d._jQueryInterface};var f=e.fn.button,g=function(){function t(t){this._element=t}var n=t.prototype;return n.toggle=function(){var t=!0,n=!0,i=e(this._element).closest('[data-toggle="buttons"]')[0];if(i){var o=this._element.querySelector('input:not([type="hidden"])');if(o){if("radio"===o.type)if(o.checked&&this._element.classList.contains("active"))t=!1;else{var s=i.querySelector(".active");s&&e(s).removeClass("active")}t&&("checkbox"!==o.type&&"radio"!==o.type||(o.checked=!this._element.classList.contains("active")),e(o).trigger("change")),o.focus(),n=!1}}this._element.hasAttribute("disabled")||this._element.classList.contains("disabled")||(n&&this._element.setAttribute("aria-pressed",!this._element.classList.contains("active")),t&&e(this._element).toggleClass("active"))},n.dispose=function(){e.removeData(this._element,"bs.button"),this._element=null},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.button");i||(i=new t(this),e(this).data("bs.button",i)),"toggle"===n&&i[n]()}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.button.data-api",'[data-toggle^="button"]',(function(t){var n=t.target,i=n;if(e(n).hasClass("btn")||(n=e(n).closest(".btn")[0]),!n||n.hasAttribute("disabled")||n.classList.contains("disabled"))t.preventDefault();else{var o=n.querySelector('input:not([type="hidden"])');if(o&&(o.hasAttribute("disabled")||o.classList.contains("disabled")))return void t.preventDefault();"LABEL"===i.tagName&&o&&"checkbox"===o.type&&t.preventDefault(),g._jQueryInterface.call(e(n),"toggle")}})).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',(function(t){var n=e(t.target).closest(".btn")[0];e(n).toggleClass("focus",/^focus(in)?$/.test(t.type))})),e(window).on("load.bs.button.data-api",(function(){for(var t=[].slice.call(document.querySelectorAll('[data-toggle="buttons"] .btn')),e=0,n=t.length;e0,this._pointerEvent=Boolean(window.PointerEvent||window.MSPointerEvent),this._addEventListeners()}var n=t.prototype;return n.next=function(){this._isSliding||this._slide("next")},n.nextWhenVisible=function(){!document.hidden&&e(this._element).is(":visible")&&"hidden"!==e(this._element).css("visibility")&&this.next()},n.prev=function(){this._isSliding||this._slide("prev")},n.pause=function(t){t||(this._isPaused=!0),this._element.querySelector(".carousel-item-next, .carousel-item-prev")&&(c.triggerTransitionEnd(this._element),this.cycle(!0)),clearInterval(this._interval),this._interval=null},n.cycle=function(t){t||(this._isPaused=!1),this._interval&&(clearInterval(this._interval),this._interval=null),this._config.interval&&!this._isPaused&&(this._interval=setInterval((document.visibilityState?this.nextWhenVisible:this.next).bind(this),this._config.interval))},n.to=function(t){var n=this;this._activeElement=this._element.querySelector(".active.carousel-item");var i=this._getItemIndex(this._activeElement);if(!(t>this._items.length-1||t<0))if(this._isSliding)e(this._element).one("slid.bs.carousel",(function(){return n.to(t)}));else{if(i===t)return this.pause(),void this.cycle();var o=t>i?"next":"prev";this._slide(o,this._items[t])}},n.dispose=function(){e(this._element).off(p),e.removeData(this._element,"bs.carousel"),this._items=null,this._config=null,this._element=null,this._interval=null,this._isPaused=null,this._isSliding=null,this._activeElement=null,this._indicatorsElement=null},n._getConfig=function(t){return t=a(a({},v),t),c.typeCheckConfig(m,t,b),t},n._handleSwipe=function(){var t=Math.abs(this.touchDeltaX);if(!(t<=40)){var e=t/this.touchDeltaX;this.touchDeltaX=0,e>0&&this.prev(),e<0&&this.next()}},n._addEventListeners=function(){var t=this;this._config.keyboard&&e(this._element).on("keydown.bs.carousel",(function(e){return t._keydown(e)})),"hover"===this._config.pause&&e(this._element).on("mouseenter.bs.carousel",(function(e){return t.pause(e)})).on("mouseleave.bs.carousel",(function(e){return t.cycle(e)})),this._config.touch&&this._addTouchEventListeners()},n._addTouchEventListeners=function(){var t=this;if(this._touchSupported){var n=function(e){t._pointerEvent&&y[e.originalEvent.pointerType.toUpperCase()]?t.touchStartX=e.originalEvent.clientX:t._pointerEvent||(t.touchStartX=e.originalEvent.touches[0].clientX)},i=function(e){t._pointerEvent&&y[e.originalEvent.pointerType.toUpperCase()]&&(t.touchDeltaX=e.originalEvent.clientX-t.touchStartX),t._handleSwipe(),"hover"===t._config.pause&&(t.pause(),t.touchTimeout&&clearTimeout(t.touchTimeout),t.touchTimeout=setTimeout((function(e){return t.cycle(e)}),500+t._config.interval))};e(this._element.querySelectorAll(".carousel-item img")).on("dragstart.bs.carousel",(function(t){return t.preventDefault()})),this._pointerEvent?(e(this._element).on("pointerdown.bs.carousel",(function(t){return n(t)})),e(this._element).on("pointerup.bs.carousel",(function(t){return i(t)})),this._element.classList.add("pointer-event")):(e(this._element).on("touchstart.bs.carousel",(function(t){return n(t)})),e(this._element).on("touchmove.bs.carousel",(function(e){return function(e){e.originalEvent.touches&&e.originalEvent.touches.length>1?t.touchDeltaX=0:t.touchDeltaX=e.originalEvent.touches[0].clientX-t.touchStartX}(e)})),e(this._element).on("touchend.bs.carousel",(function(t){return i(t)})))}},n._keydown=function(t){if(!/input|textarea/i.test(t.target.tagName))switch(t.which){case 37:t.preventDefault(),this.prev();break;case 39:t.preventDefault(),this.next()}},n._getItemIndex=function(t){return this._items=t&&t.parentNode?[].slice.call(t.parentNode.querySelectorAll(".carousel-item")):[],this._items.indexOf(t)},n._getItemByDirection=function(t,e){var n="next"===t,i="prev"===t,o=this._getItemIndex(e),s=this._items.length-1;if((i&&0===o||n&&o===s)&&!this._config.wrap)return e;var r=(o+("prev"===t?-1:1))%this._items.length;return-1===r?this._items[this._items.length-1]:this._items[r]},n._triggerSlideEvent=function(t,n){var i=this._getItemIndex(t),o=this._getItemIndex(this._element.querySelector(".active.carousel-item")),s=e.Event("slide.bs.carousel",{relatedTarget:t,direction:n,from:o,to:i});return e(this._element).trigger(s),s},n._setActiveIndicatorElement=function(t){if(this._indicatorsElement){var n=[].slice.call(this._indicatorsElement.querySelectorAll(".active"));e(n).removeClass("active");var i=this._indicatorsElement.children[this._getItemIndex(t)];i&&e(i).addClass("active")}},n._slide=function(t,n){var i,o,s,r=this,a=this._element.querySelector(".active.carousel-item"),l=this._getItemIndex(a),h=n||a&&this._getItemByDirection(t,a),u=this._getItemIndex(h),d=Boolean(this._interval);if("next"===t?(i="carousel-item-left",o="carousel-item-next",s="left"):(i="carousel-item-right",o="carousel-item-prev",s="right"),h&&e(h).hasClass("active"))this._isSliding=!1;else if(!this._triggerSlideEvent(h,s).isDefaultPrevented()&&a&&h){this._isSliding=!0,d&&this.pause(),this._setActiveIndicatorElement(h);var f=e.Event("slid.bs.carousel",{relatedTarget:h,direction:s,from:l,to:u});if(e(this._element).hasClass("slide")){e(h).addClass(o),c.reflow(h),e(a).addClass(i),e(h).addClass(i);var g=parseInt(h.getAttribute("data-interval"),10);g?(this._config.defaultInterval=this._config.defaultInterval||this._config.interval,this._config.interval=g):this._config.interval=this._config.defaultInterval||this._config.interval;var m=c.getTransitionDurationFromElement(a);e(a).one(c.TRANSITION_END,(function(){e(h).removeClass(i+" "+o).addClass("active"),e(a).removeClass("active "+o+" "+i),r._isSliding=!1,setTimeout((function(){return e(r._element).trigger(f)}),0)})).emulateTransitionEnd(m)}else e(a).removeClass("active"),e(h).addClass("active"),this._isSliding=!1,e(this._element).trigger(f);d&&this.cycle()}},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.carousel"),o=a(a({},v),e(this).data());"object"==typeof n&&(o=a(a({},o),n));var s="string"==typeof n?n:o.slide;if(i||(i=new t(this,o),e(this).data("bs.carousel",i)),"number"==typeof n)i.to(n);else if("string"==typeof s){if("undefined"==typeof i[s])throw new TypeError('No method named "'+s+'"');i[s]()}else o.interval&&o.ride&&(i.pause(),i.cycle())}))},t._dataApiClickHandler=function(n){var i=c.getSelectorFromElement(this);if(i){var o=e(i)[0];if(o&&e(o).hasClass("carousel")){var s=a(a({},e(o).data()),e(this).data()),r=this.getAttribute("data-slide-to");r&&(s.interval=!1),t._jQueryInterface.call(e(o),s),r&&e(o).data("bs.carousel").to(r),n.preventDefault()}}},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return v}}]),t}();e(document).on("click.bs.carousel.data-api","[data-slide], [data-slide-to]",E._dataApiClickHandler),e(window).on("load.bs.carousel.data-api",(function(){for(var t=[].slice.call(document.querySelectorAll('[data-ride="carousel"]')),n=0,i=t.length;n0&&(this._selector=r,this._triggerArray.push(s))}this._parent=this._config.parent?this._getParent():null,this._config.parent||this._addAriaAndCollapsedClass(this._element,this._triggerArray),this._config.toggle&&this.toggle()}var n=t.prototype;return n.toggle=function(){e(this._element).hasClass("show")?this.hide():this.show()},n.show=function(){var n,i,o=this;if(!this._isTransitioning&&!e(this._element).hasClass("show")&&(this._parent&&0===(n=[].slice.call(this._parent.querySelectorAll(".show, .collapsing")).filter((function(t){return"string"==typeof o._config.parent?t.getAttribute("data-parent")===o._config.parent:t.classList.contains("collapse")}))).length&&(n=null),!(n&&(i=e(n).not(this._selector).data("bs.collapse"))&&i._isTransitioning))){var s=e.Event("show.bs.collapse");if(e(this._element).trigger(s),!s.isDefaultPrevented()){n&&(t._jQueryInterface.call(e(n).not(this._selector),"hide"),i||e(n).data("bs.collapse",null));var r=this._getDimension();e(this._element).removeClass("collapse").addClass("collapsing"),this._element.style[r]=0,this._triggerArray.length&&e(this._triggerArray).removeClass("collapsed").attr("aria-expanded",!0),this.setTransitioning(!0);var a="scroll"+(r[0].toUpperCase()+r.slice(1)),l=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,(function(){e(o._element).removeClass("collapsing").addClass("collapse show"),o._element.style[r]="",o.setTransitioning(!1),e(o._element).trigger("shown.bs.collapse")})).emulateTransitionEnd(l),this._element.style[r]=this._element[a]+"px"}}},n.hide=function(){var t=this;if(!this._isTransitioning&&e(this._element).hasClass("show")){var n=e.Event("hide.bs.collapse");if(e(this._element).trigger(n),!n.isDefaultPrevented()){var i=this._getDimension();this._element.style[i]=this._element.getBoundingClientRect()[i]+"px",c.reflow(this._element),e(this._element).addClass("collapsing").removeClass("collapse show");var o=this._triggerArray.length;if(o>0)for(var s=0;s0},i._getOffset=function(){var t=this,e={};return"function"==typeof this._config.offset?e.fn=function(e){return e.offsets=a(a({},e.offsets),t._config.offset(e.offsets,t._element)||{}),e}:e.offset=this._config.offset,e},i._getPopperConfig=function(){var t={placement:this._getPlacement(),modifiers:{offset:this._getOffset(),flip:{enabled:this._config.flip},preventOverflow:{boundariesElement:this._config.boundary}}};return"static"===this._config.display&&(t.modifiers.applyStyle={enabled:!1}),a(a({},t),this._config.popperConfig)},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.dropdown");if(i||(i=new t(this,"object"==typeof n?n:null),e(this).data("bs.dropdown",i)),"string"==typeof n){if("undefined"==typeof i[n])throw new TypeError('No method named "'+n+'"');i[n]()}}))},t._clearMenus=function(n){if(!n||3!==n.which&&("keyup"!==n.type||9===n.which))for(var i=[].slice.call(document.querySelectorAll('[data-toggle="dropdown"]')),o=0,s=i.length;o0&&r--,40===n.which&&rdocument.documentElement.clientHeight;!this._isBodyOverflowing&&t&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!t&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},n._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},n._checkScrollbar=function(){var t=document.body.getBoundingClientRect();this._isBodyOverflowing=Math.round(t.left+t.right)
',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip",boundary:"scrollParent",sanitize:!0,sanitizeFn:null,whiteList:F,popperConfig:null},Y={HIDE:"hide.bs.tooltip",HIDDEN:"hidden.bs.tooltip",SHOW:"show.bs.tooltip",SHOWN:"shown.bs.tooltip",INSERTED:"inserted.bs.tooltip",CLICK:"click.bs.tooltip",FOCUSIN:"focusin.bs.tooltip",FOCUSOUT:"focusout.bs.tooltip",MOUSEENTER:"mouseenter.bs.tooltip",MOUSELEAVE:"mouseleave.bs.tooltip"},$=function(){function t(t,e){if("undefined"==typeof n)throw new TypeError("Bootstrap's tooltips require Popper.js (https://popper.js.org/)");this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=t,this.config=this._getConfig(e),this.tip=null,this._setListeners()}var i=t.prototype;return i.enable=function(){this._isEnabled=!0},i.disable=function(){this._isEnabled=!1},i.toggleEnabled=function(){this._isEnabled=!this._isEnabled},i.toggle=function(t){if(this._isEnabled)if(t){var n=this.constructor.DATA_KEY,i=e(t.currentTarget).data(n);i||(i=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(n,i)),i._activeTrigger.click=!i._activeTrigger.click,i._isWithActiveTrigger()?i._enter(null,i):i._leave(null,i)}else{if(e(this.getTipElement()).hasClass("show"))return void this._leave(null,this);this._enter(null,this)}},i.dispose=function(){clearTimeout(this._timeout),e.removeData(this.element,this.constructor.DATA_KEY),e(this.element).off(this.constructor.EVENT_KEY),e(this.element).closest(".modal").off("hide.bs.modal",this._hideModalHandler),this.tip&&e(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,this._activeTrigger=null,this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},i.show=function(){var t=this;if("none"===e(this.element).css("display"))throw new Error("Please use show on visible elements");var i=e.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){e(this.element).trigger(i);var o=c.findShadowRoot(this.element),s=e.contains(null!==o?o:this.element.ownerDocument.documentElement,this.element);if(i.isDefaultPrevented()||!s)return;var r=this.getTipElement(),a=c.getUID(this.constructor.NAME);r.setAttribute("id",a),this.element.setAttribute("aria-describedby",a),this.setContent(),this.config.animation&&e(r).addClass("fade");var l="function"==typeof this.config.placement?this.config.placement.call(this,r,this.element):this.config.placement,h=this._getAttachment(l);this.addAttachmentClass(h);var u=this._getContainer();e(r).data(this.constructor.DATA_KEY,this),e.contains(this.element.ownerDocument.documentElement,this.tip)||e(r).appendTo(u),e(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new n(this.element,r,this._getPopperConfig(h)),e(r).addClass("show"),"ontouchstart"in document.documentElement&&e(document.body).children().on("mouseover",null,e.noop);var d=function(){t.config.animation&&t._fixTransition();var n=t._hoverState;t._hoverState=null,e(t.element).trigger(t.constructor.Event.SHOWN),"out"===n&&t._leave(null,t)};if(e(this.tip).hasClass("fade")){var f=c.getTransitionDurationFromElement(this.tip);e(this.tip).one(c.TRANSITION_END,d).emulateTransitionEnd(f)}else d()}},i.hide=function(t){var n=this,i=this.getTipElement(),o=e.Event(this.constructor.Event.HIDE),s=function(){"show"!==n._hoverState&&i.parentNode&&i.parentNode.removeChild(i),n._cleanTipClass(),n.element.removeAttribute("aria-describedby"),e(n.element).trigger(n.constructor.Event.HIDDEN),null!==n._popper&&n._popper.destroy(),t&&t()};if(e(this.element).trigger(o),!o.isDefaultPrevented()){if(e(i).removeClass("show"),"ontouchstart"in document.documentElement&&e(document.body).children().off("mouseover",null,e.noop),this._activeTrigger.click=!1,this._activeTrigger.focus=!1,this._activeTrigger.hover=!1,e(this.tip).hasClass("fade")){var r=c.getTransitionDurationFromElement(i);e(i).one(c.TRANSITION_END,s).emulateTransitionEnd(r)}else s();this._hoverState=""}},i.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},i.isWithContent=function(){return Boolean(this.getTitle())},i.addAttachmentClass=function(t){e(this.getTipElement()).addClass("bs-tooltip-"+t)},i.getTipElement=function(){return this.tip=this.tip||e(this.config.template)[0],this.tip},i.setContent=function(){var t=this.getTipElement();this.setElementContent(e(t.querySelectorAll(".tooltip-inner")),this.getTitle()),e(t).removeClass("fade show")},i.setElementContent=function(t,n){"object"!=typeof n||!n.nodeType&&!n.jquery?this.config.html?(this.config.sanitize&&(n=H(n,this.config.whiteList,this.config.sanitizeFn)),t.html(n)):t.text(n):this.config.html?e(n).parent().is(t)||t.empty().append(n):t.text(e(n).text())},i.getTitle=function(){var t=this.element.getAttribute("data-original-title");return t||(t="function"==typeof this.config.title?this.config.title.call(this.element):this.config.title),t},i._getPopperConfig=function(t){var e=this;return a(a({},{placement:t,modifiers:{offset:this._getOffset(),flip:{behavior:this.config.fallbackPlacement},arrow:{element:".arrow"},preventOverflow:{boundariesElement:this.config.boundary}},onCreate:function(t){t.originalPlacement!==t.placement&&e._handlePopperPlacementChange(t)},onUpdate:function(t){return e._handlePopperPlacementChange(t)}}),this.config.popperConfig)},i._getOffset=function(){var t=this,e={};return"function"==typeof this.config.offset?e.fn=function(e){return e.offsets=a(a({},e.offsets),t.config.offset(e.offsets,t.element)||{}),e}:e.offset=this.config.offset,e},i._getContainer=function(){return!1===this.config.container?document.body:c.isElement(this.config.container)?e(this.config.container):e(document).find(this.config.container)},i._getAttachment=function(t){return K[t.toUpperCase()]},i._setListeners=function(){var t=this;this.config.trigger.split(" ").forEach((function(n){if("click"===n)e(t.element).on(t.constructor.Event.CLICK,t.config.selector,(function(e){return t.toggle(e)}));else if("manual"!==n){var i="hover"===n?t.constructor.Event.MOUSEENTER:t.constructor.Event.FOCUSIN,o="hover"===n?t.constructor.Event.MOUSELEAVE:t.constructor.Event.FOCUSOUT;e(t.element).on(i,t.config.selector,(function(e){return t._enter(e)})).on(o,t.config.selector,(function(e){return t._leave(e)}))}})),this._hideModalHandler=function(){t.element&&t.hide()},e(this.element).closest(".modal").on("hide.bs.modal",this._hideModalHandler),this.config.selector?this.config=a(a({},this.config),{},{trigger:"manual",selector:""}):this._fixTitle()},i._fixTitle=function(){var t=typeof this.element.getAttribute("data-original-title");(this.element.getAttribute("title")||"string"!==t)&&(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},i._enter=function(t,n){var i=this.constructor.DATA_KEY;(n=n||e(t.currentTarget).data(i))||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(i,n)),t&&(n._activeTrigger["focusin"===t.type?"focus":"hover"]=!0),e(n.getTipElement()).hasClass("show")||"show"===n._hoverState?n._hoverState="show":(clearTimeout(n._timeout),n._hoverState="show",n.config.delay&&n.config.delay.show?n._timeout=setTimeout((function(){"show"===n._hoverState&&n.show()}),n.config.delay.show):n.show())},i._leave=function(t,n){var i=this.constructor.DATA_KEY;(n=n||e(t.currentTarget).data(i))||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),e(t.currentTarget).data(i,n)),t&&(n._activeTrigger["focusout"===t.type?"focus":"hover"]=!1),n._isWithActiveTrigger()||(clearTimeout(n._timeout),n._hoverState="out",n.config.delay&&n.config.delay.hide?n._timeout=setTimeout((function(){"out"===n._hoverState&&n.hide()}),n.config.delay.hide):n.hide())},i._isWithActiveTrigger=function(){for(var t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1},i._getConfig=function(t){var n=e(this.element).data();return Object.keys(n).forEach((function(t){-1!==V.indexOf(t)&&delete n[t]})),"number"==typeof(t=a(a(a({},this.constructor.Default),n),"object"==typeof t&&t?t:{})).delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),c.typeCheckConfig(U,t,this.constructor.DefaultType),t.sanitize&&(t.template=H(t.template,t.whiteList,t.sanitizeFn)),t},i._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]!==this.config[e]&&(t[e]=this.config[e]);return t},i._cleanTipClass=function(){var t=e(this.getTipElement()),n=t.attr("class").match(W);null!==n&&n.length&&t.removeClass(n.join(""))},i._handlePopperPlacementChange=function(t){this.tip=t.instance.popper,this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(t.placement))},i._fixTransition=function(){var t=this.getTipElement(),n=this.config.animation;null===t.getAttribute("x-placement")&&(e(t).removeClass("fade"),this.config.animation=!1,this.hide(),this.show(),this.config.animation=n)},t._jQueryInterface=function(n){return this.each((function(){var i=e(this).data("bs.tooltip"),o="object"==typeof n&&n;if((i||!/dispose|hide/.test(n))&&(i||(i=new t(this,o),e(this).data("bs.tooltip",i)),"string"==typeof n)){if("undefined"==typeof i[n])throw new TypeError('No method named "'+n+'"');i[n]()}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return X}},{key:"NAME",get:function(){return U}},{key:"DATA_KEY",get:function(){return"bs.tooltip"}},{key:"Event",get:function(){return Y}},{key:"EVENT_KEY",get:function(){return".bs.tooltip"}},{key:"DefaultType",get:function(){return z}}]),t}();e.fn[U]=$._jQueryInterface,e.fn[U].Constructor=$,e.fn[U].noConflict=function(){return e.fn[U]=M,$._jQueryInterface};var J="popover",G=e.fn[J],Z=new RegExp("(^|\\s)bs-popover\\S+","g"),tt=a(a({},$.Default),{},{placement:"right",trigger:"click",content:"",template:''}),et=a(a({},$.DefaultType),{},{content:"(string|element|function)"}),nt={HIDE:"hide.bs.popover",HIDDEN:"hidden.bs.popover",SHOW:"show.bs.popover",SHOWN:"shown.bs.popover",INSERTED:"inserted.bs.popover",CLICK:"click.bs.popover",FOCUSIN:"focusin.bs.popover",FOCUSOUT:"focusout.bs.popover",MOUSEENTER:"mouseenter.bs.popover",MOUSELEAVE:"mouseleave.bs.popover"},it=function(t){var n,i;function s(){return t.apply(this,arguments)||this}i=t,(n=s).prototype=Object.create(i.prototype),n.prototype.constructor=n,n.__proto__=i;var r=s.prototype;return r.isWithContent=function(){return this.getTitle()||this._getContent()},r.addAttachmentClass=function(t){e(this.getTipElement()).addClass("bs-popover-"+t)},r.getTipElement=function(){return this.tip=this.tip||e(this.config.template)[0],this.tip},r.setContent=function(){var t=e(this.getTipElement());this.setElementContent(t.find(".popover-header"),this.getTitle());var n=this._getContent();"function"==typeof n&&(n=n.call(this.element)),this.setElementContent(t.find(".popover-body"),n),t.removeClass("fade show")},r._getContent=function(){return this.element.getAttribute("data-content")||this.config.content},r._cleanTipClass=function(){var t=e(this.getTipElement()),n=t.attr("class").match(Z);null!==n&&n.length>0&&t.removeClass(n.join(""))},s._jQueryInterface=function(t){return this.each((function(){var n=e(this).data("bs.popover"),i="object"==typeof t?t:null;if((n||!/dispose|hide/.test(t))&&(n||(n=new s(this,i),e(this).data("bs.popover",n)),"string"==typeof t)){if("undefined"==typeof n[t])throw new TypeError('No method named "'+t+'"');n[t]()}}))},o(s,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"Default",get:function(){return tt}},{key:"NAME",get:function(){return J}},{key:"DATA_KEY",get:function(){return"bs.popover"}},{key:"Event",get:function(){return nt}},{key:"EVENT_KEY",get:function(){return".bs.popover"}},{key:"DefaultType",get:function(){return et}}]),s}($);e.fn[J]=it._jQueryInterface,e.fn[J].Constructor=it,e.fn[J].noConflict=function(){return e.fn[J]=G,it._jQueryInterface};var ot="scrollspy",st=e.fn[ot],rt={offset:10,method:"auto",target:""},at={offset:"number",method:"string",target:"(string|element)"},lt=function(){function t(t,n){var i=this;this._element=t,this._scrollElement="BODY"===t.tagName?window:t,this._config=this._getConfig(n),this._selector=this._config.target+" .nav-link,"+this._config.target+" .list-group-item,"+this._config.target+" .dropdown-item",this._offsets=[],this._targets=[],this._activeTarget=null,this._scrollHeight=0,e(this._scrollElement).on("scroll.bs.scrollspy",(function(t){return i._process(t)})),this.refresh(),this._process()}var n=t.prototype;return n.refresh=function(){var t=this,n=this._scrollElement===this._scrollElement.window?"offset":"position",i="auto"===this._config.method?n:this._config.method,o="position"===i?this._getScrollTop():0;this._offsets=[],this._targets=[],this._scrollHeight=this._getScrollHeight(),[].slice.call(document.querySelectorAll(this._selector)).map((function(t){var n,s=c.getSelectorFromElement(t);if(s&&(n=document.querySelector(s)),n){var r=n.getBoundingClientRect();if(r.width||r.height)return[e(n)[i]().top+o,s]}return null})).filter((function(t){return t})).sort((function(t,e){return t[0]-e[0]})).forEach((function(e){t._offsets.push(e[0]),t._targets.push(e[1])}))},n.dispose=function(){e.removeData(this._element,"bs.scrollspy"),e(this._scrollElement).off(".bs.scrollspy"),this._element=null,this._scrollElement=null,this._config=null,this._selector=null,this._offsets=null,this._targets=null,this._activeTarget=null,this._scrollHeight=null},n._getConfig=function(t){if("string"!=typeof(t=a(a({},rt),"object"==typeof t&&t?t:{})).target&&c.isElement(t.target)){var n=e(t.target).attr("id");n||(n=c.getUID(ot),e(t.target).attr("id",n)),t.target="#"+n}return c.typeCheckConfig(ot,t,at),t},n._getScrollTop=function(){return this._scrollElement===window?this._scrollElement.pageYOffset:this._scrollElement.scrollTop},n._getScrollHeight=function(){return this._scrollElement.scrollHeight||Math.max(document.body.scrollHeight,document.documentElement.scrollHeight)},n._getOffsetHeight=function(){return this._scrollElement===window?window.innerHeight:this._scrollElement.getBoundingClientRect().height},n._process=function(){var t=this._getScrollTop()+this._config.offset,e=this._getScrollHeight(),n=this._config.offset+e-this._getOffsetHeight();if(this._scrollHeight!==e&&this.refresh(),t>=n){var i=this._targets[this._targets.length-1];this._activeTarget!==i&&this._activate(i)}else{if(this._activeTarget&&t0)return this._activeTarget=null,void this._clear();for(var o=this._offsets.length;o--;){this._activeTarget!==this._targets[o]&&t>=this._offsets[o]&&("undefined"==typeof this._offsets[o+1]||t li > .active":".active";i=(i=e.makeArray(e(o).find(r)))[i.length-1]}var a=e.Event("hide.bs.tab",{relatedTarget:this._element}),l=e.Event("show.bs.tab",{relatedTarget:i});if(i&&e(i).trigger(a),e(this._element).trigger(l),!l.isDefaultPrevented()&&!a.isDefaultPrevented()){s&&(n=document.querySelector(s)),this._activate(this._element,o);var h=function(){var n=e.Event("hidden.bs.tab",{relatedTarget:t._element}),o=e.Event("shown.bs.tab",{relatedTarget:i});e(i).trigger(n),e(t._element).trigger(o)};n?this._activate(n,n.parentNode,h):h()}}},n.dispose=function(){e.removeData(this._element,"bs.tab"),this._element=null},n._activate=function(t,n,i){var o=this,s=(!n||"UL"!==n.nodeName&&"OL"!==n.nodeName?e(n).children(".active"):e(n).find("> li > .active"))[0],r=i&&s&&e(s).hasClass("fade"),a=function(){return o._transitionComplete(t,s,i)};if(s&&r){var l=c.getTransitionDurationFromElement(s);e(s).removeClass("show").one(c.TRANSITION_END,a).emulateTransitionEnd(l)}else a()},n._transitionComplete=function(t,n,i){if(n){e(n).removeClass("active");var o=e(n.parentNode).find("> .dropdown-menu .active")[0];o&&e(o).removeClass("active"),"tab"===n.getAttribute("role")&&n.setAttribute("aria-selected",!1)}if(e(t).addClass("active"),"tab"===t.getAttribute("role")&&t.setAttribute("aria-selected",!0),c.reflow(t),t.classList.contains("fade")&&t.classList.add("show"),t.parentNode&&e(t.parentNode).hasClass("dropdown-menu")){var s=e(t).closest(".dropdown")[0];if(s){var r=[].slice.call(s.querySelectorAll(".dropdown-toggle"));e(r).addClass("active")}t.setAttribute("aria-expanded",!0)}i&&i()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.tab");if(o||(o=new t(this),i.data("bs.tab",o)),"string"==typeof n){if("undefined"==typeof o[n])throw new TypeError('No method named "'+n+'"');o[n]()}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}}]),t}();e(document).on("click.bs.tab.data-api",'[data-toggle="tab"], [data-toggle="pill"], [data-toggle="list"]',(function(t){t.preventDefault(),ht._jQueryInterface.call(e(this),"show")})),e.fn.tab=ht._jQueryInterface,e.fn.tab.Constructor=ht,e.fn.tab.noConflict=function(){return e.fn.tab=ct,ht._jQueryInterface};var ut=e.fn.toast,dt={animation:"boolean",autohide:"boolean",delay:"number"},ft={animation:!0,autohide:!0,delay:500},gt=function(){function t(t,e){this._element=t,this._config=this._getConfig(e),this._timeout=null,this._setListeners()}var n=t.prototype;return n.show=function(){var t=this,n=e.Event("show.bs.toast");if(e(this._element).trigger(n),!n.isDefaultPrevented()){this._config.animation&&this._element.classList.add("fade");var i=function(){t._element.classList.remove("showing"),t._element.classList.add("show"),e(t._element).trigger("shown.bs.toast"),t._config.autohide&&(t._timeout=setTimeout((function(){t.hide()}),t._config.delay))};if(this._element.classList.remove("hide"),c.reflow(this._element),this._element.classList.add("showing"),this._config.animation){var o=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,i).emulateTransitionEnd(o)}else i()}},n.hide=function(){if(this._element.classList.contains("show")){var t=e.Event("hide.bs.toast");e(this._element).trigger(t),t.isDefaultPrevented()||this._close()}},n.dispose=function(){clearTimeout(this._timeout),this._timeout=null,this._element.classList.contains("show")&&this._element.classList.remove("show"),e(this._element).off("click.dismiss.bs.toast"),e.removeData(this._element,"bs.toast"),this._element=null,this._config=null},n._getConfig=function(t){return t=a(a(a({},ft),e(this._element).data()),"object"==typeof t&&t?t:{}),c.typeCheckConfig("toast",t,this.constructor.DefaultType),t},n._setListeners=function(){var t=this;e(this._element).on("click.dismiss.bs.toast",'[data-dismiss="toast"]',(function(){return t.hide()}))},n._close=function(){var t=this,n=function(){t._element.classList.add("hide"),e(t._element).trigger("hidden.bs.toast")};if(this._element.classList.remove("show"),this._config.animation){var i=c.getTransitionDurationFromElement(this._element);e(this._element).one(c.TRANSITION_END,n).emulateTransitionEnd(i)}else n()},t._jQueryInterface=function(n){return this.each((function(){var i=e(this),o=i.data("bs.toast");if(o||(o=new t(this,"object"==typeof n&&n),i.data("bs.toast",o)),"string"==typeof n){if("undefined"==typeof o[n])throw new TypeError('No method named "'+n+'"');o[n](this)}}))},o(t,null,[{key:"VERSION",get:function(){return"4.5.0"}},{key:"DefaultType",get:function(){return dt}},{key:"Default",get:function(){return ft}}]),t}();e.fn.toast=gt._jQueryInterface,e.fn.toast.Constructor=gt,e.fn.toast.noConflict=function(){return e.fn.toast=ut,gt._jQueryInterface},t.Alert=d,t.Button=g,t.Carousel=E,t.Collapse=D,t.Dropdown=j,t.Modal=R,t.Popover=it,t.Scrollspy=lt,t.Tab=ht,t.Toast=gt,t.Tooltip=$,t.Util=c,Object.defineProperty(t,"__esModule",{value:!0})})); -//# sourceMappingURL=bootstrap.min.js.map \ No newline at end of file diff --git a/httpserver/static-content/jquery.min.js b/httpserver/static-content/jquery.min.js deleted file mode 100644 index 36b4e1a1..00000000 --- a/httpserver/static-content/jquery.min.js +++ /dev/null @@ -1,2 +0,0 @@ -/*! jQuery v3.5.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-deprecated/ajax-event-alias,-effects,-effects/Tween,-effects/animatedSelector | (c) JS Foundation and other contributors | jquery.org/license */ -!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(g,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,v=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,y=n.hasOwnProperty,a=y.toString,l=a.call(Object),m={},b=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},w=g.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function C(e,t,n){var r,i,o=(n=n||w).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function T(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1 -ajax,-ajax/jsonp,-ajax/load,-ajax/script,-ajax/var/location,-ajax/var/nonce,-ajax/var/rquery,-ajax/xhr,-manipulation/_evalUrl,-deprecated/ajax-event-alias,-effects,-effects/Tween,-effects/animatedSelector",E=function(e,t){return new E.fn.init(e,t)};function d(e){var t=!!e&&"length"in e&&e.length,n=T(e);return!b(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+R+")"+R+"*"),U=new RegExp(R+"|>"),V=new RegExp(W),X=new RegExp("^"+B+"$"),Q={ID:new RegExp("^#("+B+")"),CLASS:new RegExp("^\\.("+B+")"),TAG:new RegExp("^("+B+"|[*])"),ATTR:new RegExp("^"+M),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+R+"*(even|odd|(([+-]|)(\\d*)n|)"+R+"*(?:([+-]|)"+R+"*(\\d+)|))"+R+"*\\)|)","i"),bool:new RegExp("^(?:"+I+")$","i"),needsContext:new RegExp("^"+R+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+R+"*((?:-\\d)?\\d*)"+R+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,G=/^(?:input|select|textarea|button)$/i,K=/^h\d$/i,J=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+R+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){C()},ae=xe(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{O.apply(t=P.call(d.childNodes),d.childNodes),t[d.childNodes.length].nodeType}catch(e){O={apply:t.length?function(e,t){q.apply(e,P.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,d=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==d&&9!==d&&11!==d)return n;if(!r&&(C(e),e=e||T,E)){if(11!==d&&(u=Z.exec(t)))if(i=u[1]){if(9===d){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return O.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&p.getElementsByClassName&&e.getElementsByClassName)return O.apply(n,e.getElementsByClassName(i)),n}if(p.qsa&&!k[t+" "]&&(!v||!v.test(t))&&(1!==d||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===d&&(U.test(t)||_.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&p.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=A)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+be(l[o]);c=l.join(",")}try{return O.apply(n,f.querySelectorAll(c)),n}catch(e){k(t,!0)}finally{s===A&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>x.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[A]=!0,e}function ce(e){var t=T.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)x.attrHandle[n[r]]=t}function de(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function pe(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in p=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},C=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:d;return r!=T&&9===r.nodeType&&r.documentElement&&(a=(T=r).documentElement,E=!i(T),d!=T&&(n=T.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),p.scope=ce(function(e){return a.appendChild(e).appendChild(T.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),p.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),p.getElementsByTagName=ce(function(e){return e.appendChild(T.createComment("")),!e.getElementsByTagName("*").length}),p.getElementsByClassName=J.test(T.getElementsByClassName),p.getById=ce(function(e){return a.appendChild(e).id=A,!T.getElementsByName||!T.getElementsByName(A).length}),p.getById?(x.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},x.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(x.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},x.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),x.find.TAG=p.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):p.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},x.find.CLASS=p.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(p.qsa=J.test(T.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+R+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+R+"*(?:value|"+I+")"),e.querySelectorAll("[id~="+A+"-]").length||v.push("~="),(t=T.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+R+"*name"+R+"*="+R+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+A+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=T.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+R+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(p.matchesSelector=J.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){p.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",W)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=J.test(a.compareDocumentPosition),y=t||J.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!p.sortDetached&&t.compareDocumentPosition(e)===n?e==T||e.ownerDocument==d&&y(d,e)?-1:t==T||t.ownerDocument==d&&y(d,t)?1:u?H(u,e)-H(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==T?-1:t==T?1:i?-1:o?1:u?H(u,e)-H(u,t):0;if(i===o)return de(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?de(a[r],s[r]):a[r]==d?-1:s[r]==d?1:0}),T},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(C(e),p.matchesSelector&&E&&!k[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||p.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){k(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return Q.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&V.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+R+")"+e+"("+R+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return b(n)?E.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?E.grep(e,function(e){return e===n!==r}):"string"!=typeof n?E.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(E.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||L,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:j.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof E?t[0]:t,E.merge(this,E.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:w,!0)),k.test(r[1])&&E.isPlainObject(t))for(r in t)b(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=w.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):b(e)?void 0!==n.ready?n.ready(e):e(E):E.makeArray(e,this)}).prototype=E.fn,L=E(w);var q=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}E.fn.extend({has:function(e){var t=E(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,pe=/^$|^module$|\/(?:java|ecma)script/i;le=w.createDocumentFragment().appendChild(w.createElement("div")),(ce=w.createElement("input")).setAttribute("type","radio"),ce.setAttribute("checked","checked"),ce.setAttribute("name","t"),le.appendChild(ce),m.checkClone=le.cloneNode(!0).cloneNode(!0).lastChild.checked,le.innerHTML="",m.noCloneChecked=!!le.cloneNode(!0).lastChild.defaultValue,le.innerHTML="",m.option=!!le.lastChild;var he={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ge(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&S(e,t)?E.merge([e],n):n}function ve(e,t){for(var n=0,r=e.length;n",""]);var ye=/<|&#?\w+;/;function me(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),d=[],p=0,h=e.length;p\s*$/g;function Le(e,t){return S(e,"table")&&S(11!==t.nodeType?t:t.firstChild,"tr")&&E(e).children("tbody")[0]||e}function je(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n
",2===ft.childNodes.length),E.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(m.createHTMLDocument?((r=(t=w.implementation.createHTMLDocument("")).createElement("base")).href=w.location.href,t.head.appendChild(r)):t=w),o=!n&&[],(i=k.exec(e))?[t.createElement(i[1])]:(i=me([e],t,o),o&&o.length&&E(o).remove(),E.merge([],i.childNodes)));var r,i,o},E.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=E.css(e,"position"),c=E(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=E.css(e,"top"),u=E.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),b(t)&&(t=t.call(e,n,E.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},E.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){E.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===E.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===E.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=E(e).offset()).top+=E.css(e,"borderTopWidth",!0),i.left+=E.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-E.css(r,"marginTop",!0),left:t.left-i.left-E.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===E.css(e,"position"))e=e.offsetParent;return e||re})}}),E.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;E.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),E.each(["top","left"],function(e,n){E.cssHooks[n]=Fe(m.pixelPosition,function(e,t){if(t)return t=We(e,n),Ie.test(t)?E(e).position()[n]+"px":t})}),E.each({Height:"height",Width:"width"},function(a,s){E.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){E.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?E.css(e,t,i):E.style(e,t,n,i)},s,n?e:void 0,n)}})}),E.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),E.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){E.fn[n]=function(e,t){return 0': '>', - '"': '"', - '\'': ''', - '/': '/' - }; - - var defaultSettings = { - indent: 2 - }; - - var id = 0; - var instances = 0; - var current_collapse_level = null; - var ids_to_collapse = []; - - this.create = function(data, settings, collapse_depth) { - current_collapse_level = typeof collapse_depth !== 'undefined' ? collapse_depth : -1; - instances += 1; - return _span(_jsVal(data, 0, false), {class: 'jstValue'}); - }; - - this.collapse = function() { - var arrayLength = ids_to_collapse.length; - for (var i = 0; i < arrayLength; i++) { - JSONTree.toggle(ids_to_collapse[i]); - } - }; - - var _escape = function(text) { - return text.replace(/[&<>'"]/g, function(c) { - return escapeMap[c]; - }); - }; - - var _id = function() { - return instances + '_' + id++; - }; - - var _lastId = function() { - return instances + '_' + (id - 1); - }; - - var _jsVal = function(value, depth, indent) { - if (value !== null) { - var type = typeof value; - switch (type) { - case 'boolean': - return _jsBool(value, indent ? depth : 0); - case 'number': - return _jsNum(value, indent ? depth : 0); - case 'string': - return _jsStr(value, indent ? depth : 0); - default: - if (value instanceof Array) { - return _jsArr(value, depth, indent); - } else { - return _jsObj(value, depth, indent); - } - } - } else { - return _jsNull(indent ? depth : 0); - } - }; - - var _jsObj = function(object, depth, indent) { - var id = _id(); - _decrementCollapseLevel("_jsObj"); - var content = Object.keys(object).map(function(property) { - return _property(property, object[property], depth + 1, true); - }).join(_comma()); - var body = [ - _openBracket('{', indent ? depth : 0, id), - _span(content, {id: id}), - _closeBracket('}', depth) - ].join('\n'); - _incrementCollapseLevel("_jsObj"); - return _span(body, {}); - }; - - var _jsArr = function(array, depth, indent) { - var id = _id(); - _decrementCollapseLevel("_jsArr"); - var body = array.map(function(element) { - return _jsVal(element, depth + 1, true); - }).join(_comma()); - var arr = [ - _openBracket('[', indent ? depth : 0, id), - _span(body, {id: id}), - _closeBracket(']', depth) - ].join('\n'); - _incrementCollapseLevel("_jsArr"); - return arr; - }; - - var _jsStr = function(value, depth) { - var jsonString = _escape(JSON.stringify(value)); - return _span(_indent(jsonString, depth), {class: 'jstStr'}); - }; - - var _jsNum = function(value, depth) { - return _span(_indent(value, depth), {class: 'jstNum'}); - }; - - var _jsBool = function(value, depth) { - return _span(_indent(value, depth), {class: 'jstBool'}); - }; - - var _jsNull = function(depth) { - return _span(_indent('null', depth), {class: 'jstNull'}); - }; - - var _property = function(name, value, depth) { - var property = _indent(_escape(JSON.stringify(name)) + ': ', depth); - var propertyValue = _span(_jsVal(value, depth, false), {}); - return _span(property + propertyValue, {class: 'jstProperty'}); - }; - - var _comma = function() { - return _span(',\n', {class: 'jstComma'}); - }; - - var _span = function(value, attrs) { - return _tag('span', attrs, value); - }; - - var _tag = function(tag, attrs, content) { - return '<' + tag + Object.keys(attrs).map(function(attr) { - return ' ' + attr + '="' + attrs[attr] + '"'; - }).join('') + '>' + - content + - ''; - }; - - var _openBracket = function(symbol, depth, id) { - return ( - _span(_indent(symbol, depth), {class: 'jstBracket'}) + - _span('', {class: 'jstFold', onclick: 'JSONTree.toggle(\'' + id + '\')'}) - ); - }; - - this.toggle = function(id) { - var element = document.getElementById(id); - var parent = element.parentNode; - var toggleButton = element.previousElementSibling; - if (element.className === '') { - element.className = 'jstHiddenBlock'; - parent.className = 'jstFolded'; - toggleButton.className = 'jstExpand'; - } else { - element.className = ''; - parent.className = ''; - toggleButton.className = 'jstFold'; - } - }; - - var _closeBracket = function(symbol, depth) { - return _span(_indent(symbol, depth), {}); - }; - - var _indent = function(value, depth) { - return Array((depth * 2) + 1).join(' ') + value; - }; - - var _decrementCollapseLevel = function(caller) { - if (current_collapse_level <= 0) { - ids_to_collapse.push(_lastId()); - } else { - } - current_collapse_level--; - }; - - var _incrementCollapseLevel = function(caller) { - current_collapse_level++; - }; - - return this; -})(); diff --git a/httpserver/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css b/httpserver/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css deleted file mode 100755 index 4664f2e8..00000000 --- a/httpserver/static-content/open-iconic/font/css/open-iconic-bootstrap.min.css +++ /dev/null @@ -1 +0,0 @@ -@font-face{font-family:Icons;src:url(../fonts/open-iconic.eot);src:url(../fonts/open-iconic.eot?#iconic-sm) format('embedded-opentype'),url(../fonts/open-iconic.woff) format('woff'),url(../fonts/open-iconic.ttf) format('truetype'),url(../fonts/open-iconic.otf) format('opentype'),url(../fonts/open-iconic.svg#iconic-sm) format('svg');font-weight:400;font-style:normal}.oi{position:relative;top:1px;display:inline-block;speak:none;font-family:Icons;font-style:normal;font-weight:400;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.oi:empty:before{width:1em;text-align:center;box-sizing:content-box}.oi.oi-align-center:before{text-align:center}.oi.oi-align-left:before{text-align:left}.oi.oi-align-right:before{text-align:right}.oi.oi-flip-horizontal:before{-webkit-transform:scale(-1,1);-ms-transform:scale(-1,1);transform:scale(-1,1)}.oi.oi-flip-vertical:before{-webkit-transform:scale(1,-1);-ms-transform:scale(-1,1);transform:scale(1,-1)}.oi.oi-flip-horizontal-vertical:before{-webkit-transform:scale(-1,-1);-ms-transform:scale(-1,1);transform:scale(-1,-1)}.oi-account-login:before{content:'\e000'}.oi-account-logout:before{content:'\e001'}.oi-action-redo:before{content:'\e002'}.oi-action-undo:before{content:'\e003'}.oi-align-center:before{content:'\e004'}.oi-align-left:before{content:'\e005'}.oi-align-right:before{content:'\e006'}.oi-aperture:before{content:'\e007'}.oi-arrow-bottom:before{content:'\e008'}.oi-arrow-circle-bottom:before{content:'\e009'}.oi-arrow-circle-left:before{content:'\e00a'}.oi-arrow-circle-right:before{content:'\e00b'}.oi-arrow-circle-top:before{content:'\e00c'}.oi-arrow-left:before{content:'\e00d'}.oi-arrow-right:before{content:'\e00e'}.oi-arrow-thick-bottom:before{content:'\e00f'}.oi-arrow-thick-left:before{content:'\e010'}.oi-arrow-thick-right:before{content:'\e011'}.oi-arrow-thick-top:before{content:'\e012'}.oi-arrow-top:before{content:'\e013'}.oi-audio-spectrum:before{content:'\e014'}.oi-audio:before{content:'\e015'}.oi-badge:before{content:'\e016'}.oi-ban:before{content:'\e017'}.oi-bar-chart:before{content:'\e018'}.oi-basket:before{content:'\e019'}.oi-battery-empty:before{content:'\e01a'}.oi-battery-full:before{content:'\e01b'}.oi-beaker:before{content:'\e01c'}.oi-bell:before{content:'\e01d'}.oi-bluetooth:before{content:'\e01e'}.oi-bold:before{content:'\e01f'}.oi-bolt:before{content:'\e020'}.oi-book:before{content:'\e021'}.oi-bookmark:before{content:'\e022'}.oi-box:before{content:'\e023'}.oi-briefcase:before{content:'\e024'}.oi-british-pound:before{content:'\e025'}.oi-browser:before{content:'\e026'}.oi-brush:before{content:'\e027'}.oi-bug:before{content:'\e028'}.oi-bullhorn:before{content:'\e029'}.oi-calculator:before{content:'\e02a'}.oi-calendar:before{content:'\e02b'}.oi-camera-slr:before{content:'\e02c'}.oi-caret-bottom:before{content:'\e02d'}.oi-caret-left:before{content:'\e02e'}.oi-caret-right:before{content:'\e02f'}.oi-caret-top:before{content:'\e030'}.oi-cart:before{content:'\e031'}.oi-chat:before{content:'\e032'}.oi-check:before{content:'\e033'}.oi-chevron-bottom:before{content:'\e034'}.oi-chevron-left:before{content:'\e035'}.oi-chevron-right:before{content:'\e036'}.oi-chevron-top:before{content:'\e037'}.oi-circle-check:before{content:'\e038'}.oi-circle-x:before{content:'\e039'}.oi-clipboard:before{content:'\e03a'}.oi-clock:before{content:'\e03b'}.oi-cloud-download:before{content:'\e03c'}.oi-cloud-upload:before{content:'\e03d'}.oi-cloud:before{content:'\e03e'}.oi-cloudy:before{content:'\e03f'}.oi-code:before{content:'\e040'}.oi-cog:before{content:'\e041'}.oi-collapse-down:before{content:'\e042'}.oi-collapse-left:before{content:'\e043'}.oi-collapse-right:before{content:'\e044'}.oi-collapse-up:before{content:'\e045'}.oi-command:before{content:'\e046'}.oi-comment-square:before{content:'\e047'}.oi-compass:before{content:'\e048'}.oi-contrast:before{content:'\e049'}.oi-copywriting:before{content:'\e04a'}.oi-credit-card:before{content:'\e04b'}.oi-crop:before{content:'\e04c'}.oi-dashboard:before{content:'\e04d'}.oi-data-transfer-download:before{content:'\e04e'}.oi-data-transfer-upload:before{content:'\e04f'}.oi-delete:before{content:'\e050'}.oi-dial:before{content:'\e051'}.oi-document:before{content:'\e052'}.oi-dollar:before{content:'\e053'}.oi-double-quote-sans-left:before{content:'\e054'}.oi-double-quote-sans-right:before{content:'\e055'}.oi-double-quote-serif-left:before{content:'\e056'}.oi-double-quote-serif-right:before{content:'\e057'}.oi-droplet:before{content:'\e058'}.oi-eject:before{content:'\e059'}.oi-elevator:before{content:'\e05a'}.oi-ellipses:before{content:'\e05b'}.oi-envelope-closed:before{content:'\e05c'}.oi-envelope-open:before{content:'\e05d'}.oi-euro:before{content:'\e05e'}.oi-excerpt:before{content:'\e05f'}.oi-expand-down:before{content:'\e060'}.oi-expand-left:before{content:'\e061'}.oi-expand-right:before{content:'\e062'}.oi-expand-up:before{content:'\e063'}.oi-external-link:before{content:'\e064'}.oi-eye:before{content:'\e065'}.oi-eyedropper:before{content:'\e066'}.oi-file:before{content:'\e067'}.oi-fire:before{content:'\e068'}.oi-flag:before{content:'\e069'}.oi-flash:before{content:'\e06a'}.oi-folder:before{content:'\e06b'}.oi-fork:before{content:'\e06c'}.oi-fullscreen-enter:before{content:'\e06d'}.oi-fullscreen-exit:before{content:'\e06e'}.oi-globe:before{content:'\e06f'}.oi-graph:before{content:'\e070'}.oi-grid-four-up:before{content:'\e071'}.oi-grid-three-up:before{content:'\e072'}.oi-grid-two-up:before{content:'\e073'}.oi-hard-drive:before{content:'\e074'}.oi-header:before{content:'\e075'}.oi-headphones:before{content:'\e076'}.oi-heart:before{content:'\e077'}.oi-home:before{content:'\e078'}.oi-image:before{content:'\e079'}.oi-inbox:before{content:'\e07a'}.oi-infinity:before{content:'\e07b'}.oi-info:before{content:'\e07c'}.oi-italic:before{content:'\e07d'}.oi-justify-center:before{content:'\e07e'}.oi-justify-left:before{content:'\e07f'}.oi-justify-right:before{content:'\e080'}.oi-key:before{content:'\e081'}.oi-laptop:before{content:'\e082'}.oi-layers:before{content:'\e083'}.oi-lightbulb:before{content:'\e084'}.oi-link-broken:before{content:'\e085'}.oi-link-intact:before{content:'\e086'}.oi-list-rich:before{content:'\e087'}.oi-list:before{content:'\e088'}.oi-location:before{content:'\e089'}.oi-lock-locked:before{content:'\e08a'}.oi-lock-unlocked:before{content:'\e08b'}.oi-loop-circular:before{content:'\e08c'}.oi-loop-square:before{content:'\e08d'}.oi-loop:before{content:'\e08e'}.oi-magnifying-glass:before{content:'\e08f'}.oi-map-marker:before{content:'\e090'}.oi-map:before{content:'\e091'}.oi-media-pause:before{content:'\e092'}.oi-media-play:before{content:'\e093'}.oi-media-record:before{content:'\e094'}.oi-media-skip-backward:before{content:'\e095'}.oi-media-skip-forward:before{content:'\e096'}.oi-media-step-backward:before{content:'\e097'}.oi-media-step-forward:before{content:'\e098'}.oi-media-stop:before{content:'\e099'}.oi-medical-cross:before{content:'\e09a'}.oi-menu:before{content:'\e09b'}.oi-microphone:before{content:'\e09c'}.oi-minus:before{content:'\e09d'}.oi-monitor:before{content:'\e09e'}.oi-moon:before{content:'\e09f'}.oi-move:before{content:'\e0a0'}.oi-musical-note:before{content:'\e0a1'}.oi-paperclip:before{content:'\e0a2'}.oi-pencil:before{content:'\e0a3'}.oi-people:before{content:'\e0a4'}.oi-person:before{content:'\e0a5'}.oi-phone:before{content:'\e0a6'}.oi-pie-chart:before{content:'\e0a7'}.oi-pin:before{content:'\e0a8'}.oi-play-circle:before{content:'\e0a9'}.oi-plus:before{content:'\e0aa'}.oi-power-standby:before{content:'\e0ab'}.oi-print:before{content:'\e0ac'}.oi-project:before{content:'\e0ad'}.oi-pulse:before{content:'\e0ae'}.oi-puzzle-piece:before{content:'\e0af'}.oi-question-mark:before{content:'\e0b0'}.oi-rain:before{content:'\e0b1'}.oi-random:before{content:'\e0b2'}.oi-reload:before{content:'\e0b3'}.oi-resize-both:before{content:'\e0b4'}.oi-resize-height:before{content:'\e0b5'}.oi-resize-width:before{content:'\e0b6'}.oi-rss-alt:before{content:'\e0b7'}.oi-rss:before{content:'\e0b8'}.oi-script:before{content:'\e0b9'}.oi-share-boxed:before{content:'\e0ba'}.oi-share:before{content:'\e0bb'}.oi-shield:before{content:'\e0bc'}.oi-signal:before{content:'\e0bd'}.oi-signpost:before{content:'\e0be'}.oi-sort-ascending:before{content:'\e0bf'}.oi-sort-descending:before{content:'\e0c0'}.oi-spreadsheet:before{content:'\e0c1'}.oi-star:before{content:'\e0c2'}.oi-sun:before{content:'\e0c3'}.oi-tablet:before{content:'\e0c4'}.oi-tag:before{content:'\e0c5'}.oi-tags:before{content:'\e0c6'}.oi-target:before{content:'\e0c7'}.oi-task:before{content:'\e0c8'}.oi-terminal:before{content:'\e0c9'}.oi-text:before{content:'\e0ca'}.oi-thumb-down:before{content:'\e0cb'}.oi-thumb-up:before{content:'\e0cc'}.oi-timer:before{content:'\e0cd'}.oi-transfer:before{content:'\e0ce'}.oi-trash:before{content:'\e0cf'}.oi-underline:before{content:'\e0d0'}.oi-vertical-align-bottom:before{content:'\e0d1'}.oi-vertical-align-center:before{content:'\e0d2'}.oi-vertical-align-top:before{content:'\e0d3'}.oi-video:before{content:'\e0d4'}.oi-volume-high:before{content:'\e0d5'}.oi-volume-low:before{content:'\e0d6'}.oi-volume-off:before{content:'\e0d7'}.oi-warning:before{content:'\e0d8'}.oi-wifi:before{content:'\e0d9'}.oi-wrench:before{content:'\e0da'}.oi-x:before{content:'\e0db'}.oi-yen:before{content:'\e0dc'}.oi-zoom-in:before{content:'\e0dd'}.oi-zoom-out:before{content:'\e0de'} \ No newline at end of file diff --git a/httpserver/static-content/open-iconic/font/fonts/open-iconic.eot b/httpserver/static-content/open-iconic/font/fonts/open-iconic.eot deleted file mode 100755 index f98177dbf711863eff7c90f84d5d419d02d99ba8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28196 zcmdsfdwg8gedj&r&QluAL-W#Wq&pgEMvsv!&0Cf&+mau`20w)Dj4&8Iu59zN6=RG; z451+<)Ej~^SrrmCp$=hb!Zu?PlZ0v^rFqOYfzqruY1s`+ve{(Uv}w|M+teR4-tX_6 zJJQHDgm(Majx=-5J@?%6_?_SRz0Ykss3^zpP!y(cg+5#{t0IGvlZlxgLVa!|Pwg%0HwaAkJPsR_7CkF z{hz=5BS2$bQO4>H%uMR+@Bes%qU=0}`qqrY1!(P0t>lnf>u?>hCHF7DiD%jIRLs_gA0(b1L}rzgltYVrt?gc2Y5;9UDjQ z%B)P;{Yp$h?WOgkCosju&-Q&Abmg0GDQ~^0YA77V?+nuN;!-_LToFFdx5>D-3RhIC zNim@Y28=&kzxC#&OZZhTUDD)z++voc1{on3eJelI&j0@(PPn1`HTMH@R>gMK0^H#} z-APZ<6H9s`4L|t$XFtpR3vV~DpGXL)8ZghQI8nFC#;Gm~d%|gaTbMPC42!c1B?miM zn$?TN(kwg4=NH!N?1DZwr|Va=QM0@at3QmtSVbGuP_f*EuIqDh*>o`umty&fMPWVN zwOSy=lGa!#OKqKlS=4KL6^YiDEHv;MA!Dj|%KqdbXOLRkVPgo+>xM z`tdLxr03~jdXO4;l(4}>Kca7fS2gy1&DtubqsnG6amCcr?ZNni_*#ur)!una=lO+a z(W#N+^Oy#G-fw#XCIlD!Q7hD3IjwB$Uoy5LHCCk7M6R+q+PRlLC+2F#Og&0KX;fTm z9gRV6t=nO-P_Az=CG4l*~#0dwv=AFvG8)~&n&z! z>wcqjdUo&ccd;$(NdM=j`265c&L?J1yxG?F>}_{_wry>?^aan|yPK}R#cpg(b^$xz zf;Gl2?&aw=%jBtFht&{S}(z)fW6^mCJSIuQ@i4|p+ zx3$z#v51krkNGj$t;x!E@Z?f6a(ZZoC>r5@Ucl5$FlAy4?Q*}B&hb1!m&U%lE*Euc z#N62h7Dtl~c7f-y5Wr$VDS7_#wX$QaKmmSK`iqLyDz`g-`54&Z80Kl-ofTt{b;TI$ zT#%ThARiNAa&`dV8`oF>zV?w_b1QPe8_mRA%fyml9N}zE z_-m(6zyG|m?j+Mnf7=xbb%mHqB&x=o>~}ut(o3hDKA)2v)LFgfzUPV|zwQq${}Jm! zdvqS0#f$auxa~yCyx|1clRx73VPI)bD(DG&?EH&%UAHgnwu8I!`Kp(SFWc>Wqg^Ma zTe*j+Ez4Kzf`(q!&Qco{4bZc|i%U<6aYU6B7)Lx7;53d@W>5_ia)5Ny1_i;Fuu5e! z-gKnZ5^0T^BYvyJ8eYL}Z1AdPGrK^uOnkDgwNvdLC@Di@t#zMFFbngC*yBaZnjCxO zZVNwAs{vvUm;SyZn;h!w92-hzJ6O%btT}YL>chAEtV)iFcrVtkM#9EvCDS2-twqu&y5y= zw;q?%OgQCDn!(c|X=^MS%LcRltks{LOR&8^`AO+?V#}7fxh-2D&&;XX#mAnwc+n^T z?I3bku^;?ONNGpAEzQ9|wZK)t4otF{`3c3+*b1IhG!ph>Qy^76GG!OWj>gw*J9S{; z4GguD#dS*bxuJZ1h^DeJ+j4C4fm1qeo$MT>2@;LZAJ13vO*7V9&^G2tG7zXZ?FfUm z#SMB%w5<{KY9(%XvO$a>;P-@EExte!yNWhJc8Fzlj6qNMLkn-vTJq?^8$)^3(jB7q zK=I-s|H2zsK0QCgqux+AWHJJLC*aI54Qv=}8o8CR zZwEnEGeI;95)@8khtt_i7IdVSr-7d=zV}u=kyugRRIfhw zeDDVL_QJF74|wmnm%D6ymv^z?^V}7hzydG+3&|d1l55zYhOj3av4&o`Cs_*%Sec7K6kNmX1R1PD zYix+tfd4N`+-xrWgR9=NE#s(Rcb7VHTc13*dDZG`u2Vy5+-xoVUX3HO%~S7URi&d_ za|fSnjU2xwx0TQZaKH4&{58k8C}uC~%bS*!t{HKh8i(U_G87Y4V6Mbq6(WCwXB8|!8EMz7QHK&Z*mcFpc< z+RRN&4^&tAL+^tIcvp=oXtiyp&{<>WDx_onB*c$TJG+1&G7a-fJb(lhUsyZ?n4aYuiGF!~%5BNht zkLp&(Oy-jvTIYsHHM$C!I<(f1-`DJlUJRPI*qqTW+kTY1z~}7?FWT8-kChzvs)6UdU2dnB zx$Q4tyPa>#r3G#wn2l*V56=aR2F{ncODvttVSQ>#9gal)dghYmi{bh)=H+FHv=R)hRtN(5RM_@E0? z5kM8i9$Uerye_+vY3w_3_P#}l!_lo1O@m<2iy=ee^_*n$LO%GqY8Q0?Zgjgfu%~GcgW`lM%ck$vJ0hs4ShNL&iUr07ttjmJdpcTs@YpWWi zLeN`YSMXY|ok4QJ?b0l&5gLe$Y$tuGLVQ^KYqd>=*0HTNl+kS35%>Tm0`e`E!ED_IcN2j(%)=h7jWUMUO0+h zRRdK=F-j8tO~s;7T+L5ZJE`9#xx)%NSO@&}!yd9s-zo3*_M|@$v_@C3vckh1zbO=c zQz)I*Tce|GeeMd4hi+VZwk!ITF`O4lyst z4Y9otCo>pme1^Sp;8gd3{bk67rC&829rHZ0Sv4^W_lM?+#W|mfdf9!dfV9s|K;O|StI2k1ficm_+HH-M&Az?i*JgaZ@5^* zE(GBy_gO3&{S94&SP6KeFT!J~`_y882z_O7zCy_m6O~Qphe|_ZM`==gUbZ=u2Swa{ zc-fe%m1d0D?+|)|HxUHK2lEHO%w;$(wR`cy*WG%iYh_pcDb`1TTj~Ka=bd}qEvd|b zQ^m{sB3zJTR-u==fD1KM#C|~QSdzg!U=2oM?a81uk|lZ~xEUA=&kOD%%>%Gb(5GU} zTOiHa&bDc8$;Tnw1g$O1?*a*kxmaWcc5HS9ORvEu4`$0U9^0!Yn(iJ=IPSjNkr=(Z zDY5+W^zl3}LDjB$vt0K9RLLL5oR)B01*NRQyg(`CyrhZKYKCkpBzcJRl8dOC)PO3V zwaRCOc~t7^!d#+yVgv-}OF|o3m8R8-X8{D#>>(A*N?k%eEp2Xp{Og1~APhL#`%a==_CxDO?0Cstm3 z30%#eV0U(fut|VC7qL}fR)`ZvgHV2zC*{}rc8UrQR$o+3OBx1mZ zBw=TjS?FXCbR;9PLY)=VCY?28(R%*NYUev|5yJtCsjYSrP2lsA^AtqzGR9J<&#=SZlzmY*a6=bs1jPR3mA)Spy%lFF5 zROWpz3sBDaoT_RIIQP`UxG^?pxxq~=8DPB}F$ARVc7;st8!RO5cGmB4ZoCptXt$F* zCv5*@5{La6dkp?4(js8{AS3-dZwU(s)Cst!XwFM`ri$l@b{jSbv$P3IT0yOVSP=dS zw*x&V*WCoyCHggs=e+QPsqGa4jr6auy%nO1Ao}q)D@u%U$o8tSy3nH?Dvbl+CYu7R zr;${9Fe_A8p_~#-b)dOUM&F@rV13*8{M%o^J~;k`hJ4<8%LsADky~hvVqJxtWL9i& zd%G1Mt!u5vSyM$+o%}ek3E&T+d^?dS@rBYBXD1idLoy_TzhGTt(IHuqpa=xQPQX9) z0h)5@Nist!gP>qOtZ~ zMv}`QE9zVNwYYBcTms~PKGwK=(ESy}0lC<7k|w5-tgTAbC1>SlGFV{0;z+^k=% zP^`6tvGjFXO#;T4IOYvy2(y&V4OomZUoa&6Vs1-oEuS+>A1T9w;)~}99&%k-92Wn0 z#WQ5b|rc;Pr&qX~%&%}F#z(-avRX_b{G<+PY*7c;v8*q~hfsmb>XW+&kft>v*aLckMzT1J z?H52T$v0c|wF=q6AAu|`zT{OizHk$e;I$04CdhHNvo^$$PQGVNwOorbI=H7r;%%PvE>$cds9X%hLl`MJ6ID0UQ$ zMeHT$iSw|nEZP>KML>Fm^x}gE6TyOH{baI=g|o?MIs%(H=}Lgtd<{kFSU|8gs^G;wS0(6~;HoUQld?%1QRZPOq4L+V$^Kce3< zza;Al%6f$Xs zJ(ifhc0+%g-EIkP+x_5%O&`B;lgFbvI(tX2(;pCqr(#uYQ^?=!6x^22htq48xpO$v_M&$&HhkRZI$5SG*{TDTls&4?T2*ow$^%;=-wcMati4n z1CHQ>9wQCHD;N>p7-?idNGxoNs;bt2YwvLPeckc+x|?c4{(9F?>4DPUv%A;0{U0rT z_kOmD&oj?W>$p&VVcQqtdrO##R}$gZvxB^K55{&58Yt zJxOe?lC{aLO=P4@bLhDSp?60bYv?&Ikwm8{*lPk&G^LoJkdZLui?+rM>F(~;>w2o| zMK;_&(66yNkzdnZIw!7G&E(FlJ&^0YY17!o8++wN$M&_u>xQ?M7Ubo=DWd@UWC>?f zaBRpICMlP|)$9eavi2=$}kiDm__jweO@3rN;(HfCW16c9Drzu=v&AdeV|?K z)Hl>6;GWe_22rqia&JR(5=A5kv`TN7kZQ7Nx(gj9+tU~<`a?Zgk%=6%J-S;Vf)l z0Lt7Py8yV%l2=b$%8RSCQEe5x!D~D$o5J(-tk}HN7&Sr#rE{V&8p{&>vO=@mh5fr@ zQ*622sGaQeFjBNykn}REr5UPzt2F@U1^%tXhqD=YE_!)(NR36wpAto)W}`tTHWeJ$ z>Kc}gmd$AFZ|-gi@CbSTFbq6RJAy4%%b{gEY$%uTDdmFttp;N%I-l% z_DCo&{xE-elH$n7{aCg!AftazXDcW*!Ul!TUdgkhUm~V-!*`ujvXDvFDD7)ohgPl3 zWm1X0-gs9>w5?TZZfdBjTAsney4@_8{!`-jJF=) z!Ih4dvLfo`b6!xSXZ<1gZ}Sax-i2Gee9%xRy`{56px72K`EN^adc9{21=65bkhPMa zR}Dn3Al|?mA(VFLEopIu&Y`6UD>6tJS#HW#Rgp`MU*q7S=7Roe3s? zbg=ZL(wEq2hzDcPE1w=LJ;!!djFtF|h&6!Q0rm&jArNo?F@_L_;&0BWr8|IO@M|p5 zV^z@OMSa^7_Ik3gs==b^kpd(=UXG#yyApH&grKsGYS>(CXI*eP5|0)*5;5XqlEGv) z>GAT5Uhjg%i|r)ZqCAxW=_qVL;vCo@d{ur$1HGvFS~T1cs1i7rfLDhc3FNwt#^9_X z`3W{;p$@^_j3^24E}?yX_{*-JGFZvcEqWTGQ3FhTSQW5DIvH?aGyF zk3DtFNc2_PSEc&;QuIYu!pDfmBKavGX=2$iW)X~27!K12bis%qj}Q|O76PUUm*Ff- zh(K=yW32f=f-Gtf8ik+mT7n?g`{Fb;KX*699YJse1^RPncoAwWVN!L?8DcsO|&<8t7Kdq z`Q9J`nkB+!vSBC#S1)l1?-teTmXcyN2z!u8TG~Z)8QW1+P4O3{b27q$os{tyrP<}z zx7OA-`w?YU^oCs3PI!_{W{^hEMU?qN`~?|#F(>0GzkJ~2VzhR7p{k1)r2?m6sBWH{_0ElUbM_IgNLK-IGf3H)siHZ*NlW8BqDLfvrrdWs4Q)9dtse@ zdgUjCVS;eqtTrRor(4+x+}wGcodNd|HfhW?)@zo&Kqz^^fH7$!vL>6cBDm6s!HHpl z#=MPK9r)$MtSMq*b3{&d=aeH*<1sr~L&)!RxEiuaV}1e(iF*QComGb3c$)@#%l813 zpfU5g?P{nz=baV?-BPtdTWz*ha}(MUGZoWM{SRhCnFzkYoX}SJUdUO7!Q6JDaqr(o zLb8vfcTx_Lc_9mdGtxeS>Lq@OQ_38%N{X~2GqXscyW%7GGs(zgkD-Vgl572IYkT7z zkYbx4!@3a-Yf@}N*%Eqw7JY+R{MNh>gF=GJk+TUtTB4p;&mta7RDt|*^%O%D@{~bW zj5rfJQ`?DTU`|A(F)!2;bd*BO#H?&*-40?SRIJPwWee=&%AG603XhI~c)|FF{nSOFGh!?# z$5_gC)e2iJoat~E2P2Di)sxrX1@%rZu%q~ai52n-sVc2aS;J)k-@p zd;{Wy3fO83T!q5&L-ERaY7XE@%u(n#W=fLr#fwEffiJ}Ja(e<+LE<| zAKks(g4^Amu2r=T-DK~?6Q#RO-ipICub*04fAsAZ{tmxK*q(*0z{wFf2t!Mmg~HS< z>`uZ0#bj`lsuhmsPTqG=(;VIR-t}1S__ab%HRvO3wh`Qv~V zG&_H|9c+aQBq1r93w9*CE!)muNoGLTzeVug92sfn5XkrE$Maj-qZVJPLz8<%)fWDT zYO|`pyy$C&v*cMl#O}-w#qaIxfR$|J=B6QX#Ts!(SZYHyqH|Va4G|3|{NW@V%W!qt zet-|{BU!&P7E4MthFhYdjup5s;)wu1vE>0W{6qMs6irp&xM52#`!HY%^9b?-BDCbe zxT3yEmE)D3l9RN7s6GvaZ1A$ap@)-g-y;2CG(Ru%Kn)<@5P3$(YF{3Ys4sm1mF*`z zWJN{{f4O};u>=p;jThsI!xA9IeMQin>M|XGoeaHWV?;bj0bXenCTp2cMTEYoihVET z)k=SXLAtLHE$8)bgCWbk^CZ^uo50^ynC}X|!3)9CL!8!NHBV)%i$OWY;Q<)FNR5Mo z4G0$|PZum+RFegqHeo^SJ!b+lN01IFab2NDZcAX#&JK1aZhOSX=S_p1CPXYFPML>S z{t1QZBuJ+dieKX3Gqtx4c6JWlTKmkwgbd#yxGnlb7U3qvWdPWihk${mv|%2t;aZ_f zErt@qWwkU`(l?~sxh#bEA_&UDvxt>Oe1dPg3>+>wAcoRtAd+J3N%#cL(0DFAuU26n zES^bVhJ{)vSfFOi9XS8Yx-}iIfApF2kMsF8>z+9uIQIDYXFmEm@P_a}#%Khw&JNO3 z7{ZQ{X%IssbOJEqkCBHx!uFCK4rEXK<44fI@&%>k_5|L9(4Jeg2hEx^JvcAZChO9L zXUGK8BgJV18%zJ^ca5CMmp}G1PyqzQqs0E2t*dmW%(5p;&en#281ton$6v&pbEmcw=4n?au4S-Sy0OJ!_)R437?}-km!s`%H9AALC89lE}Q4u=a{lsF?svCed+$tOaa z7j01y!_E-)lp}n->@^&SN_b&c_#Gi1sao0GfB+13L7b4F;FcvjFxlAyXuB3Cz*OnS zLFh&Xup&LLHOAWIaWJ;Gp|13!8P;+CbFV)7;c4bB?f;u|8Jq=COLwx){kM8wdEn7k zcQE%~oIlrf&ql+pbLmMzUxg2m>^jTN?ub3@vBo@-2+8o<8-?zdFfJ=@giXjUz22DTppvsdH%LW6F|Deg9C$UdSM+ zp7x>W(CDkBH(v!RK|E#3)|M^z&|%-f{gIZfE&V6Q9)0!IN5@WzQ~pb9rV1&%>T3ZX z`D6q>&~aZGYfl21IG+XS6HKNw`!b@b?0XiT-D4M*6e4FY{oGzG+F64gv%yqkd`1Ny zq8KZR&sg-iQhbIXD9|A=I$A3-(&ZcZ!(Y^Fjs_FH{2%G9mVVYK`jKbF20-6h3|u3L3WtCZ?%+>khd2<9P#On9qR?tn zD3Q`R#3ncc!J<>KUS1s7Jz#gM>M!5}2?cAq2L`%pf+4FV@C#LS+sik_1<$|B-OC^4 zc~K&91~DqX1|25-$#%9k?h?EXv{($)X`)ya*weB@HV~>Po#eq8OdMbMCb%Whq zt->d?0gkZ?msD9O$U4ug~o53-O@Y zXY)D(L1$-uYkOUfV_X05!g^AJDrjj7EYO>jJw!`)Ub{9IZ>u7C6|__a{914>6a(r- zAdQtqM)(Y;zq%x0Tq$!HCGA(#kukJu`aN5E8$&hQ_ie8UH4b#7DV(;!5I-P$_+G5Y zv(FmA!*rt@$D7<<)0J}cuUXUYXkB@&h#z*4P$JCDMPmANCCx6lGA+BR*!x7Igsq!& zng~K&B|pbm9V?97=_G<(fuzEJJcu|49L9g*%a%Z~Sl_EX^8~_w^k+V=>UyvC#KSEs z5Zw;m{_<-o@%`vaFGcm&URL$!^UuTMWXKPK-uM^!eL^_$094|_*&whq>dvr}r|-VI zbncGvV~A$?O@8#qvtM}oZA8yf*&c}1D4`gv zO6G7O=P!87;&V8M?59KS=?E0SB7G~Uo{)jDpY!ktmHUC9gJandKaOyhDJ8*2JWXR; zqFYsXfeG=kfY(_q&NzA!ra&#WB5#Wz{F=hdkYX#IW}QF$Nb#xCUqAgCix$6p@7Pfc z;v+vS{pj@5%=eUDdgHZwzpNjH=DZ{aRDohqOagFMYYO@(FbTNpO_-?tUXFIb(H1*E zM`hE5{t_FW*KdC6zu)uF&mYv!KO+?APQyexUwY}Kd;a@VH|r1n{Gn&gOJ%!kC>3&` zSjRA6;Sq9MnD&ZP`jJv3l(dveW`K|@a{7}r4HRZ4Ni8Pn6tPJ#k9QV@o%CYqoRF@? z1&?-$bD~@TlI#PuIM0a~cyE=U8=wl{QDu`X+%lOkp)WQl+y+~I0)nr{TS`MM@i?dG z!Hu`OJ#Re$k`3kjUKFk-)zFzjPXGpqjQ0<5BRHvT`n68n1WDt$)8LXx794u=Jl9inhOTl zy4*tU3>eu#sT3Fv|_Nmk$>MddiLLcl?ftEQR)K?w&D2nwZuD7ZAh`NI%oX?s8k zMEAs_A-z8f?rCt%O1ysWHp@C9+BVuO+wo}IE^kwuTNAvv^5k5M&d#;BEuEgT8fWL0 z9aW)2tK^1}=hl|eE&K$b(ZW&u=HSjE^TXmVpU0gy%4kL=MS`L6Q%MJjmI&Jc^M!YV0ahT)5@ za9#<`svH+wRt?I;;PUeFb@@K~un?<%EPlC1B&DB=kR@r1F@m%gzFk>ER!6uB6>bv0 zWamU)Sd3)3EctQeU6GgcQ{XzSTRrG!5QiMChEIC=GQpYzT>vrtt^61r^j~-gzuVb` zAFm8Gt!h#=l(bPf|8ICxfYb;QiA3f8HDUKtEU^)LXy>qjibDbva|2t8qkJY%y!_+> zo&3h>Kcexv;0qLkSc@^b5Q8Z62^{^lvUdE$vSn);tt0S$=Tk_x-d*aFu!0Ro-Y9Op zM;sS`p0Y&W%WI9jRbE%@t+Ie$Zn?Z(pg^bE9+ zJX1I?X2i=u$_Bkf#13LZ;3nn>0eJ#+fP`L91YozIt)D|_xuBB&(Hm_1fDOI8MxOB( zGCOz#C^sFg!x=PeGCKZ1Co<gp2|!4jrbaSO6X!>?9ULbX+xTXvAmyQl}9%v~VI= z3!M8u(_J*DN5n14CUSX+?wpH_?oUJJiCINd(OXJh+ks_BR}#7t1V)I&!e15kkn~O@ot<>Ic)hij70o`d z$5cbTGh8|yZ?ffvN{0daPq(P5rQP=gIt%$7Pi?-Yg`I4&9r$qRpXgL5=4R-lEwC5Z z&PKGL;Guw-I3Xv6FR~bjNJXixr6V{?EQ}zK$$_4FBGB5oLYR=u#~x_PWUkePBgr`}zS=;U4%-t?Dj4?Q=CpUG}+675F7%!W>pkV-far zsGNdN2rIgXFUF}%kaB517sm6;&K|lz0Wlx9i0PzofhBucDgzcs`!|g>Tuce$Fc-)k zK!Nqpt_MFS-1Q(hI@u3M8X?0O+3IDm2HU%sVg<_U2YyKyZ9D6$#d$%&>K6MTM2V(V za47Nq3y5op{f}XPEUYJ0mqZ+5Rbxjf%)C+$0ZvpyN{nDm*z3`@P@M;xMetFn;L>IZ z8wblNZ?4Fbzl#nlzhLK+A}Re?Cc^K7lh&nXoMQed0&rwnBu$v~U^qVr|Ce~Aq&Fl{ zc0(%yk6aOtwY4-g7(9i}m(#l)psZmmBE>jlN=z9d8Rnlx%+s>8>a4xUr|?sHlYYdg ziWn^jq5W)?{KY6=#%omY)$MzrwCg%u(OG$<7^6WG0VjHA1-*3wa0)m1-DC^^oXB*6 zcMc$4h(@p+R+VrgF-XFSr3H|T1Q-khK^aaGJmqVG5z!q<>q&nRbO&)SkbB{)kHpAo z1eq88W)k$;6=L{^0e~qsM8N=XGo90gXe+{vmUIJpZ$KMpV;hdp3Y!M)_ZXCNyrKj& z0S4;`oiNA_(IJf}y-Idn{9nm!^>p9}5`n8g}>V zUrayz^{+gV{$l?8bb55puFaX}3@zx6u|0dn?kJrb+O=ZEu3wh*9|1d+{9F_%XFJ>6 zAZ!`*IyQe&kWexolH3mqGT90gLz3Vz%{5t^R3F>l)mM6}Dc=;rzVSX*dQr#$(5P?| z5hVt(sSYrJlWqR{?Xxg96*D6-wK{Y7L#b~VfIer zzOlAP7Mk|$iayeI{Y>M+!^!Xd6GQO!KQ+xrrT&F?_WiQxm?Z??tp^etdbtAaLlWc)xcYL#)OVvH1n*7eUFBOS(lA7c~Y z2IQT6?~!HXyAD|W6W!IHsK42@>i;O!z%+c8z28&0^cmqjR^UAl_=pNvLsh%<8D&)c z7}Zx><*HKN`22)XY&|}#it4`i7q*Ufty6iA@|D*VYWQAlm+O|(%KGK9_j;b{S3Xl& zm!5w=ZB#zQ&Z#x4Blyo$o9;7x(e%Ge z@0jD}A@g4Ilja{g{GwTJL#a3tQvK_O{*O0kr>aOb1>I2meR$p|~I<9pbbUfuaS7WJ}sJXx9$(nD~{GGGS zdDMBz`JD5I&XOzR+UnZp`k3n}*Ppp9?wotK`>6XQP) z-Rt!o^{eV9>OWfl#rhxAml{?z9BBAz!}lBBY`D7XE3jegVp>?=*qV+`US6knS)J0B4UWxp)&DplOZMN;nw(qoEY)`e{)Ba@p8&Okq zWAyRpUq(x@q1aUHSnS!@f9t60*w``K@k%EJ-V)#Zsd5032=w9NmwcF+>f1$LfnDs6 z7U}S?@}QAt@I3t&BTrEn|J%r`N*h~g=j5;%tTT#VU)}> zSRnqBk>{{x{8uBdDx=D;jJ!#yWj7mnv(m)wHS!iEz`m%A;1%36$|PR0O|RJ2lquyy z_}z|3p3V4bcq79>yq^0oUc;>^cZ-*CA3$!ScxCqyksijo!DdjFK>a?X9e~Xd{LLyW zVXIo9>@(_8D(m**rQiEd`yie>f_D}vBZp@ukId-W)Q7a~y_zD2wHmLmtW zjfV~%*?8#i{uwRN+oyFLIC5lm<%$*iP`Zywd+*%WdvN9m+NgNf_%+jq4q`=?y>I*$ zl-)9|yywVQV)R$ObX>zcG`v@-2X?m}%(4&p6dGDKu$9`bgGX*Ta{G+ludUSjd$K)= zzJAoYvN>h3qVnEvK;J!c_|97n9n|`J@uw+(-YnpC5Mx+2u|u;n2Ybr1lh~+SdI00R z+UKVz#3^9LnaWIfqmu>pDjVJySH-H8^~wf7XA>~z8s=a%piM63Mzm5b^D-avvjFTs zb*!E>uttV}2*j(kFb(lct$6=T8*67#7GoWF{c9KNhW)Gu@x&`wAKvbapb3^@X_kSM zpJM}TB~B-)0?GVe8ojwvlaOqwE^C880lpmR-lTvTbZT+rh@z^=v2G z#dfm~usj=QH?TeIMs^e1%Wh^9Y!dWyn(1tY?PL4d0d@=2t}A7qEw zo$Ls^iydWmvt#T->>l=EcAVYI?qeTe_p{$&A4R=}~ryJ;px8{wBWs(+ak*ctXb`wIIiJIh{RUt?cq-(WAYKW6jnKeCtD%j}!%PuMH$ zPuaKFx7l~tcUh7BC-!ITd+ht{RrVVDbM`v>3-E^j%+9g@!hXnp#Qu`~m2xFed4C_r zX@~v(8>f@ z^K^!%vpk*S=>eXemG|%WfGs83cc(#vc`*}9Ovq_#!@obuBGd!E+*&NRf@a!bd zPVwwC&+0ro!?XK%u8-&Xc`m_oNuEpbT$<-HJeTFU9M28#+$7IU@!T}e={z^XbNl!} zA0O!F0|`Emkm zHOZ%@_|!C?()rX3pW4T#`}lM}pHA@UB%e<4=`^3t@aZg{&hhC1K0V2&r}*?VpVs;G z44>Y|^**lmb3MWJB-c}1PjfxP^(@zOTp!>FWY?#-KFwiu)Mto(FudR2RY_h7N?a=_ zyYd^xHEqk+73YpE1TKJCP=e1W%5egj8?mFeloRAV??P{s?&NM!x< zXm4a005N+Y6@X4bOM5s*w%T8^-qJ!;x^~iM&?WzC9lcfYveKkp=s=Nir4{<3RTUKQmsl*>#sPK=L_ zHx^j;_;{qCY|qb(kM|VRxVAwnnA#^XAoIxfe8C(UE?6SN82)&HP4pB@@d(DH>1WJS z!y4U@ofoP`3d+QWg4z{E>4Y?vVhesuxa#NFn9G7tZ|J7SUocRb(1oMDj4G0iE*kj zv0e<&7JuGat&D6K?g}pg+8$pH_$t{7>&6g9Fxv@j!->cwErNiO(nydjXpIFdYa3NKRZDLrPK=)_eZU*Udc=*J`nOaMC z;c$0jE5PK#+`QdA1%Lbuqci|GQyPq)Q7Ns9pD|HdA3tNJv>|@RLTO|CjFr-+_!%3e zq4*g)rOk1rP}BV{7)T2S(u@W)4204!2102o2102B1EI7H1EI7X1EDmEflwO5Kq&3N zKq&2uYpVpFcf~P(_k=crMVO#Pn?zdZB&6z&7rMF&UDz&hVCp8I)K&LOWHJ{aI`y74 zfG<6Tp2am_fkM2i!2Epz%Dt6PS$=CpTuX~__Mr~jaOHLd6}alKs9XtrRnXe?Ly_E> z70i#B^kd!_=v5z?0M<_CdJ2hnZ*WylA^F>?0>h?JJ%y!E0_|F_wuyEoKzPlG6PqHN zKne1o*PwUUu1SVSN%Wrv2?+rE@h_?r>?7SXCwe2Aw(11h$}HX1dSx306WT;AtuR5G zdF_t;SGcBXjbFhF!5hYhiNM)FDA6B!jBLc#!YVG`C)m`iTT*d8GNDHb>d2%H8pB5> z8~6r`3`8wzXbaTZbVmBMRJYd ziuDeU8)Fc$e~xpta2BEhJE9 zQ@oHuGD=X}0Jv%!!L!P6x+YHOSQrIZH^-k>ly%5#L55N0+W7NKlw605DA`JNhH+~f z)uGIGszaF_REIKSRA&g8>!}W9c2XV6?4ml9*-drUBJ%;NLzz6)q0Bhdq09|bX9Sr& zREIJ*QXR_NM0F^$m+GuR=4PrxnF*>xnMtZcnW=aoy9nlKx+n~ySQoif$ju0RLh))` z?28w2i?#RDg{XZ%vdqYRqR@Tr+G9AMsVLf0GmB@H{k&9( z$MeMEdX%D4)$7*{jm=ME&&yC9P z5Iif6Z;~z1Ves>XqTo5s;51bGZ?#U*(Z8WluQScPTCKR04^gV`*3_0;xaw6`H2dQAVS%Dq4X|gY2a8zpT7?rYl=nrE^r*8M62n6<51-) zbynb5S0dELz_CRMSC3!?)zGWZ6^+q6Rmd)Y*8ZBUCJ<}6r;#h%J5x)=g(6r@tvg%QbyuGN*SfhP>NBf2*-2qU8YRMQ6|b} z;F$KM%Hy~<3adCsiN(GjYLsD{siZ5nVVe@DOMA2KAY~Rx2cd;R)a$P(!%7Qt%L)sk z@+zaU28|pPHEKq2X;IXiqOz$`nZ+~8GK)(eFN}&G6dToVYFXLL^xJNmg3>8eI%w9E zK{E==(8dTQUv@MLhxx@buqz6b&|WD*SrPXC?#a{f^yB2XXq?mKjKrag%Hx!QN(%nt zF~&G05e;>Du=J>LGs=p}rWY2(MWsi@4NMsr9~*~Smp7+esHiC8(M2gHqewnEbuuXM zABBsBrL&5PXGFyf!iMu=%xEE=ZeZ7e70)c3F)%nfq6_oCcYtzkr`1MTZzU9?0QF*CfW*)7K1+6`zJgVd<6P3we@&Yj6RAm~7d6y!czsZgF& zo>Jy1)yhJMn59aMvO;-UaVvGov&t%^L0PM;S2ie{lr73OrAgVTJg4k}8rZA6r0iE( zl>^Ev%3XlkfxQ4KXr?WRVk*Q!0#o@%6eoqB`XTXm>W>P>32 z+E?wT#;CWdgVb0xUQJY!)l@ZIyIlaY3g)!hB{L%Rm;@bYK8iw`jk3PtyUMRi`AuSjk-d8T6L>+>a*%9 zwLx90u2(mxo764pHnmCJslK58mwHYWaq$U>Ny#axX>qY}adGi+32}*WNpZ<>DRHTB zX>qx6d2#u11#yLOQ{rReWO4N=iyn=sX$fhGX-R3xX(?%`X=!P> zX?bb+X$5J8X;X4zbK`R3a}#nCbCYtDb5n9tbJKEjbMtcZa|?2(lt(<>luU@)VRFGVdQjl7ZR*+keSCC&&P*5m^=>NN#xgfg(Dn?P4flQWzP#8$% z84yb?u*F@_s&^~*fCcYWSAuxzK|ZTNKx;rk>p(<}Aft^Sq|G3utstiDAg3K5sAly! z^?7v{2y3^xN8PKwsJ^7`Q}?SaYODIPdO$s>zM>vd538@Luc>Y7Z`9XSkNSpsL_Mm$ zsUB0`Qr}kJQQuYHQ{PuVP>-u8)DP8@>TlKGsi)MB)ZeQgtA9}csD7e;s{Tp+O#NIv zt$v}NQU9#|Mg3C!O8r{>M*XY$t@@q%H}&soJ4pKxB9cDXsV`ZAzG-WYZlE4Bz2V*riE+Ww5zoU?HcV`t-IDkvuQmwyB4YS z(yr64*KW{m)Ou^b(j1yoi_-dNH)%I((b_FqU(KcU)B0;M+5qiVZJ;(tsnc%LVzoFe zUQ5stwInTBOVLubG%Z~ltlh3dEbSp}v^GW?tBupfYY%IWXxZAM+GARdHbI-HoFTb;Go)k{B$pqOQiQUI{pWUN>k4Jhe?yuQ9y1MILy6)TSM_%7{{hw|abi?Qy z=H2k}jrZO-{>I09NA}L>eYm&(S2zD^!LR_Y|9CP@b8P0uCiBZ3fs*P%i`a_?% zK1=)TxoO?a%cJK;ABz6*maA^L_m+jXeAxH;zLWcY?YhzRtZS#M#r37@d_Q}?n11*4 z%kHlsJ}nvp_nZLZXJ*{fZuxmt!r=nao__3rwyzhCR}d2C)`j zc8l85!WXxMv_$fce9w!IEG_;8c3(DM?9aAFFfY%cKeZ#v8`AR(_jF|0qr&{rBFFCX zN4tE{E-TOBG5Rl6Y)3_rBVsuInb#N1nAac8^ax+OSM}BKoDhB%EsAj>4%;~H;Gx(Y zv=^bm;moGyMGm^iaWU4Wb5!K0=#UNI!9slFJKcYI{Yx6Wct7)+9}FzCPuTe^Jm*d3 z?!p|ryKlZG4Equu8(^0 z?rlSuA(};~{m#1{?aPFPl|EBeJImnj@lxGq@a}dI;Sc9Cm|p)v{cg6Gotymk%u|Mc zy7<^GhKcU_5uyJpiT5ls4)XE#cSW|&uV2IUKfKRXBjVha*(#PUgy(d$+Wj>m$I4d< z4`Z7;5EM zsp7?2%zL4^P*jl{qh=Ytxrf@jykoN_o{btrMf%nwxW}tKq7JM~CNHu}0 zz8bok{tiZ;8fKh2rH^}~=nw2PJH6-B8*doC z#ivk3e`DO9VJwxU7Tq~+oN;QHe(Kc0vy5x_oAi%iprZ^CWq#m9}4 zr}WB=3wE$(*1US##*GFq`kg)VZhd3r>M~Z$iWihrRvIUV=`X&x&BKncBW15W{-O~v zXv=J0v@cp^zG!o{`-Zvv<#r}c;c;DzpVEI_J#EocHkB3CPj4_V6k>n*Z4TTO<_bN| z-k$y1RKuU*Ptm8oHv4UMobhyi1GaQ#@EXzGzW32Bqu2;0(!~wf(s4Ly%cFa#Ihsc) zr$WHZ=d(Imz2~zqhrZ}YS`lB3l~xanOr$4e8b~TIogqC_eSNS%^H$7Tys+93^TZy} zlQ9>T$*<{^ja3^RzUM3(8yhz|eVW%RdRk}h7E^iM@@J}7EvTEf!f=b8b{;K;h*qXA zK`;HnxF@n-ScDhS&f5cn#1mi%ZQrf}9WAM;S>p76YF*;4S?TDw!?M!tUg_jxthVp* z{1)4{EASMn^oQx;R2^bgI}c34*6?`!(P0# ztl9Alt9|+zX0(YumW5A>5HW2+Mpa2=5u3mY))($5*-^6Zsr}6Gt+MQ6FE;LIGTfFO zJJ#=G``Ig%d#iR#_(X*8X$vunL@#K{Y zbjIEj*Brgc@Q=3~{oy@+4P(a2)r=<-&(m0>^blHHoY0)?=7$HS-J4fb`WSoI=xDXD z*Gpf`+mrU;!{4!g8C;9|T4)Z}`7Ha`S0)}g^2#em9424KfD2-{cH+db4wvt+HK>`K%$s#4xy7*gcJA45kR1*_qsVdDy%xHSZgILS)QiRT z!|4;lQ&WczPj!kIi}~mtk_H}AQh*{oBvb<85VYbA@#1<#jb5;5`t(HwMok6tAJ$V( z3_tDg9rpSUTZ+pu{a6C0@38N%g%-k*Ej$*N*9As{00u8gKEyEC`BrmW=%Axjk04o( z;(+e*e;J^{Z6+1^z7%cIV$xag2T_m5dx44|AzSU{u*4XvBw?|{TD-Nq+0l_@kq^U{ zfd1S|9AXS6Vd5)e9W)=9P(ez>e z|D(Mp*1c_@1u+C`u;{}%N7--K{)Rmpwrtq4dG%h<_15ZjbJxvnC}#zR*TRlfy*}k7 zW6DbpH$KFS2p4fKhEEa~M=7nV-AAt!w8;O=${bg&8;w<)CKsg8Y+5B_kmY2H)wOZ8J_ zN5*a&W;Cr?zm{+Eh3oFxr)!th8j}v{{tCatKJ=kcL!GSOxWvH|_Lm=?|0-mpi-%)# z{eINjL!A*z|M4Rb)ECV#^?*H7CgD+Nh1?as~4BgDxtwR>sTAp zS=lq?wX=vkQC8CR^Y>Au}aih*=HkItHXx+ZAW&0uHgQ+9ESW*Zn?U<=ujnkCB& z(Q8EUR{fLH8GNt^XZXty8K0&bGs;D;hSJ^DO$|*A4cHk&c&6@Nx4M2kGngA=*XH0v3OCrvg+U32OFpu^X_o z$mz%eO991t?Ed*(JM+!A`r9F#E^Qv?0PtPPsddTw0z4>t!kO3R^$nzvuw~1ZFEs{= zk-F`RTLR?T$0CKB|ADUT9h}uP3+}32US|yCxXZh|ZdonvvVGxy01p~u4Ppx? zNfC$5%g;t~?Q19oQ$67OYpyv_gq_0`8WV;k4E06(fi`^6rm&OR1gwMtf1t>eeP$JW zx7+D*2lTTXpoe*T@ONmSwpV*QhjIY&Xk?0hV75F^BU)`L+M$| zI<{d=?ONkAXcF5iwQHBInTuik(VxW%PoZG(`Z;T##BAh%|4oHB2MUq@e$JmDOA*W7xUFP+GDlEWOyOfdHL#%VFtLHk0aL>oqb=3`X9YY`oNX3ayTy}Zsyu&)T zp?aO8!(mz1(6G+g;RsYDE&_zY3Y*xHyS?}$bVpVV0nCA6*)9Nv(#HAvb2FM}?0kYi zbLrMu+sd{Ze1sKC1gPdAYY6LNT9%lVt686%g%6+rwJYzzsyFxXZMQJg`i zjEA>1&&LJb%i4H&^BP<^bt;>OuW7~==EZ&Un{i>-Dco1QM#mLBTe$5(CenhV#3OHp=L5aC?6+aMr34S)3pyq!n`I|KN;uEi=E{~*l}_Y? zw|TRz!IRU&Pk`XO0qVnvl)u@oHmkhi3YDriJKK5zY+wQ+@I4jPA1vm%*N78@?CxR8cq+BKU#(3LsX4^f) zG>K-4;n-%1nH+mQ6WefXGo2h4P&5-7aA25i;}BP9To@>_pPkKrwrbTP!0L9vNd-&N`?Qt~w@PCkx#I#DJdxMt8^pU`x z@YlfjlAJ--gRCp(UU~q*8q%p@e$z#AngELs$>U5wF2LIX*)TqXM87GSr6LUJITK?> z#lV=IUQ5v053aofMZtk*i9&mN>8LwdoFRY@xE6o}?CVi~NN+N-62Nvu9}qQib}^|N z@SNvcJF=iqZ6ALbVPt^NDw_;Snu&(u8e+Y7 z^yqt?*;aP%fzijS48D4#zHZs(QudUQE%g=H$ugfUbT4xo-=Q&9w551k)wZhUCC@YC zV-U#4mJi>2^FwEwm3=t*%@K`;Sp9)Mw{}hwTMtb^TFk-SmNjfuO>K=a(Cf9bJ+qt3 z8p|4sS3bdvAztV-npz-vpoRppD-y79fgN`x4K{!awaQ!&U3>*v8(r$ziCR6G;Vc zQo%dPn7DG9HG&5wB^4Fv)zzY2tYKn?A=3Db;zpi^?M7^A4#sDQdcLN*!4UWRM@k$> zgc}q&Cg_u9CCO3~V~{6=5Zw7zDMO`iEkLtGWRR`kSsE@T09G(fgTz`=5fQP~gr@sDLbk-_3w#{RMI7`&7 zBvd7|MP|ZB-I-|OTbZxBulu_r z_4?{f3)cos-nEN1ET}gIefPm}{n#<~_lJ&+ezQLtJ=z#Ca^Sa++fUZdhscIQVTDm+ z;kqcc^IoEtIEk$%zYg+_9Ihl3f@03J9l)66a42P%NZZQumxE8sAwUIsEIAcI&+ zfBq={%|F3k63}^>gP6x|+j60z0q;f2+ijQ{lB&#UF0l!WypaTU(7F|^WkX<0qS*w| z55g)-$DCw~95w>o-T;gy*^;m?O))r5;v~o)*>(>bI5`x$$F>EYTNuMOj~C$tJdS^S zS2q*%EFJ?$K}tBnnA993lR)4~whvZqT{AcT+}2I_L#(=L*&DN7Jw3Ejhh%9)?)jhj!j`R za~D4U#NMg>9#}r1Cgm^lPBP&3-OU#ng{Z_R|cOV%&mcy#+d>77?Q#$W&f(GnMyP8Tf4RaEVX>j3uFRiR3V)hy+ysmzPK&k!bBIG|ja0!VOiJ~lMb%F6g-Mpa_JH^E3v0uo`fA7d4F7z) zIAE==U)12}h_N)(*Ecx%fuO4s-oAjV({~u_Ai=LW4ggDnzdcFQ0?JDa5AU<2yllAi zy#&$WC6VkCb9p%!(KPL_TrLy5!{JPdDOgTsCB^{0$szZqG*{H)ak2>6Z{1Rj8BJ6C~CDa}~hN7;aFXc0O;4N=;fPz08;5m@5i ziEsIL{96hgwXq}6Rk7a)q(j8U3M5BdJeKT4jE#*L2EIDjP!x?JRgK4|Z<1k9#V#-0 zBv()h9j#Doh@Zg5la6s3ErWlYB&3Tx6R>8`8rgcCm-W0muySs5YU6b z9-iPi{v*!@f*}Yi(U7#>f|gsrfWyuV zzW@6=R}8lY;_R1%+et$ZotX9t_94E*B+o8*H>wbDc*=l$J4%#9I6%^q*X`EV*EF(5 zEZK#;0n?8IquhQwp>9+Unt}WVtog;bfH(`SDq^|@2M}oj>qyR!;j(2===ysgP0%#a zk~iqmHKV6ANhFDgP{GsC#rBLa^E=|43vSC0{yD8WwT`)xuO7pX>EbCj z0bpnE+B;2-_iJaZQT{Zz4%tz|n_7`81?p9m|ifZNpOY2LQ2 z*~zw7Y@JnW{CGt#y={xwkFZ7OXrxJwG&xR}3=&W%kvyl6Ri?eoA0r+M;g4bYU~$tj zS$Rv1eN0XMoL^5fCQs7mEvlZwo-!j9>)ED;`nATvgZiF5C!cN2+h6eX$ozZ*f-vTi zdYh>pglUZa$tR3=&-kRcdD_Ou>nm&Lu*wyN{~GbObcgC08BBElB;)9q&#Hdgv~%^2 z^;@?Z2M+3M>l-$+^=1&_DOORvXr3`?l3rAlxj3)2VE>8_T3XD;>+4rGvIeu>a<**6 zat0{3h%KmI1{iTr900zh6}Lw4Re$^L9~s^rwrbyLM1joVbsZW#^5w&tH0klBCC`*R z^Hc+4W~c+`lp^&{HdL%%w0_a1xotH@Tg`7bz5DJJ#%om8&ZYrlZE{4FJ^Pt^D@Tno z=j#e1Ut7QW(otVNvdKM9EDi#{r%E;4da z3rYY@xgnv*r*jx80S&pKRZSO-vdI!|FO{y|V5S#xy^!(6$2s3($JW2L!@aC-3A`T&8#Gq! zp1X}5Wrq&oYunu2RgH$rt1qivT({J{^R*3cGQ@R*Nnrl=P~k*sLI`(ayRb)ogHzlj z6l^y+DZoLlD+~p$JE<&#PDPUa(h4N&B!?rd1Ww0vrzXydpIEiL>fqi5z<`>#~JpNFmqun z5f=~?X&jw3Bp+;5TpT$&nBm?2@BdxH!gW|N#p(ao!8fo zLXo&N#*3-4{ls^HJ0~xgI*Co9a6FtfK`R}Or5skPOV|VDwS4h%Lr~t&MID{3+s-l3 zkE_Q|yDvF7_&PAPz;&-ug=a3-DyJwz6a8zG7U(d`Gp)B*{y&pcqwc{rZ zzKb{OEiE6c*k7=}VEF@6fCSuv=?fNAvIVObtY#ZmuQr}_fBjwN$pJC?V~?@hUw!P= z$3A7RzG}dER1-u71^XY_{0N{ojC{yJf*}%jdv!mO%iyCjZ4onAO45_~%NLD|BFZd6 zU5YW|wnx~c$7eqL%DA0FSqhs`Q?jIFQ}xD0TbXhCgc;!;{xzHqCxHqf9c29bL>!_& z7q9t>#Yy|*M@CH_vD~nIw6k!-1eR@#AhBg-uTMWXX{&MG;j&LEpFRnRR3hDKTMI@_ zM?Mu@n>hZ#>6t8(J-BP42bz~2v&Q63$Oj-}Esnx|!tpiGF1gmt9NaiWFg2$rggM-2 zX>uYHis6ET#>%*o{Fgp;;~pGZkj~QC(Ea1yq2!%5ZySU?S(s2f#N==t|Lua!95k+c zd0mYwe|IDbAsq^)8js1g+kSu)BqtKZ1!GuZ!Tt9cybbUN6x*b1RVf>=nr8e=LRKt&Am7KttP~DM?F&vG2p-}FU}x!0mZE{a z0y+pCnED4ZCH0T#x0AVyBoiq#K2xfzTf#(zh_)9_*VFGC4;NmD5mcTWN)+2T2)>Yq zy=m_og}WZecxk$RY{LG#*D;U19%UCIrnHz#6Cc$r_{%5T7Ti|E-ZdhQeU zec!zF*O&fktS#nM@IZ2G~apy$t%;kLyig^3mVL6kMkbky1 z8j_tAZ=ADwmU{_Xz~&pa=R_51Raw{?xO`VG*j~9AxlV5$IPm712PThpu;R)&3ue`r zb$J!)p&DCRW7vjoU$D8dnVD559~kW{W^*cMEm%^6Rzb2=qRL85x>p*uy4Bk^%2rX$ zF?#ak(awlx;gf-98;X#k!3?vI%pA&zvzHbc-uZg%j{5DJ@Y%KTI2`;hR&B1_ zTv=bnN?GdEvg}FOlSbah#8pPAx5>&*@7mUOu+!_^JXZmQeN-eaDEtz+Nc@ai#Kxhxw(7?33w)iF4OAd_@m(VASU zPsLh+d7rat}dTRi8YyGAhNs4ca*Owf`7*4 zwYY0|iWmdLm

=q+oq7+tRRgr-9Vc(Lh=j6D4m!A>yC8%GnaP7{>EZ zX-pf@FJa{XJP#(u2LqqMU@wxK*gp@RI%Nz)Cil1@MXAUql8E#os&k%ZryhS}tU+!w z>9z16Hz-^mcBo!f4A~8e2ds3 z&cO2VMT!&rgg+8S7IJraDbK`0mQqOhIZ?*T#B+fQ(sxP4LH{J`Bc%*8f;>BtVQ{e! z?6*NAV;&_i^dFY)R`P{8C~r8&YP#5-_90GjzqEF28zgpiOJ6Iw)*QB5DSygpgG{yB zZk5V|mftjmV1|4Q4$mtp%5$Riygfy&4&Qi7>z+NWPTpM_oIu;KH$9OqtH`B%_d#Xi zu`OSI`oVV)B~VecE;QLvrv%j>=h`zIF8faA!5Dkq8bRA2Xw7wp0| zUi26%dOmDSx1!w>qVJ!gTE-uk^z!tVr?-?JVux7E)|Yp^yz9Wh7SEr4Jb@@APd9d1 zMbFnok0Zk7F)CK+=d(hWu^G=!+dgf3VawD*_npb+S1sZ_41SnL1mdRViczLztKEF3 z!Ib}`@_+&{5ft7b#Q~Tk6R%(tfJ=IS(rhouxu=P?orJU2_7X)O=+z1^A9<{4N?-DN zaSYpC5~(>AvQrsrm5OW#xf5s_i8M`jg6vbe806et>4vWU2lEDM1T$!UNMA}z^0FmF zMw(ngB#XBe?a6bT*Doel#v@(hm(K|ANF0XD7}#52DdbEM6XwW6EFlhYf!2`_IsGAr zvGa+ozam?R3$rCC!tFwC2Qrgvan%FD=*%{&x^Eb=P-5)1Ta*D|9a)jKK0^kC+42=> z!JCzHQQ5XNa5v3R4B*o!1RQRh)*&ul)~p~hEY13>QZ8uFw9K*bA{r46zR1YGilP8F_Xw6bMUB{ z4;CDs1S?3Q6;{|NA_2}?dW}b5wRPSHF;xI_I5h~`2B1DD1<8UKP{`$JzJZMTV4ClF zdxo74!5bpjhT)YM_%rYZ7~V(lV3~t%8|1dh1#d&%i4>h}cnJaTJMb8p^betuO{5zL z1o;jlv?E_qKrldh*U40Gw^d^tw}c^n3fsim%$gQ%s(^QIQ^nuJxOFA#N_NcKQNN>p z?Q@HEEZR}PuV+n0)7B=EYY4fL7H*E_2bpux#>%y`<$94cG#jQ+(IETWl3T^N3N(49 zqM~$RF*9J(pS5mb8`suvG}u{wuvtQ5yz5Y0-qhqoEVgMszaCxgnD<;sy;0%TE0$Nz zTTp@f#3sDn1S{EB)9wx~0vMMN3Z%mwvqYr8Lfm}?tb4Hfz}$UC>=eDBxNZiUei_US zx`G_fv*(vKR~vi2)645iYfEd5l`=~}7kXD>N5rI9LaEHfJoi!C%B8pj=uHj9}Wg(wmndeUV#b|UDAV)Y&Z zfRy$@;tUobDOdRinxhwthKBi)BZr3hXG3D%73QCBCPktaP@{Cg$kd|1Jw2_ql-0Ot z$udfp9|N957A(C3;!BBKy7ZDV+im`GmsvHI=OFiW*NVsS4-%vC_eJy zTTzdDBV(;_45D;|S^ACD*6fX>x}8hWbuh2E(~wM`(hKNhXc!NRyo zCB2kHNuPxO&1q73Gmx4u91RKw6Fm!rdXM2r)4zR-YcKF{#=9{dI{n*GhUar#sJ|7x z_M@5s_;x!RR{lV~@kX+K`1#j2yv^Xnee%!~hUbj_!2Ub8Wym^|tUtgMYbt+(`gv9M z6U;IGHQog*HpD^Eq8Ajf5&H`^&w*HC*y=ZLHh3#Ps5e(Xk0d7!`xe>Mv`28RX1x&u zoK5JoyBiRUV%38yvizpm2 z(`yYEB?A6Pd)Dw<1@@8ZPlS>dUZ6=L}CXP~r@~)LaVY#s)J) zo#8U3?Yby7y=LlzEGJec1TR@UoFsD4XG~Jq87{8}EK#Y!!h`-!ywnizg$~0Jm5P{Q zr-HsuJ)Au5ofDNWv)RHg7}T8y=LF!F;r7dI=pdSgO2fvhukr{I zF&schP6Qb_z)6U2Ai|0#Fgpvr1W9T~+DG!)KqOE>;pBorgdm(U5`tM-PLz^82;3`? zE_fROig4+E^3U$76@0Tz-CYxG})-B(dRFjKX-BUq$#7z9)MuHBw*zX$1g|K;fJT9{{6r9$S+^-e2tDf zpZ{-d2kQp+o$Ck7{@t@t{m%Dvu1oj-Cv9}T=l|mPN__^)g8TotAN*om=eoZ%*3NbQ zljHxbonLxRD!=R+o>7(s_E)R}`s#dN=i|=LtG(8ByuVbh^F4H|{?PS4D*I3Gy|k_W f%X4~$E_2;^J#ifP;CI~=<%5iE_!YyhznS - - - - -Created by FontForge 20120731 at Tue Jul 1 20:39:22 2014 - By P.J. Onori -Created by P.J. Onori with FontForge 2.0 (http://fontforge.sf.net) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/httpserver/static-content/open-iconic/font/fonts/open-iconic.ttf b/httpserver/static-content/open-iconic/font/fonts/open-iconic.ttf deleted file mode 100755 index fab604866cd5e55ef4525ea22e420c411f510b01..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28028 zcmdtKd3;;feJ6U)xmZaM3$bwn2@oW}1>CTclqiYRLQA$5Y6)oBGM7s&UL;16CB=~) zH%=W_6UVZgVeQ0|xQgR(6Hfyvk(0O_J9V>Qn%H&oG)e0>@i>{hWS-onNvmm7eN1S+ zzjH1~P?8h3Z~l59fphM;=bq(ve&@HJt1v}T9Lj@=s?4rmzvGs>e#M?e$-DSAY}wuu zPnxz(DGIB>^~Cf&le3EBXMcd}6Zj5KA3GXEIX;t*;HP5m?7n+mKJ@c`Tz^VYD(~Jm zd1MylPFz2T)UxmH5A7ZOe}4HVio)j=WvpiZ%%sNt@lV$&%8rY;pWcrG(tJLATS5ef5?>;m=`T3U~TdOF!ucQC(+%tJ%mOWkhLq)lj+7BL_yl3W< z|K$8OuAf04Cua{GIr?|bL{U+0Z%`D&^z7l8*&pAf{=TBzgX+qM@uk@--(Pw5FDd=Y zzv;PiF*WcaJFOVej)kLlWmcx_K_#l7Hdl-))s-Jiaq+Wt?>bHS=G)5KZ>d2Pj^cL) zspv_s6cktVJbfGVdn<57wHg$I5=3giAFkhi>*`hfDp#)t<$c^@rlkfMM*)4yKjpoZ zm;e7O&j~k_zvW&)&a7B2n1DOHt25zBxS|PHxb6pE|LkYEcj28n_7e#qH3-ZzD|Xba zuyCr&LatB>-zH{GA;V(qa?!?47iYCXp*YJ<^ZA9f8oR8`&1u?oZB#99!|V;=FIv_H zHB=}yp=sKjTsBRN!=aeIVp3RFXLZmQUKG&EInIE&niKmm!2v$!20ko9;D~#VS11nc$`+=KtG~yf>$N>ebwp;yRE`v zGH}Jv)#<|c{rH;oR1LoSw#IV{&!ba4$LBE(`n=!v1WX7n_@h>+xl&r**uQ0L1!}B7 zt%+QDbF_1>eooBQh?%++pHi_R?rNvaVp0_&7C-Jcx2Da0VHnH(`yji@Q4AK*~y%C}@R$UciWpw&Fz=BN&REs|Hb5 z;$@}9KzIq9aGHV#O5h8E}wr4JV`QcE{(tKyortc-Ac zv8~hc$>PQ3trZG48duddZHX0S*S59PQlWs6zK{7a+O3K5cJSm-tA>$kafivtXzwF&by768I+`}rql(K|3%uZ`sLDML~eis`agzI^b!&%^)q#exy z{uPQ>X;RvWcC-W=e9lS}(GIuYlzx?4YHksgUImQXzoMzdf+Q*$Kg_9fyOSJZs$*<<+E(%oGdnwYpO{(HB(_-7zv zf{W|>&!PC0imz2WsU5X!4}vIr{4C;UXb`h{hi!c4o#Kn{u+t~=S@!wOPZV$8Jb5y& z2B{D?Kb}81xtV=Fdw=ovEV7czOS)@RtV$L75Hy$i0P=${%0+O6L9*X{n_ULtT`Uma zcpe2nR-kN&c4Mx7aJ`5UC-`?oL-n;aHU{{!w7-%2v5+p0DI98!q+H=t!kzY;Lk8jw z9$!4Yk|kTp^6XKUi`{*~_MqmmFZ`|Dqdj=ZUUQlSi+|q{2y_IPLnLaD+1c-X(xDa4 z*gYOQJE*Z**8?vU0$$A%qWMuB6`;a#{Ho zt(sfqBHoMjtCFy>n+Y~b9K*m+LKs3S=}r*hvY}^>Jv{vG+rtlQg~72wVC>ju4rR7% z$sGF3*uqQggM&0jfww#&+H;~s;H}GHHxf>{6Grf~aLOFbL^J-3H)Hl@=HhJ6PkvH7 z8{f2PZf?^i$TM?l@X8ZUUAdwcfOZf$EZYxWC7`sT-KIvruTtPDUw=L zK&%PU2IwJhOkYnG7;3ptY2dV;w43plfJ`Z{ovO3g_gK62-G8vEK~3AYZ{eI3GQtww z@naTIz&YGdTO;7iFb!-NY#O#Y?0Lu^g&BK5+2eYB9kt&Chy zfn`Q4M6*FP82LQSjArinLqVwK=$geu>6<*q=jB~2_&j$6Ca}PZ|3b3InB*GPsR8WC zdaR*a?n&0fd}iig5CvB;D?tY9&>S72HQ@i#6f+u&|KzB3ZAsgz*zsapcJtE*H?CND z(=BR1jTz0wKd7>$x43E@tfF{qbN1lV&EbE1ts7D9GGDu?OG5h7FYwkgf$VxLUl*#P#m;wC zHy9Wj9BCPLIK2U%W3wr4q*}&xM$b{3ll^&h&^+u5hcn=JN7hh-m1 zUgY!Eg_o@Ci6@G-`&Hk0cZbvNW=`vi*luVYA0ZEs-s1)rt%np7R@|$dpbgX{mqGDrvr8pyH$VUJ#p{eOwmGZp&nc8YPIm z*Gqe^tGyMQPwYJa8z?`>2;_3sX zzCdyw-DiScxfm(eg1j!u3zB9pwPDrk6lbXw+0Ifwq8%#>vD54{>7}xcq{~ehO9(P< zALw#-N2Ix$ldJ~$!4UT~G4MeLq#}SSf<4y5q~rirF2v3jJ*|iQU?^1886#}I!lG_d zy_LnY6<*bzuBw=0M&@l~+a$}X0^=JH6Hh1O9908c; zM24g{$zMn|S**+aX1^KBA#1BaN`;`eysqH2ZYzW2g4@MeR3kJH8QJdA7^F_c%u#cc zmXKPcMWmFrIxV;^*H-~nwrliPJmz0iUom!V^aVD&sCQ=N^)>B~OnXf`8B7acfS?sM zmz3BmqjPhm|D_g7CAdXH6XO%~$OS3Oav@MHWMv=`v3~r7K+uWp8xx>F#1a-+V=~Qv zF`Fvw#f$dJO~t?4#4h8)Ub%1#ziJRv9mOb#dp8scdT}K`RcWVwm*fsJ=wJ=-+Y5Wh zGJU7C+glS}pWhtmVI_r!+kTVJ|0Z8Nt2IYPTY8;k8V}vL`9e!*w5``x2K!p@dCP@J zqnH~wX@C(UGlzwx3v(o{l^9}fkQ-uq0ZwKx(D*cab^n>pe(Nic3yZ&MI5y^bY@=#m zChiT)6$*16H3+kob7x;&O`PP)cwb`d*sjCS9UuZw1#tWlj0FyOKb%#EBWezp zhTw;O0^xfl3+sJ9S}43FdcO5a0lN@{qts`ip!YX)1!5)OjlKwvrS4OW{UP*~#rX;) zLrhdQof|3+jUA&&@p;+iP!1Gv*WqPju2dQ^X0J`?3GTQb93RXd05g{0xYX{I58ra< zxsHL3+B2+|0JqcwWX>adoK4B}{xgMZ`yyPBV^*P;I)DpR6~ul(>sW%pJYe>Rqpbslp0X^vu63MFpo-IU6@N$SCoJNeMx8o)D97z!m@tlv(mI$ z_AG!vnmwd~S*c6Nr=`uUyzkPujZ5P;`h{gy@;nS%@0}F40_I7`LvmCU{JmdUsjOGF zD6ZA^jT?rC1_x4ou{Mulf>DEz2bSiv6fL2=39bdS7w9i&4y4JXSQw%|!el_I9Z4Q$ zDG01&A!rFgAP3Afg8NXMc4GO(m%!D$adxC5fK3AAxq__%vqFqG8iev2JRu*qp@Q62 zfsQZ1C?)F0siXs&TJQ_8rz^0}Objx#D+!&*3+C6HBEhQw1xxi?E8e|SfZ(UwmBEXM z-nk+5LH4QfkP#RTmL(%kiReXDqq~HZ*U&u@<+Kk8UVSa)6Kpn4BkiDNptUIDJ=SY@ zkBcBzYMiV{WwxV*=RsldIPBMY8zuXlUxEGF<1E?hVZYXuO{sF?wJ0zat_j%kx*L8!tfj+p%JQRk~3}w^rf?yJY zV*aWYrv`*%%l5>JXW1UopyOI`2*sdC8Wo|OnqPt!t+O9|CrR+?>x$HS#99MhC8K(2 ztxNDSC)1fhPHLFk45>^sQo2`KrV{UaMSyb7V^>v+&%V1B#*MK-)2&Wo$pGuMh#??- z+z~K1Z#9v)+g`idzW#bVq1{gMoUr|qNgVcP>@oPGNQ;2&gN*d=zAY>uP$%G?qB$?& znJS(q+O69ljM647X$7?cVnO&T+z#}dTz3P!v*_0-o^!(wrnZ&|G}6Dq_LPY(g6PNI zDl5^)A=|6O>OzmUsWc9Nn`{cOo`#dH{)|vzg>p(T)qv(28GVPgfc0(R^Y45C`{3jk z>T)^vff3@4BL`@XVqJxtWK=AQ4deCDx>mdFRTV_l$&Uk@0RAA#w-SjGUnp%cc6wng zBttUz3)V#z9g-ypia;Rj1pHGUpea|MCNrcm2%6F;>`Bn~;(lO%I2D0PEi9;hV_O|{aD zG1j=HZ0Bz@2u7Al4yhUFui#VCE=icjV$D@;{Qkf@_DBwYjSE z@S!s+2@6-AIdr(Qs<<)W9Xp22I@sW81Nda{lRBinMQvcmvc4D} zLItj=PwpZ>n%0P559kRR$zm|JUk0@#-)zO#%47#`7_zwdl2=Xt!c9Pe*D}}|AjerQ zSP+{a>434-Yiz}?7I-fQ38W)|0rEo`T{eJzko;$_w15_n{Aa|Ner3bK;auwcn7 zxeVbVCyG*_N#y3{=jP@k*ikeVv6rAH&cn8{Xj_C90qGUeiw7c17z>i|lF2F>$|NGG zFl^?G=caFSZhrNtCbr30Jnv@h&bMy;*x_A!?!5cO^i{?EZD*nOm1baR{Lbv5ag7`~ zoA1lsvs+u;qCND-)US|#M873|N!As}KR)pK63>MEvy5i~s2TlB_7w8{(;Aj&1IcNN zAM~-r$Nn{PC0fHWl|TF5vZ0hKf0u0d-g2pwEq|L_`u^ogj2cV2#AB?2SJ*2o0=ED* zL{5Nvli2|hJ;Dug8es@&;u^Geaw7soNFmp*NZ3jGRS(Qa0oVHAJ**PA7H>2(F}oq$ zOy-CoQ%U@a#>sm~*h2PD$fRlZM11<@b$u;XtI5A**Td^JeEhZzE|+R+?;gEHdq^0b z3Ki820dJ#Sa9chfO08aR_L^Y{2RpcEEkB)iT#W{No=m1waKkbWTZrM=(#$fcZch%=s7o$M7zP?Z2(a; zB$=R);Sl8umil$6&d!xy{U7 zTUQUS8Qxr6ke7R>^aAXYC7e;gu_0d=q+9}5vm3<^{F*cC(ti4K+YnD2cX6hz4P z!uKNNd&!H<2{pmgL?(!72E_9eo zSG~XB4RmEhJ~vdTc1F5Iz6)NG+)&>wj$`oJ3_5Pd}~f^(Nh*@hrj7 z1gjn9B;`XFAPDnS$e(eAGO&FCD06e{GT<^xUOjOsFK*CArCIO>xBjqf3eVHCV)IgC z)Cd(6FN(%!EKBsu49#*U_V2b0(dBldRNYQLU(#_1KMyUGDW*?jv_%{gXX~s6RWmv zu4+v?2YNR>)Xx2Z#@@bq#+n*kRaHjMTE^5$lUwb7HQaAh(-zfgc3OR~RF&doVs1y+ zYOwn~7HDPFBkNgnMPpjER{0JDeIo;&8ne5-(Gd%^RaRHkR(Sm;V`Y`On!E3*XtG(D zN%d5jDt&6Cd~JwZQ#_fJ-TjR0kx*c~A^yrF#gUQwv1DUFM*E(|dMFi}xyUNZGLT0Id4ixx*U!xSYmhON8Q9@Isb_MOI zQfk3JD!$fO=e3)Nzajpi%y{b(9$e{YDJi0EKIaBSdfpp=|29`w<6gMa%?EXb(p|hj z1d45PlmE8(mfL+nS0HtI1^h{XUeyu3f_MXOgizX{x1_`sI)|1btjHi?WVtC_kpmw- zwit{nag?!sX^y-0lUF8{0{=MR_U%(oxug#5u4*_^P~05cHzr zYmrc$uR`El99|uAB#`Sm5{0vh#o}=cSo9X ziN3x>U{y!QDt1I90Tl4u>VbjPC!RT>C)$dwE0VpvN%|ry;iJc6k^JP7G_m9uGYQ5i z42LNMx?n_*M~Dds3jtGw%WxJZM4&fb^Xc-Z&@90ZE#n}xH|H^K?F2PgiU8cPzG*X;t<{~s@Ewc#f%^JAcM5Di|8`8 zt)i0RFNzmsgatb-<1vb}%dhXOu5I)p%B$7pyVM&>MF{e|PB~fa2F@KDSj3l;*s{#GqTM7HF%D=1OirTVkeS`pN&nEGQGf zH<%OJD%}g%OE8$*N;K~M+ek?Ek@QZ=K{797A#g_8M^L@QFL6qlBUVX~c4TH2DRftS z1b-$Ond~tXaYJ&gcXf4ltPN6Z17uhyqG1h+MJQWB&(EN5FpJ-r7h+IAP&slo!ADEf z^Tt`kgNZ7TUv8XYs6w97>53j_Vr6P8kqpd!*b?5bt9S~%0;F7}5P?W(7@-wX9l%d=znfr%CJ4UDvf z0&J@Ey?1+whJ!}P_Nt|w7QO*-LIrHK39dq6`Js5_95n~<#OEk<95W@!_{x=n7RMK2 zd8s`CD?jlZ8z-IvKWGYV0Z@q$6U`BC@J7k43WpDZLn-k5GBQOQAcsyg#4r*Ipio9c zP+$$N7F9%~gOi2PZd0A$HRN;fm=U9+Z&pMvM508voY3C|NIgC}UlXe^X}0PW9j;EB zW;EY2{`hNb&z+~i*UqTH*B;-s)r8xfu8tMeHqBsd#}mbSPv42dG;f?)T7UHI6#fpc zOW2-;t-#I^I0!>aiG{+{EbLCg0>xx-lp4&R%$|PWU@&Owy#L-OvL|mAf~roRAr4^Y z_z~mXO}wZx+En9mn8_apw4m8}L#<#dTp$Ta(Oj@2*=@;o21_yny8b=XdlV?<*`^&veDfVWp&KJeGyLt_=znKkl`P~Kc#4@ z499g_ddY_YQ55{%%4XPZk^pu>Y4Mg>6C}e||^>sa*Z2KnZ52N|HnG0$F z`G&|dLRS0Ictm~a3n*_t;UX(CV)#q#-_~f>Ap_1oY%e$hAj8a(^$`M0)JOvzCB)@7lNe+IIY1- zo=lq;gL3r412BA%8V3g(5H3WXE?B&%CiB@X!h+g;(Ew(SARSWTIs%W~6~~^P9c+)^ z^_Yjx8wT4Ah*(CPG7k;>8HMV^Nv9KvU;N;6)priIw-4S~{oKL04BsKRE&4jp z09c=gfI(1c!91En)k2qA3?+ukYH6&bZ%DawSqSkJ5R`@I5i5=O1kY9(I9#+r45iUP zB*og3@Clru@mxKxR$w12o=IT3g<2?Bpk~bJyY$?eRc&v4^tnq<^7&P3p1b5b@#LlF zKKcgmhVVezd;C~u8|f(wVMmD+h#?X>0T}j1$-^FId&mw4vM2uWBWPghg3?lZ0&fCn z&neo2W=)zNoR=wsdFjG6WPs_B;xzpA#sBsDdd}d?wo2 zxy~oXeDy!@moVoT`iN2=iZp{$KdYD@q7d+772=l>3u#7Jq#sw@4>KUdK*s*)*};K< zD=qs*TPD`sYBt+z%vTy%Ah5Hscqz^j$umjo(RKH4{n;~HnGa{`Ag*0*8Qs@1xo!{K z>rTr*H*RZ0%vka7lBW~Nr0s*K`pnO^GN+^oa?hy3My}H&3Nk`qUpOUBgK5&b3{E6+ z1b$sN1C6!8lia9u5RHvA)p}i3A|8Yh5rQ&ArxZ2i&@$Pmg~)GS)XhrwQ{d@{8!^!554>LAvO5K>rXuKdhv6bW;n7<)3zPK z9EB}PoDri~XFAj55uweCwy3afX9&4U5x#ErIu1m|-LNbCo{*2!V9DHo01S3noRFa4 zmL)qd+1Y()yBa6JRO!b-=tdf_B0aA;%39@dFt(?zrud^7*7o2FuRZ?ZY33~M`@4&2 zoCQ&fM_Bv5JKe87^!RJrnDehLUF^7Ty>8dJ`m~_0!iPw9on>ct#GZDUqb^B=WcclE zLQ5i36wFmZR>(p~#lDuOb@Vej1qc+vdV-@T(1@19Uc_KX*q1^@T3xM+_Gpm*MLTjc z2(jGH%jq^$TTovd-6P$T4r}T*LK2IFu@GcS@Ed6>R7H$mjpV0v3QWbukrt99M3;=z zIfCS4%8*R`;85Eh$RNqC)}hGI=xfEdUIQvYJY~w}rcL+JVc)@h;ik<^eW%ABf9X5yRtP?g%n=#HJ^ukG6EmyxUY=0CxJ|y&w}&`CR3b!1<_R2-3!m}wu(y%k+T+m zZY>n7tj>zrP}_RkjV>F=*m{c3SoFD4e1=87T0&n67J{Z=6Q)_163G85zB0H_ z(Au8}+P-+khxyz%%_9z{L=g$8nz%U7zo^<6@lATSdmFMx z=dG$^7oYz?@vE($YK=UsHGF;dO)NW7{HKxJpJ>gdK2|UKk!QvFLEoBmTqB7Jhkz08 z;EiX7I1r9d8V5om&}x$?k_S_^Uem`#Y=r0kg^X z3srSmOE<*@&%MXpYait~Q35z~@=dZ|1J0yBSuS+P9D>(@7K@?U4HT;ads=450zws` zlRP+siGytb_CG(cX0WrP*tznTr1iQwGKO|lpKDWheV}UV-mO)E z`u?^Qh11sQ;s<08&r4-__E|l6m~NEfcoSQzI+C`&Rjc}J%>y@!_+c9fCBocXAf``O z((HmO!?LTgy-zes*t$ul2_w{1@^hTkF~i86N+8%3NGkltgNSp$Vf?4QZ1NQfwcWwz zoJS=im`4^#ef% z$Fjp-9N{ieN`jAgn#Q)oYbum#!N+`Vd!;zz=!zSB)!2%>C5-TE3Nu5Bt$3ET|L`M) zXNrIO?CUI2`11W@$1sSG{IK|=v(GZmGg|S@*YE$bb_|;Hk{nP0nn*DTz};Yj-$Q{( zz+HFTK<#&Pvt}$20%^zDIukuy*M=p+L9mCer!h%P-&e-=Dcd zd-&&%Ja*|rBpHlgj|u+pQLG^Fgs0ZF-fP0 zO@ev6y&&wQSBe*fbS*A;q+Og71>FE3$v#kx^PGr*cUK6y0jdBVRWixKEt3ur`eK8^ zZLsMlAoyCWsW{XWi*bq`Tz|LI_4ZRB*-*~!M`06>G@)GEH8S_T(q2FxHq1xZ-*MKR z+Dd|UN{^ZLE``^G0$t{$BoUA^*&jm(}czG*v{jdvpQ*XlUZ*!1?F zZ|g~=dbWN0t)|8!3%Btt_g#2mV@s1UYkEa`}7TW_;u$D?h#yiIX# zP2f=Z$+;+Ci{KMi885SW&_!riG61xao5WJRr(K1GuPAc@k!@df< z3%=;Jt5;-`y)a9{Dk)=z;fpSFUJ1>r6c=1l4NAn|+VawM=|20g5UYPIez{8|#h;6i zC25S&gR~dEU0y?0N4N?VZVr2W9e@7{jA2)adP41?rJgqjDNB!`AOM`^3=%+y;A7fL%L+^HAY0{O1?gW7mBC+sS zg;MolS0cwW+7k1NNA#tF?!UXJZYP>`?JAVE^eRRW-GGoGzksjj8MI7=*yAdty{o?6`3 z+}LcNSuA^;WQ5+|)84wapH#SqzEiC_i_dx- zjS+`+ZbKP<$(S&knbTN=Jsm2i;1j}%F5-)EDifq!+RugY{F<|e4p2bM$0=euDO_O5 zUY1OQ1=9XaVGS2k!Z^$YvIkILEwt;w&k1)u2#!Yf1CmC_a7MOz8LYwfET&k2()xj4 z5=L7tc&c$;P_VkiJ_u1FDHR+_y#E5?T72IV*dGgPN!2A0hgj9vF$yy;*F&)9Dj_9? zF(>TxNK2r`h0P-Ps8n!ivxM}6<&-y;<;mYghm~Kn@=1{te=HN>_rXc)Vk1s5{}cf@ zGA)oMOnNY!AB6u)JW|pdk|;Z&6@f?g#G)-t4RtzCq4VYRZU-o97>h_T4w({DhDe6_ zrx5eBEUma;E$}J)6yKsBF{%Pa3qokUP$7RY%2)6j6?`@8ZYb@VMptxJ9x2AC(?r0D z-dRC!odBFd4PGZ10{|y7UErMqh!>&}EQeJ&+(-^8dK4Ji1iVaXO0NhL$H6hxHaHA#NfZiL> z0@~PuBecS%LHj)lr5vv)0Zo9xI!q@FGDCDoBSNoIAmYF_4-Y>~azSfk>LVYSQkx@n zHEVY6TvJn58|vr`*3ukF2(GC8qc_ghS~ZjFu20P^kE00*-yN+t;&?1_ zAL@M@ukB`etEERI*cM*gv-V3slWmsB; z*hOEK8nYN!M5Px6s4QY&04kWm!Y=nVt96?jFEJqLh)Ba?`@hECw1N}Yp?$x*s-k4u z6PkN8U5%Hfkq#gA>FyeK{EaWB9{u`P9!q^OcWF8`x_jrw^b5KcbkErC-DCF@FAnYO z>Dl?qlKvxLr;?wGBIPU>8ta5DgI>qxO$ZW7=0lSEVL>Kafuc(iJQ{RN7ADmv_I30Y z-)_h?1h8-1PZVDgasV_c+(bmm88%cvxwm2AvEJ{#OL$FRY15;&?SiL5a(5$gS(n{$yiNQiv|mJiq2XmbB6LtV%ZnFb z>e8>l6tQsyO~HCE`Z%MYC3qJ>TO<6Ou-m=2pHm1lh?%FL47`gAx(K)w!rD>^;rFx{ z_bvK84O?!7-}5`fZ*JRQcd04CA_RuK_IPd^Vor1)=su$*hNlmJHLdVl)RFQ1-KbT< znX)lb3|hy(c8qiw_kD~_gd31|_P38LE#Gy(YM<(?_)+Q($BO@@R07lRS@wQUc^A=0St)(r{b2RV>%P}q%j>+K{O@Y# zy~au9*WJSyMVX%7unzF6{JHXc`FO$4m(BOR>Xko3d7L#{_8gVH-)FCF>;L36jbRzA z%hwZm{o{l8$){wMTa^>algc-hpTqZfGn-lxVE@EzyqRbDX0Gx3_$T>`U}Med z4)vH?P=9H#8Fm>SFnrPQKMn61W5yxl9^=!-ADV)uoav`#pE+m#l=)}o%NCQR#?oOq zVVSeMX!*Y7rqtF@l3^cDs7b=m7|sWD<7`BVym{@Y&&Rs z#&)sFR5elcVAa!A->UitdyD;;{fzwu`w#6!N7}L3vDfi2$1{$-f2db8eJy$^Z|K7%jf zyV-Zx_oT1jd)MFWf3n6`^JL8%wQaR4YA0$xTKmP?AJi7>R@CjU`)b|y>)xunTyLvy zsb5jQqh70jp#JIlUo|KVS#Zz?8_qWr19br{@QJ`nfxm5RZd~1XTjQr1Uv2zlQ*+a? zrf&v^f+vD!gD(ev82nYJF?3t#Oz2yopElPu4>wOVpKAVU^Sj}i@agcY;h(nHTQ;`L zwmjYPot7)D$=3T?pKg6KVu-AdJQ?}xNHIDTor<1_J|F#WZ8dG{+h*HdZKuFn;+sEJ z_9GI3K3x2g4>MhPx5z87i~Y$W9UfL5*7FRWr~j(wDGKBN)$^*-!Ups_PD8RIdfuqm z*=O`T-k!r=g*3$sBoz}z$vlGv;=ky54r|8$t>;x`RQZ*jHz?KY4n1#F8rc1M-lX{0 z7nKp^Fy8h&sT{?xrUaEK)H#6sar_>|%!4>ja|q=}MS2+T z2Ae@y9QAvVwxPyR{LLx@uvPUad-b}M%DUak5tMeLg&EX?GCp#6X7cEa7M%J}aBKI* z?%4w(UQ9batSpXD>?kQfc>*z1;_Aj-rj5 zlxfismg1)ALkE!@&`T&)4xsD+(%&}n0gQg9m>13SZUK=#lu>z~(gnL)7iQUud=d>U z8`wZ_=fR@~j@~_^^#uoleO;NZcyAwSUEiFtSW!`Sp^L)+#sM*M>ZDu$261!d@R0+D z4hH+W@rUa}fanZH*R_0Nhh}FEc9mu)u~E7D5XO0<&reZ^Q^1Tfl^O6xCll;d7Q8X8 zf>kPOm34s524K!j%*Lufn;guEXr*fAW*+8cKG=b3SS_n#^$Y>PA9Iw!Sf-uimhgA*f1Mm zYuP%so^4>G>?XDmFD$;9-NH7rEo>{>#>Uuowu9|tyVwU{IODvpM#M>`C?% z`!xFudz$?R_F48h_6++Yc9wmfJUnc=!^5d1n*1oz7+3E^S%u4%ksW{ z-Z#nnrg+~p@6&kS4DZ{^$5T9>=J5=VXL-Dz$0vDwipQsUT;uT> z9^cCoy*$weuQE?0cp}LYDV|94M207_Jkie+lRPoS6Vp7Q@x%;I?B&T`p6uhvI8P>c zGRc!E1YPlDh9|Q;+0T=cJUPXa(>$s1f@<6PbJ`~=BX4XgXW~4Q;F%=PqgQ9Fd}@kMP4g*@PtEYDy?nZtPxtZZ zIG;}N=_H>{@#!?5&hY6hpYG?=lYDxLPfzn{jZe?;>AhU*w`~4l|1WJN*uYz)E%B3gjC&tIe>+`I0d_0_2w&rHW$Gh@sEVwS1 zH?&S-K*o`+xx6tvoHvDsG5qm7o9N0LVquIcsGT!T4F~Ct>^xsFl2<0y<<*W5N=JgH zf~U~(xn5)IscpH5t@V>*@|#un=G|;W9iN26)56 zlXFPd2MoSSKc1O1cJf5ZDb?O3z_inc)p6R#&A`I ztFF8Q%{T=}f`Gs@hMl*MOaxC&1oL(Ptt;=0ZQ7ALXVBJ;x8$p4!Y8`&uGpq+xlP+; zVSNbYZc$zxJEu5CcIM7G93y!)Ih=QN5`qG4htJvQrwTuL=EF*;ty^>F2x|eX;Zs;# z>b4^k#$%;?y}VD40PpGUIA*c|aRt$vF2nIrF6a%5O4FjRHJr-Oc@Vq02`8y|qBUpq9 zTC_=|`F298&RD*qGv9&j5(B1g07~6(zl0~VVWLyNwFdB|E8n%a2F#a_b>x}1S3tSD z94gCi^~8cHG0tApVe78nuAl-p92S);zOM>eyLKp?J=ep$m`NYzje*|qkqKb!WVS0G zk9GT3bmbGjt12*T8r73n3dPqN><(_Aoe2=$bn4WG@CHzV9OyOZ9ky$NAyN|kr$9n{ zz<&ITDtYTj=gg_@a4@*y6xvEJ-41rkHu46viCV$@1a0Qk+j3vwK{Z(a6}%9?P=mY~HN@&3D2JDSMB;$3hqQyx(+$sivU$77&VM~1hOELt5AbK}O zbQpwJ05n-qoVQ^227~Lv8>ll{t$qPAnt%>bWk;?%xB^U%Mywa2u_ch3T5)v~ZY{D^ zxlq?5*F;!f8H}+jKcJ6bq_i{>#CNX+Txlr>W8q*oL2W&#?uzm5bDhkCjkjX47^}Hd zymGNv)Gj@`tjPYLas1& zMK?By9OD`g3lQiEz|xCYmQXO-Y| zQ;g6tKMJsJjGb4MHOOp2hEe9`*m)*OZb3$rY^FNHxV44qP-ZLDq0Ba_LzywEGla}` zszaF_REIJ3CWBKf2?R|71YVQ|0s(nD@ zsOp`ueE(wAyXZnxy<6m{>OCSyRS(AU1B+D;(S@iwD{@rzgCa*&568X&|7J-t8t%+n zX7Xyw))T~Px)cc5g)s;q?2{nMQly?erx=GJFm%Y&vMl`uxQA7g=s8tcd#;5&vJJxG tBe`>`w)R|vu3oY{2>a6NN2Vb$p$g>T@pFo;#)kMsZl diff --git a/httpserver/static-content/open-iconic/font/fonts/open-iconic.woff b/httpserver/static-content/open-iconic/font/fonts/open-iconic.woff deleted file mode 100755 index f9309988aeab3868040d3b322658902098eba27f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14984 zcmZ8|b8seK(C!=Cwr#($lZ~BhY}>Y-jcwc5*vZBlYh&9^ZhqhW{ZvpRobEY2 zRim2jc2|&)0Du6#g(m`l^xtUf0|3Fv_;2t37YPYfIRF6U=Qof04SefskYWWDCf0Ax zvBgA?Sg zQ{3X4{N{ANb;56uL&kuESlGIFd~-hEx-kF%7M7U{z_qbA{?BgvJGPPkQ1m-q%+}E3 zdtHw2HU7t!7$h5R$XB`1U|?VZ2x4oEo(?{~<9cW^U`%1|L<`O49o%ya3Cchk?TQjvHN{6At8vTKtqH+gT24Lz@);yzA(}YXmPMtu?=J) zB`AsehXP=+al-fk06b49&+lmeAMwbpQMYtnkU%E5*g+%ehk}td81f)!!euyQg~T*2 z)@9npKco9a9KNs1`!r1D7wjizEmb+j<)@`LL%3o_S^DOxFhSl--hj14 zM#H5aHC`i!yXJ}d7a=RP@L93co8&-xe2dITtXa!y%MBkDB~oaSX8=|B+}p%5@uonM zn_)dskE5dgxwy$B7UDtO_s#N{dQ@IiYRc?**2_dj%d{C+ob@a*k&~f+QCmvu@MvPv zXAzzv=m(mV@f35IWRg%#BWNS#Yb*+XqhW64orn;jVCARAp6(CT+dJl6*AU;? zM*P*yjc8Zknkp&+s)x#G((ur2&&kDr+QHf9@3~dEGc~r>L7*Gzy1Zi26w8WWema4O9nUHF1Ay`VkG|KN;jIkW!y|Iqm z_{%A18!12g;hLL=>v$cmr4i55J7qcYXU=B~yAkp<@s~C6tv|V{8@vThN7>Ar*+kUT zG#R!Mo!W$4Nb=yBdJDs4I&6_7L__a`awb5B)C3Ey=!p>9V1OES1_-UBB15l>gAY6! zgAcgD1lD&~n=am~Xzs0?{DhP>B#)UnBu6*&eKAo@JpMbD(YyVmvxqj z&@&kK=UwrH$rMA@KCPr0_vdj`DwkaL#P-jJHm=bJ?i!1 z8}!q?ktnS3m!tlo1#^A;Kj@_YSVeWK>j|c&ToS7G_GF@PG48OmO z9f5EK30J^t+iqJy*#ApP50`b1Itps9p(Y}?<(r0xM8Llb@Vv_bC)p7#QQo3mf&A%)o+*0URgNCG za4$QHzx$SKgZ`gRt#R0@*1!twSlSHhsoh;QsLMm8r|!LTG;ZrmyWdoHUi$My zm|}07P^J|LaHp^NgRiGf&NR(l5NXAon_%#8@W<{J!y{jdzW4$&DU}1qKxKQX)8XSL z?2mV_=`AIG5HC-7@$7A6{NO&-ydr#n74Uj&pF-Z$8y{E$zC4yusOM~M_{>Se`eA&?^+`>z6+^^e z-9zRTW5i&l^d`h>3TNz)Nke3o@P4#IaDYO_;5OYM^K&LQe2?L@Z-9NqAh8)@a0oa2 zBgZE0*v2lzCWIB9Dg+PnN60WgJt9X9;>y;|Kz%P)#Ht|n&;k+1CZVGLZfL=$4YG(l)XI zh)7x3yd;LHCXIWu%}triolkzfz}&Mv;H7!jBuw@gw*s$C$eu=Qa`1sc z5B}ui$H!Ce4T7GYUs-(D)QtlbRq-=L`#jXs?`*z*GJpGBAOxgH)eXYY$Hg~AG4DOq z=I=cl`sYCiMJzXE)U-~?69#ZqtZ&+AQf<3#MTmlm%g{%Umm_j2vh91ay zqv1Eg^xKZrziV{;&zZQAcXh9BJ$2;6V~=dAB!U$EAp{B=FqE%)N^YkP%oiRBdy5yc}^m({p@zFIc>%w~m)m9mf}!-OfW5B#m6e+P`6X=P7dmh0oT$%qeiyr_JA?e>=;4&-SO=&B8d&53>ph7P{!2UjA~-<}+y zPd{`k0wz%CSu^`360$||g)I7cO(uA+j+wedG2^l`$+y$zR;9Uh)P|Z7YDCGkDr?Emz*2pk z=&{N3d}iyDCb5)=dbZCriD^F425+7nvY$^RexMM&Y@~fu_8dox`Rv=J+(Qc9 zWn-qPasT@eA02E~FvN~G5E{6FE|YOYXW<6Lr~;=-HsGPY*-BMa)A~nN0YuSZvNR`; z?3GZSJ9gTT=B1hQ>?q8Z$4Lc+-+cJDeA2{i2Y;$GDd|}~D%QeStOPVz3q!BG*3_3< zsN9j}+#54rC}E;sx!5Odt+_wQl@-R;EOL%rm7PhG84}(HzEmEj=aMrK zIbG|+mgHB(oqX}A(s99tu1a)pigk_tAoUw~m?aQ&b3GAeI>XD0@EuIa$5l*WS1n*g zVJzBC98rNH+I+s$#v@W|d9@)RcYCycT4=Se+q`R8J-~u{;9-d3WS5+P6N)5m6Yiaf zW5r-x?=Ll_GwMmLqv7bF{L`WyIobWu>Q~t8YF*XhO1GVnn(*7@JyIqu1`U@KGOlS7 zDkIuCSkaEPKx|W0eg3B=i?9iL1FUT5wishps-be9I&>pL2hh8|-SBPq^WaW#5tOE~ zT}eCEtSL~gqcqjWVd7I9gOLIKbVX?4W{OO%%C0HvcP#h>_@M-fc}T%}R9KJL<`U9V zXu1u!HS7X0Ez~@YB)L|YW@u9W5-|tHX@2Vd^Q|Yoj6j=D&m1~FnIk%im7$;J?kgN=T59<}6@^cfW2XSeDIy;+ z;ETOlaWdwo5OPoV_ct=W{O6{#XMgMJ$9oeE-~m`CjpUZsw{hJ#0gvO&c?Cy}%w9Ms zF1qLs5n#X6OVn!u32_b_qY`#EKw4CB&te~7XZY(jWdCXUQ92kuUn~8)qF)SI2<%X% z$*37c99~#|tO)1lveW3!TBbb0&BE?sJ2VN2b`;e?d02KJA-GD}T=1K%plNHtYUYXp zgJD%O29qwCKm_~M0K>`K8^SP{D*2gCTZu`SM9S}-Ykw9zDoswD2oi?2TS?0j|YT&|8hjXaQoPL@9w`)i%-M<8&28g z`*F!&y{zlqjf@rLrt~FRSN5BK<&28)W4m>{vp08~u*1zMt6=`$Tiv_$EYw^6mW-W< zt8zy&d5h9t;u3Jj2lY=`hj8Cq$z7Jwz83FVg8EUT_;y_|+qcUF=C!0ITJ*U22Lx;V! zcKoPS=n8#~`Z=P6J*6*B$?-V%RjyUCCvVVwdl4E(WA=YtevNLvY$%)5Bc}Fw#;j-I z0#n6dHjW;Da&pE??)2+d3EbXdopfMeK@6A7^s%KeI88UNE8A_UQz9pRg$VLmUKJVl z4I&pPU<9*3OS$nt9-xj5K$8UbcV(lbl*jMiig1b^fo^TkNqIjEk~>Q^*t@Y56IUj>ezm7Kz-yTs!n(QG%R6u)`W@o3~fE4rr$BH|lu!66Zt>E+mol2P_*O ziCJ0f=UY}ApdzPxn7#+JwBo&4_`u(lc$Y5=bBVwn<&r;>yAaRJ-31VEoTj>*61yyd zp3YVTLPv?QW5862ulNZ1OgO37-b6gtqu(;CiQAmQ# zCr+Ycyg+WEcZ!?X&fSUptp-8 zOKi8O!M8Q-*Qu1ps0AggluG*V^1Nk{%4)ki%nw(VY+snRW|#=(2QwJB9_$3%HZg&v zGierEtLuJ=$|~f4f4fwK5=?TPAjUyj8Yew=i=kkkgavOh6g$X3)xPOz)zymuI+`8M zw>dd|>IZAe!R{&|(y{JJk1V~blgfVPyc@hkWl%sl(2&%1_ zBayVylj>~>f=ABwi~c<+Iw4?r-Y>*Ha5S^04!G0F`%{@_*=~3GPH#N7wy(VW#9K~% z^A}g?O}_Q?lKt*@WTk_H-hSSv3-$^pR130pW(KZ(yEogRXYxqJ=3(mI^u9}QZvQ-a z((-M|R_NJHj9Leb)GgW74j^HIe+xHZ9kE0~@bpOQ{p$rbO7MWSD}JS|^sjCkYlGuC zUORP_Sk^=&Xl>}jo)cc3(U8>A$EKMhU3Op5&q?!5bIRWKQy#{mHJe~z zpD_@@wKexPN7*mrUJtXFETM6Et`^w$d}C!Oti(ItQxZ<}ac+wqpcwP31>V3Xy^R=>z5USMBZKK+o&=70h3Nk7J|rhq`+&2=kGz zbKt(1>sMjxt*%JtH0X1QUjjrO+!WGqJ~>^oI7Jo_J)Kc&*z0~air!w9jp!g4?wfgq zJL+up-MtWP-#IVzI~_ZIvZ7?AAS3Z;mPEnwP_cT! z*JJkw8oBTf-J3$s=O1WSr-_ar>?Lq(5SfWB(V-~fojAhaKW3_-Gv)6Cs%N6kHOpSA zcS_*;`P_me1{t2on+Vr1a$ReDFnK`uz3Z3nG7l^pUjIFTxC`QjIs zw*4v<4CwC+ww4{v+O69!bR4?vCk|s{UsX-Jfap8;>_AXh$l|f<;E74Cz!jC7G9IXy zRd53A1wnR`fLa1lq+bZjJc+3|#A70PRV!DqsMBI+{Y`^Fjxpas$8>UHzBCi7^C*i6 zK(hW0jN5kPJk|E<^L0~z;qgZas_$AoR&%@#wjhOvWDm=21DL3NucshN z&4&0NC>nxBdAUC#X!+LbzQ^kjjbhE1k1OVX7~$`<-c{$9+pA7>tr~|B)r7k3PQii)1bP3cLR~PA43g zv4&593)87tEg~Q62W|9|3QnF4m?e!IAcZS5Ibl^1YcsARB`ADY4@045znu~7a01Rh z>+l$JuFC|4z7hK3+kCD|DCv!`W2+C<_BhK-N=Y> zl~TeiuMqwCt^g2?J(W(R_x%hzZ2vT01(hBOkf{W6GNbOatvp{|VWfZ@Gaj%s85B1e z{1-eVWEKKhhEWhGjoh&iS!ze1fT3o7ow#1s4uhlLS<=;VminN4iuf0PSxB_tM4{Q*zUBpS#fqtC8M||{+PW- z5(wRsj(WEBgf#w`o)_kNV2gkk)eH-#tUQ@!r1^IZh&ZD0`?tbafwU1|CVhznf zNcNSz+~+>zhi)M#9b%<-D2l7HP?UKitR+ZD(RSuH;DtL1{iZh<2ucun!sawL z`=q-fJdKD;G+Bv51liqQ+tU(A>7MJhhOnA&5qu5Rl=-K7=a^Bc5AfVym}bjN8}a31 zSC+FQ2;YpbwsQh&KyheTK+B>WMu-W!SdTKbq+HdKtis?NxkRxZ$qSeOCGaBhz|Z(DEp*18 z1VY0=kluAfiGjwwj;QdjMMGCGU*OjKSx<7Ei}Qj)i@i@!ss5pK%B8wKW43@}FZc$1 z-YoNXL5^b2WSlRy4ve@Z5jq~L&dXc<&fA`H7{ix;`+e}9bh&Hz9biU!LH$`ro>n{E z60{dR1cz+zB{R$pgoATCvTD1<7#BtK@y^5If#X$}l~ytQCQx-!#mp8tbkW2!!BzcyD)40=2|*Yu0mzK2QhCp1h#(R@$2;3wHfiXgEyLjy>&XZ{&M zX|0LbwAC69Uagm>U>z2#~Po-F%98OE1a8pWC?$^=_E$3P3gIXP#XRT!S%HmE3Nof?Q8}oXNel$6zZ6o5zeox?V*DP z#;gc)w7}{?5S6x8>d);zSK@Bkb2cjyb4fpGEQY8yvG{d=<)f#aeV&c7cz}dINU$Mi z(%?!S-H5nn;V;BHL`q}2RFUQG#`yzUbSbPC|xe%Okxc%);L zG_IfQ50^C{^A+S3h12axEIV`>eqL^5>t|45rId@hnBdprP!y7Z)cQ%p(8ARJ5fkIp zsXBB>UB(p=2!Bb&w+Ydbzv(Zoq=hleRCOX?9E-CqQnFv*KyBvL5g10fl#6st3l1r^ z{nu}0VD+#h3EPFLP)&G6MVtXL zojBMIJEED*owWecK9Axcvs^)EyxTG6kCj#khg~RI92J@%q-I~YswpGSNItHCSVz-Z z$aI%XJe@qt>YU7K`DFEY%(uxUQNk=Y1!MdKB!^j3lDhl& zB*r^qUR%{ANk;qd1q6@ttEMdwk?leq$2=`&Sl6|!Y!1R}KfWg7%;x6J6}JEmGNXFm zg|_y^m62>BRdyx`Y%_8b#P`(XCq2~>tsGTcLL!`UA*V>h`1J*&%T zdIHFYXJMi^OA7M~hfB<*ZueY+JM&>+Qfs#=kiLtfx0Ft)66%I_u?evJL21EhB1K~o z`y+e<;GfX>bBQsII2~e7232`QBzVq9t<1BI9gB&3v^Ec(tsL>=LHPD(3RZhi>+eHu zd|8z;=K=UNDEvmBsN1(=_6jNRl;dDjM9kO}*MC(c^F3lY{V&6y`f`AQZw?~-MqNy@ zTjAUYNJv+3iVw0y+J$1+cV)GLRf00|eV_EtDGG}ZM`MgKy1E3@Y68%4IWb*yvmw;1 zW4+u|$L@h*3@+;&b&FewrGx#rG#a-Y6k`B#0lUWXJ{=|geA4hq+^u1speQWAISOkxN6G2HT#(@9Tx^dB9XN_J?3OOn|~ zl$aAWj7%vg4nFC>fH5@o+O&Bq=Yw0FizVKxE{rDu<>BtzXAf=xem*|A%c3k`_IB1; zS?QAC^M3G%gl?zt#n9;@+H;`p^q*0YcXU&pIoTNQ@}1(qL22#*r= zZZi_}Yy%6t5zSkDn-$(McjvFXR9jx!dN;Or+L1<0IbO;R%_-O(w+5pxh#!$=qJ4Y4 zYD|XROqif~U`MF-?cxEZyv;j173tj z-YY(e%y5_KiS|+MCa32c^uh!YtRyu#U+7JX-2>9+vtNsXrX)PoX~9gbOv0o7fgfj} zB`?g8I*)BLm-MV-8F|9RS6zfd%mWs5oU49T_0Hc?R!?L211om!o0F5?OCs*R=6-{c#%b^7GQ}uK~jPH z!qWw1S0j(t4IW+yW|v#OYAN)jCMFo4AluBz$FX=j+Sk*9N}jv6sek`8*blveRYyK6 z@$$QlJR0o@v$S+f-zsLw0nh#kUV&fD{$c1Ky*FirKmqzg+)FWg)*qYr#!&xh)r5FM zyIhdtLDGe=z-F!B!f`gKQ;5@DmkA~JFJ)}&q2vWU*3SVpi6R6uxf)tZkEGzFa5#xh zgxWZZW?URJ?Z)bcPP-?uZsE@O`(e|((Jc)+yo;i4MIL;)hlm(2w741^jymCajG}`Y z0+9`yJ4PswEoFzGwoK&Bt{R)>WKNgeyhyZZrCWq%%VuYWOSZTCmc7B@AINXaIYw>g zD(_7~W$3#FFPFybE@REcF<7d=>Bl!Qs|)m~SLEeCXQD;JBti`=eSRQFLEkCdcI{wy zZh^j@{zDOlr}L}zgS3@RiQBzf2Jwro|}z zp(8`DShFcww4*$ph=`Zv&Qf;2lWqEvw#uf03PUx5*6Zt_ixy%t9Lsse#_!)n3$--l zOf$;2nUJKM8%rIVj%qU1>XT_ym2MR4aaD{P*8oOSZgIqcWfWlkoR%D~ll0=66q}CTgR^m^OW6AzkH7eH)iozB+LoEQPHk( z#`+MS)QEj`X~>v7ZPYe^*p)Xt3}Ja0T^Df?O^X*F|EApS<~55@Q05SkK0sF+UD=#y zt7#A&M)vf*n^sI0F~cOr_VJvOH0Xd?%4c zS9%8jMQZ#au03wIpvh_4m~jGGx}6aI{d!htmWrf+Ec501JY=~N`(k@SGWn!aRsfxN){B8UN2djrCZY-c;VfAmwKt~0mYbZs}* zN)bzhWb*t}1j2|hWp6O^-@hIy=snZ+vUl(7haLy(cRSqP)j6yC>k9j)-0U_2f`oC* zDq6$j2-(gxSw{;!Dp96XDiCcn<=s}RfXP?}T|Y2spwLwsB6ETb1}TfF=R{7Hzpnh5 zA8mde1`9$mIOIAp6)$HGzWUmv@fqHkz82Ew-Q~St6-GJ%T zoE#?-c3l0~iaA9*ZHhlS4{FA<9Xf40OlkBmvD;}@=7o63Ay)&<*d*Y$1s;!ljpE;>z#T%*x>L7ZnjI45Ij{?bC*!?k!+qG ztdZ3sm+s_sl6t;4RC2XWn51!HZA6K~SFd{_-)wmP_l?z2qE~E~<2OIQ+O+`I`?nv4 zTY=XT@qB)6R50(?106eq%h-+tvkEe1h`*@lmM&+x3DEC^osEhDdqcgXu%ke2MH&Xk z1C-O3ZCc_QBqYIvgg?eabiv}wJFj##c2D8mmh`lixXcu@YxCQrG8!B!t|Fs3VzCQ; z9hr_t$>&PsMb)7~T9Gy2%f@h*+#5)SQ1_;4J^h9y10)bshZ z;l2nhm_6Q$h;b}ZWEkFj``_4Ccc@<0bZ^yIU;nEXlUv%4ty-&3ERH>Fs*hBk2V4(@zX=>s`_S;> znv9FMT_}=x6fgK5Eocs51k=oLfx-1*kl`Xt-`Wy>}^8>`FDC3BHmx0tiP7SUAm<*Y2o55|>ORCS?h9s0JBXbw;#Cph$cb&794ji= z+q>GiW^0_In6F@|`Go$PG?<~CdAy08(5Tw{%|4#eF}0z$P|{heEvSj_fb)BSxH5<| z05&!eJ_hd`J6pRTn3-`De*kX~6ob6;5$76=(raIQ zLf|D#m~aFvX;k~)4ngj9jDkYEH>=9Bl0Y4lFbo2hwZ;8SM5yle*pjPB#+xSFQmlZS zx-6>M44W~rAali^78Y#mRKbxFx=eMiUEa9z(ucTGd4XT}DvL>5sH(2)4?_+6KO;-8 zrn@NfBWJqrmF0aeV)74j{RNieoN=x1WWDtZBl&cYz_p4>6*bDFG3D`jit{?pN}=Kb zA$HRnUz77!U1Y__9o>Mc9eAhu-xJAe)|vDDd>|D0$V1~)51#MF`!ucYiH0PDBh7hd zP@~9L9U6_>0ITN)i|*;n^J#Cuv4^nl9;%&+iqY3>S?5D)G#pDe#$!hX0bHuh9I~vq zA2D4T@VATH2!##Rj~ya`D*lSE^NQsk@^8~~tHFwqGoQhqMQ94Y#*!-iK3j^ml#r&i zOqazq3pA5ARb?ZISzwF}DezJS|A=-F4_sjNEx`+yGyRH{IhD+PA05?2fF70oRRvbTyn=GafV{2>-SOR5)yp}dOVJQnupdB__2H{ zi%Re7Q-_+nW%M@Y$ImbA3k6IhfhQs^_th%;8QPSFoVu@2dYLVA7&B7wEV3z3DWY|4`dJ^1W>(H5b9w2ewH26TeK*KTVdYH@0yhXow`Vt zEiQb%wNti%zh@KY^!l}LTgdz&+oC$>Osld`vBzQUXWP=M-9c}NQL_(n4;71kn5XGo zmVOZ3ksQkzy(!yLlj|9MYY%lc=Ah@ZOz?K%F2w`tdy65K9JF()4*MSTo^&Wn?TB3P zh4PYQtzNI2laZ^V1u@2%VYXofo#$f9?} z{g5ky{arkjo0YZngdjFBkKC`Vo`@ZkWNC`C_ZF7g_;LQ^=gJK60isc0nfD||;QbLh zqm?XPW>-Ds0dZJbpO zb}am_%z^ldSG0U6@a*@mqlI3hkR}r6(>VCjfiSOI46I~*s;(97Ro)8+>zQ@jlv$49PArKvxkxgwBdB;#)2(4-!CdDVF!4L+<>%U)0rggTDio~bmuS8 z*DD7#>a9n~qz&fVQ)Srb$Y8w@3@3OW!=V6HjEqk8@ilHta1dF<-HO!0i~(!}5~#<= z!n4PX!FG>le~I^w5dGJxZstqGGH1pB;o}eE(Eh6Be7L8vtB>x7O+Oo_hROX4XeF%iNrNuDbMF%%Fj5&tjH zZ7s_!M;$vi4iUxIB2MrA(l$%5jD^&&(JiBh?Iq~B=emhrk`8_i{Ffx(xx%$@JBb4$SlNt~?WQ(N zrbFis>F-n+Ewf$L%LDR}95)U!ev7AlHLtPc>%(EeK6Xt72Nfmhq@VH#)l!BvMwO(w<36$uo$fW(#UmwvEP`o}J zPq{_b+bON@JG)PrK_|W_HmDM^PA|s$o1Y4khOl?^I?z#%nE! z{XC7pZ{9)DmQ?j7%D20V@pyT&Qdj#Tq9{+FAHx6pAWx)0Eu9L z5P*=4FobZ6NRH@+n21=7xPVTSv+KMKCW`On=9T!~!Jpg?S1Asw@0mRV42*4P_1jnSrl*M$yOvfC< ze8(ciO2@{;PRE|bp~m6EF~AAJsl@q<^NGucYk}L0JBj-b_Z|-(j~tH=PZiGu&krvf z?;0O~55)h8AAsM8|4D#LU_uZ>@SEVAkd#n}P=_#?aDecVh?K~UsE=5H*n_x`xQBR& z_?m=}M294iWQb&!6qi(l)POXKw3+ms44W*0Y=CT+9Fbg_+<`ose1!a!f}O&PBAa53 z5}Zw{%81H?s+?+r8k<^z+JSn2=DS1cf3GEvp@e?oJ^-k!K_hm=RJ*f~ zEPy^8)bGD}--KRiQ5NiBg;%7?zy1B=B*CHtc5B`!uGQRYFqnRBRXcLS z5pE{wla8bepSRui&#pNdE4gXH30(*{{GCl_2&(6MoneF?{$&T+Oa5g?MnXO=2THwJ zNyu0l{80#UvlT~tQNytW?0(Xc(S$a90`+1L4jIB^YnjWGh~q2PwiAbQyrJWIs()GM z-LTx|QI(~BF!yZyu3jYOyxi)d6q1}%F&nsTiNOoMg)@>4DswO zd7&f@=3|L%Ce-$h8rp+jmYY_uB#UFDQ4=Lb^GwKDnU=3`E4&nCwr*b=o=B|s^hs1R#V!agd6;mD@GGo*1m^2txCCYJ=jET}Lb#)NzldN#7*)#TZtJX7)bZh()DN<&DULB-z4J%ASOCDOS zi0&0yIg1V%+Atv2pu!%dK1bsWTZ|X)or9^6BWGs)3I=Y28W_*KeR-jvY4B^gK*h{y^sAn)+SUTnDOF`orBX|!{9+a4 zVtJ-&laFDBi^D=mo7d6d<;Dz!8i#DF~u*T d`d@*P)=+z2O9=Gccp2C_0H}G=_V0V@{{Zm~b;kez diff --git a/httpserver/static-content/popper.min.js b/httpserver/static-content/popper.min.js deleted file mode 100644 index 8a17212f..00000000 --- a/httpserver/static-content/popper.min.js +++ /dev/null @@ -1,5 +0,0 @@ -/* - Copyright (C) Federico Zivolo 2019 - Distributed under the MIT License (license terms are at http://opensource.org/licenses/MIT). - */(function(e,t){'object'==typeof exports&&'undefined'!=typeof module?module.exports=t():'function'==typeof define&&define.amd?define(t):e.Popper=t()})(this,function(){'use strict';function e(e){return e&&'[object Function]'==={}.toString.call(e)}function t(e,t){if(1!==e.nodeType)return[];var o=e.ownerDocument.defaultView,n=o.getComputedStyle(e,null);return t?n[t]:n}function o(e){return'HTML'===e.nodeName?e:e.parentNode||e.host}function n(e){if(!e)return document.body;switch(e.nodeName){case'HTML':case'BODY':return e.ownerDocument.body;case'#document':return e.body;}var i=t(e),r=i.overflow,p=i.overflowX,s=i.overflowY;return /(auto|scroll|overlay)/.test(r+s+p)?e:n(o(e))}function i(e){return e&&e.referenceNode?e.referenceNode:e}function r(e){return 11===e?re:10===e?pe:re||pe}function p(e){if(!e)return document.documentElement;for(var o=r(10)?document.body:null,n=e.offsetParent||null;n===o&&e.nextElementSibling;)n=(e=e.nextElementSibling).offsetParent;var i=n&&n.nodeName;return i&&'BODY'!==i&&'HTML'!==i?-1!==['TH','TD','TABLE'].indexOf(n.nodeName)&&'static'===t(n,'position')?p(n):n:e?e.ownerDocument.documentElement:document.documentElement}function s(e){var t=e.nodeName;return'BODY'!==t&&('HTML'===t||p(e.firstElementChild)===e)}function d(e){return null===e.parentNode?e:d(e.parentNode)}function a(e,t){if(!e||!e.nodeType||!t||!t.nodeType)return document.documentElement;var o=e.compareDocumentPosition(t)&Node.DOCUMENT_POSITION_FOLLOWING,n=o?e:t,i=o?t:e,r=document.createRange();r.setStart(n,0),r.setEnd(i,0);var l=r.commonAncestorContainer;if(e!==l&&t!==l||n.contains(i))return s(l)?l:p(l);var f=d(e);return f.host?a(f.host,t):a(e,d(t).host)}function l(e){var t=1=o.clientWidth&&n>=o.clientHeight}),l=0a[e]&&!t.escapeWithReference&&(n=Q(f[o],a[e]-('right'===e?f.width:f.height))),ae({},o,n)}};return l.forEach(function(e){var t=-1===['left','top'].indexOf(e)?'secondary':'primary';f=le({},f,m[t](e))}),e.offsets.popper=f,e},priority:['left','right','top','bottom'],padding:5,boundariesElement:'scrollParent'},keepTogether:{order:400,enabled:!0,fn:function(e){var t=e.offsets,o=t.popper,n=t.reference,i=e.placement.split('-')[0],r=Z,p=-1!==['top','bottom'].indexOf(i),s=p?'right':'bottom',d=p?'left':'top',a=p?'width':'height';return o[s]r(n[s])&&(e.offsets.popper[d]=r(n[s])),e}},arrow:{order:500,enabled:!0,fn:function(e,o){var n;if(!K(e.instance.modifiers,'arrow','keepTogether'))return e;var i=o.element;if('string'==typeof i){if(i=e.instance.popper.querySelector(i),!i)return e;}else if(!e.instance.popper.contains(i))return console.warn('WARNING: `arrow.element` must be child of its popper element!'),e;var r=e.placement.split('-')[0],p=e.offsets,s=p.popper,d=p.reference,a=-1!==['left','right'].indexOf(r),l=a?'height':'width',f=a?'Top':'Left',m=f.toLowerCase(),h=a?'left':'top',c=a?'bottom':'right',u=S(i)[l];d[c]-us[c]&&(e.offsets.popper[m]+=d[m]+u-s[c]),e.offsets.popper=g(e.offsets.popper);var b=d[m]+d[l]/2-u/2,w=t(e.instance.popper),y=parseFloat(w['margin'+f],10),E=parseFloat(w['border'+f+'Width'],10),v=b-e.offsets.popper[m]-y-E;return v=ee(Q(s[l]-u,v),0),e.arrowElement=i,e.offsets.arrow=(n={},ae(n,m,$(v)),ae(n,h,''),n),e},element:'[x-arrow]'},flip:{order:600,enabled:!0,fn:function(e,t){if(W(e.instance.modifiers,'inner'))return e;if(e.flipped&&e.placement===e.originalPlacement)return e;var o=v(e.instance.popper,e.instance.reference,t.padding,t.boundariesElement,e.positionFixed),n=e.placement.split('-')[0],i=T(n),r=e.placement.split('-')[1]||'',p=[];switch(t.behavior){case ce.FLIP:p=[n,i];break;case ce.CLOCKWISE:p=G(n);break;case ce.COUNTERCLOCKWISE:p=G(n,!0);break;default:p=t.behavior;}return p.forEach(function(s,d){if(n!==s||p.length===d+1)return e;n=e.placement.split('-')[0],i=T(n);var a=e.offsets.popper,l=e.offsets.reference,f=Z,m='left'===n&&f(a.right)>f(l.left)||'right'===n&&f(a.left)f(l.top)||'bottom'===n&&f(a.top)f(o.right),g=f(a.top)f(o.bottom),b='left'===n&&h||'right'===n&&c||'top'===n&&g||'bottom'===n&&u,w=-1!==['top','bottom'].indexOf(n),y=!!t.flipVariations&&(w&&'start'===r&&h||w&&'end'===r&&c||!w&&'start'===r&&g||!w&&'end'===r&&u),E=!!t.flipVariationsByContent&&(w&&'start'===r&&c||w&&'end'===r&&h||!w&&'start'===r&&u||!w&&'end'===r&&g),v=y||E;(m||b||v)&&(e.flipped=!0,(m||b)&&(n=p[d+1]),v&&(r=z(r)),e.placement=n+(r?'-'+r:''),e.offsets.popper=le({},e.offsets.popper,C(e.instance.popper,e.offsets.reference,e.placement)),e=P(e.instance.modifiers,e,'flip'))}),e},behavior:'flip',padding:5,boundariesElement:'viewport',flipVariations:!1,flipVariationsByContent:!1},inner:{order:700,enabled:!1,fn:function(e){var t=e.placement,o=t.split('-')[0],n=e.offsets,i=n.popper,r=n.reference,p=-1!==['left','right'].indexOf(o),s=-1===['top','left'].indexOf(o);return i[p?'left':'top']=r[o]-(s?i[p?'width':'height']:0),e.placement=T(t),e.offsets.popper=g(i),e}},hide:{order:800,enabled:!0,fn:function(e){if(!K(e.instance.modifiers,'hide','preventOverflow'))return e;var t=e.offsets.reference,o=D(e.instance.modifiers,function(e){return'preventOverflow'===e.name}).boundaries;if(t.bottomo.right||t.top>o.bottom||t.rightwindow.devicePixelRatio||!fe),c='bottom'===o?'top':'bottom',g='right'===n?'left':'right',b=B('transform');if(d='bottom'==c?'HTML'===l.nodeName?-l.clientHeight+h.bottom:-f.height+h.bottom:h.top,s='right'==g?'HTML'===l.nodeName?-l.clientWidth+h.right:-f.width+h.right:h.left,a&&b)m[b]='translate3d('+s+'px, '+d+'px, 0)',m[c]=0,m[g]=0,m.willChange='transform';else{var w='bottom'==c?-1:1,y='right'==g?-1:1;m[c]=d*w,m[g]=s*y,m.willChange=c+', '+g}var E={"x-placement":e.placement};return e.attributes=le({},E,e.attributes),e.styles=le({},m,e.styles),e.arrowStyles=le({},e.offsets.arrow,e.arrowStyles),e},gpuAcceleration:!0,x:'bottom',y:'right'},applyStyle:{order:900,enabled:!0,fn:function(e){return V(e.instance.popper,e.styles),j(e.instance.popper,e.attributes),e.arrowElement&&Object.keys(e.arrowStyles).length&&V(e.arrowElement,e.arrowStyles),e},onLoad:function(e,t,o,n,i){var r=L(i,t,e,o.positionFixed),p=O(o.placement,r,t,e,o.modifiers.flip.boundariesElement,o.modifiers.flip.padding);return t.setAttribute('x-placement',p),V(t,{position:o.positionFixed?'fixed':'absolute'}),o},gpuAcceleration:void 0}}},ge}); -//# sourceMappingURL=popper.min.js.map diff --git a/httpserver/static-content/styles.css b/httpserver/static-content/styles.css deleted file mode 100644 index fedfd863..00000000 --- a/httpserver/static-content/styles.css +++ /dev/null @@ -1,71 +0,0 @@ -/* - Copyright (c) 2015-2021, NVIDIA CORPORATION. - SPDX-License-Identifier: Apache-2.0 -*/ - -.table td.fit, -.table th.fit { - white-space: nowrap; - width: 1%; -} - -body { padding-top: 70px; } - -.no-margin { margin: 0; } - -pre.code { - background-color: #e9ecef; - border-radius: .25rem; - padding: 20px; -} - -.clickable { - cursor: pointer; -} - -span.jstExpand, span.jstFold { - cursor: pointer; -} - -.jstValue { - white-space: pre-wrap; -} -.jstComma { - white-space: pre-wrap; -} -.jstProperty { - color: #666; - word-wrap: break-word; -} -.jstBracket { - white-space: pre-wrap;; -} -.jstBool { - color: #2525CC; -} -.jstNum { - color: #D036D0; -} -.jstNull { - color: gray; -} -.jstStr { - color: #2DB669; -} -.jstFold:after { - content: ' -'; - cursor: pointer; -} -.jstExpand { - white-space: normal; -} -.jstExpand:after { - content: ' +'; - cursor: pointer; -} -.jstFolded { - white-space: normal !important; -} -.jstHiddenBlock { - display: none; -} diff --git a/inode/.gitignore b/inode/.gitignore deleted file mode 100644 index d393ff6c..00000000 --- a/inode/.gitignore +++ /dev/null @@ -1 +0,0 @@ -inodetype_string.go diff --git a/inode/Makefile b/inode/Makefile deleted file mode 100644 index 1122a376..00000000 --- a/inode/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/inode - -generatedfiles := \ - inodetype_string.go - -include ../GoMakefile diff --git a/inode/api.go b/inode/api.go deleted file mode 100644 index 0bd69866..00000000 --- a/inode/api.go +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package inode provides inode-management functionality for ProxyFS. -package inode - -import ( - "time" - "unsafe" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/dlm" - "github.com/NVIDIA/proxyfs/utils" -) - -type InodeNumber uint64 -type InodeType uint16 -type InodeMode uint32 -type InodeUserID uint32 -type InodeGroupID uint32 -type InodeDirLocation int64 - -const ( - InodeRootUserID = InodeUserID(0) -) - -// NOTE: Using unix.DT_* constants for these types makes it easier -// to expose this information in a standardized way with our RPC APIs. -const ( - DirType InodeType = unix.DT_DIR - FileType InodeType = unix.DT_REG - SymlinkType InodeType = unix.DT_LNK -) - -// NOTE: Would have liked to use os.FileMode bitmask definitions here instead of creating our own, -// but unfortunately the bitmasks used by os.ModeDir and os.ModeSymlink (0x80000000 and 0x8000000) -// are not the same values as what is expected on the linux side (0x4000 and 0xa000). -const ( - PosixModeType InodeMode = 0xE000 - PosixModeDir InodeMode = 0x4000 - PosixModeFile InodeMode = 0x8000 - PosixModeSymlink InodeMode = 0xa000 - PosixModePerm InodeMode = 0777 -) - -// The following are used in calls to Access()... either F_OK or bitwise or of R_OK, W_OK, and X_OK -const ( - F_OK = InodeMode(unix.F_OK) // check for existence - R_OK = InodeMode(unix.R_OK) // UID:GID check for read permission - W_OK = InodeMode(unix.W_OK) // UID:GID check for write permission - X_OK = InodeMode(unix.X_OK) // UID:GID check for execute permission - P_OK = InodeMode((unix.R_OK | unix.W_OK | unix.X_OK) + 1) // check for ownership permissions -) - -// AccessOverride.Owner means Access() grants permission to the owner of the -// file even if the permission bits disallow it. -type AccessOverride uint32 - -const ( - NoOverride AccessOverride = iota - OwnerOverride -) - -// The following line of code is a directive to go generate that tells it to create a -// file called inodetype_string.go that implements the .String() method for InodeType. -//go:generate stringer -type=InodeType - -type MetadataStruct struct { - InodeType - LinkCount uint64 - Size uint64 - CreationTime time.Time - ModificationTime time.Time - AccessTime time.Time - AttrChangeTime time.Time // aka ctime; This field is intended to be changed by writing or by setting inode information (i.e., owner, group, link count, mode, etc.). - NumWrites uint64 // only maintained for FileType inodes - InodeStreamNameSlice []string - Mode InodeMode - UserID InodeUserID - GroupID InodeGroupID -} - -type FragmentationReport struct { - NumberOfFragments uint64 // used with BytesInFragments to compute average fragment size - BytesInFragments uint64 // equivalent to size of file for FileInode that is not sparse - BytesTrapped uint64 // unreferenced bytes trapped in referenced log segments -} - -type DirEntry struct { - InodeNumber - Basename string - Type InodeType - NextDirLocation InodeDirLocation -} - -type CoalesceElement struct { - ContainingDirectoryInodeNumber InodeNumber - ElementInodeNumber InodeNumber - ElementName string -} - -type ReadPlanStep struct { - LogSegmentNumber uint64 // If == 0, Length specifies zero-fill size - Offset uint64 // If zero-fill case, == 0 - Length uint64 // Must != 0 - AccountName string // If == "", Length specifies a zero-fill size - ContainerName string // If == "", Length specifies a zero-fill size - ObjectName string // If == "", Length specifies a zero-fill size - ObjectPath string // If == "", Length specifies a zero-fill size -} - -type ExtentMapEntryStruct struct { - FileOffset uint64 - LogSegmentOffset uint64 - Length uint64 - ContainerName string // While "read-as-zero" entries in ExtentMapShunkStruct - ObjectName string // are not present, {Container|Object}Name would be == "" -} - -type ExtentMapChunkStruct struct { - FileOffsetRangeStart uint64 // Holes in [FileOffsetRangeStart:FileOffsetRangeEnd) - FileOffsetRangeEnd uint64 // not covered in ExtentMapEntry slice should "read-as-zero" - FileSize uint64 // up to the end-of-file as indicated by FileSize - ExtentMapEntry []ExtentMapEntryStruct // All will be in [FileOffsetRangeStart:FileOffsetRangeEnd) -} - -const ( - RootDirInodeNumber = InodeNumber(1) -) - -const ( - SnapShotDirName = ".snapshot" -) - -type RWModeType uint8 - -const ( - RWModeNormal RWModeType = iota // All reads, writes, etc... should be allowed - RWModeNoWrite // Same as modeNormal except Write() & ProvisionObject() should fail - RWModeReadOnly // No operations that modify state (data or metadata) should be allowed -) - -func (de *DirEntry) Size() int { - // sizeof(InodeNumber) + sizeof(InodeType) + sizeof(DirLocation) + string data + null byte delimiter - return int(unsafe.Sizeof(de.InodeNumber)) + int(unsafe.Sizeof(de.Type)) + int(unsafe.Sizeof(de.NextDirLocation)) + len(de.Basename) + 1 -} - -// AccountNameToVolumeName returns the corresponding volumeName for the supplied accountName (if any). -func AccountNameToVolumeName(accountName string) (volumeName string, ok bool) { - volumeName, ok = accountNameToVolumeName(accountName) - return -} - -// VolumeNameToAccountName returns the corresponding accountName for the supplied volumeName (if any). -func VolumeNameToAccountName(volumeName string) (accountName string, ok bool) { - accountName, ok = volumeNameToAccountName(volumeName) - return -} - -// VolumeNameToActivePeerPrivateIPAddr returns the Peer IP Address serving the specified VolumeName. -func VolumeNameToActivePeerPrivateIPAddr(volumeName string) (activePeerPrivateIPAddr string, ok bool) { - activePeerPrivateIPAddr, ok = volumeNameToActivePeerPrivateIPAddr(volumeName) - return -} - -// FetchVolumeHandle returns a the VolumeHandle corresponding to the name VolumeName. -// -// Note: The method should be considered a write operation on the RoodDirInodeNumber. -// As such, an exclusive lock should be held around a call to FetchVolumeHandle(). -func FetchVolumeHandle(volumeName string) (volumeHandle VolumeHandle, err error) { - volumeHandle, err = fetchVolumeHandle(volumeName) - return -} - -type VolumeHandle interface { - // Generic methods, implemented volume.go - - GetFSID() (fsid uint64) - SnapShotCreate(name string) (id uint64, err error) - SnapShotDelete(id uint64) (err error) - - // Wrapper methods around DLM locks. Implemented in locker.go - - MakeLockID(inodeNumber InodeNumber) (lockID string, err error) - InitInodeLock(inodeNumber InodeNumber, callerID dlm.CallerID) (lock *dlm.RWLockStruct, err error) - GetReadLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) - GetWriteLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) - AttemptReadLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) - AttemptWriteLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) - EnsureReadLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) - EnsureWriteLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) - - // Common Inode methods, implemented in inode.go - - Access(inodeNumber InodeNumber, userID InodeUserID, groupID InodeGroupID, otherGroupIDs []InodeGroupID, accessMode InodeMode, override AccessOverride) (accessReturn bool) - Purge(inodeNumber InodeNumber) (err error) - Destroy(inodeNumber InodeNumber) (err error) - GetMetadata(inodeNumber InodeNumber) (metadata *MetadataStruct, err error) - GetType(inodeNumber InodeNumber) (inodeType InodeType, err error) - GetLinkCount(inodeNumber InodeNumber) (linkCount uint64, err error) - SetLinkCount(inodeNumber InodeNumber, linkCount uint64) (err error) - SetCreationTime(inodeNumber InodeNumber, creationTime time.Time) (err error) - SetModificationTime(inodeNumber InodeNumber, modificationTime time.Time) (err error) - SetAccessTime(inodeNumber InodeNumber, accessTime time.Time) (err error) - SetPermMode(inodeNumber InodeNumber, filePerm InodeMode) (err error) - SetOwnerUserID(inodeNumber InodeNumber, userID InodeUserID) (err error) - SetOwnerUserIDGroupID(inodeNumber InodeNumber, userID InodeUserID, groupID InodeGroupID) (err error) - SetOwnerGroupID(inodeNumber InodeNumber, groupID InodeGroupID) (err error) - GetStream(inodeNumber InodeNumber, inodeStreamName string) (buf []byte, err error) - PutStream(inodeNumber InodeNumber, inodeStreamName string, buf []byte) (err error) - DeleteStream(inodeNumber InodeNumber, inodeStreamName string) (err error) - FetchOnDiskInode(inodeNumber InodeNumber) (corruptionDetected CorruptionDetected, version Version, onDiskInode []byte, err error) - PatchInode(inodeNumber InodeNumber, inodeType InodeType, linkCount uint64, mode InodeMode, userID InodeUserID, groupID InodeGroupID, parentInodeNumber InodeNumber, symlinkTarget string) (err error) - FetchLayoutReport(inodeNumber InodeNumber) (layoutReport sortedmap.LayoutReport, err error) - FetchFragmentationReport(inodeNumber InodeNumber) (fragmentationReport FragmentationReport, err error) - Optimize(inodeNumber InodeNumber, maxDuration time.Duration) (err error) - Validate(inodeNumber InodeNumber, deeply bool) (err error) - - // Directory Inode specific methods, implemented in dir.go - - CreateDir(filePerm InodeMode, userID InodeUserID, groupID InodeGroupID) (dirInodeNumber InodeNumber, err error) - Link(dirInodeNumber InodeNumber, basename string, targetInodeNumber InodeNumber, insertOnly bool) (err error) - Unlink(dirInodeNumber InodeNumber, basename string, removeOnly bool) (toDestroyInodeNumber InodeNumber, err error) - Move(srcDirInodeNumber InodeNumber, srcBasename string, dstDirInodeNumber InodeNumber, dstBasename string) (toDestroyInodeNumber InodeNumber, err error) - Lookup(dirInodeNumber InodeNumber, basename string) (targetInodeNumber InodeNumber, err error) - NumDirEntries(dirInodeNumber InodeNumber) (numEntries uint64, err error) - ReadDir(dirInodeNumber InodeNumber, maxEntries uint64, maxBufSize uint64, prevReturned ...interface{}) (dirEntrySlice []DirEntry, moreEntries bool, err error) - AddDirEntry(dirInodeNumber InodeNumber, dirEntryName string, dirEntryInodeNumber InodeNumber, skipDirLinkCountIncrementOnSubDirEntry bool, skipSettingDotDotOnSubDirEntry bool, skipDirEntryLinkCountIncrementOnNonSubDirEntry bool) (err error) - ReplaceDirEntries(parentDirInodeNumber InodeNumber, parentDirEntryBasename string, dirInodeNumber InodeNumber, dirEntryInodeNumbers []InodeNumber) (err error) - - // File Inode specific methods, implemented in file.go - - CreateFile(filePerm InodeMode, userID InodeUserID, groupID InodeGroupID) (fileInodeNumber InodeNumber, err error) - Read(inodeNumber InodeNumber, offset uint64, length uint64, profiler *utils.Profiler) (buf []byte, err error) - GetReadPlan(fileInodeNumber InodeNumber, offset *uint64, length *uint64) (readPlan []ReadPlanStep, err error) - FetchExtentMapChunk(fileInodeNumber InodeNumber, fileOffset uint64, maxEntriesFromFileOffset int64, maxEntriesBeforeFileOffset int64) (extentMapChunk *ExtentMapChunkStruct, err error) - Write(fileInodeNumber InodeNumber, offset uint64, buf []byte, profiler *utils.Profiler) (err error) - ProvisionObject() (objectPath string, err error) - Wrote(fileInodeNumber InodeNumber, containerName string, objectName string, fileOffset []uint64, objectOffset []uint64, length []uint64, wroteTime time.Time, patchOnly bool) (err error) - SetSize(fileInodeNumber InodeNumber, Size uint64) (err error) - Flush(fileInodeNumber InodeNumber, andPurge bool) (err error) - Coalesce(destInodeNumber InodeNumber, metaDataName string, metaData []byte, elements []*CoalesceElement) (attrChangeTime time.Time, modificationTime time.Time, numWrites uint64, fileSize uint64, err error) - DefragmentFile(fileInodeNumber InodeNumber, startingFileOffset uint64, chunkSize uint64) (nextFileOffset uint64, eofReached bool, err error) - - // Symlink Inode specific methods, implemented in symlink.go - - CreateSymlink(target string, filePerm InodeMode, userID InodeUserID, groupID InodeGroupID) (symlinkInodeNumber InodeNumber, err error) - GetSymlink(symlinkInodeNumber InodeNumber) (target string, err error) -} - -// SetRWMode sets the package to either allow all read/write operations (RWModeNormal), -// allow all except Write() and ProvisionObject() operations (RWModeNoWrite), or -// disallow any data or metadata operations (RWModeReadOnly). -func SetRWMode(rwMode RWModeType) (err error) { - err = setRWMode(rwMode) - return -} diff --git a/inode/api_test.go b/inode/api_test.go deleted file mode 100644 index 2538824e..00000000 --- a/inode/api_test.go +++ /dev/null @@ -1,2328 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "bytes" - "fmt" - "strings" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/utils" - "github.com/stretchr/testify/assert" -) - -const durationToDelayOrSkew = "100ms" - -var AllMetadataFields = map[string]bool{ - "InodeType": true, - "LinkCount": true, - "Size": true, - "CreationTime": true, - "ModificationTime": true, - "AccessTime": true, - "AttrChangeTime": true, - "NumWrites": true, - "InodeStreamNameSlice": true, - "Mode": true, - "UserID": true, - "GroupID": true, -} - -var MetadataPermFields = map[string]bool{ - "Mode": true, - "UserID": true, - "GroupID": true, -} - -var MetadataLinkCountField = map[string]bool{ - "LinkCount": true, -} - -var MetadataCrTimeField = map[string]bool{ - "CreationTime": true, -} - -var MetadataModTimeField = map[string]bool{ - "ModificationTime": true, -} - -var MetadataSizeField = map[string]bool{ - "Size": true, -} - -var MetadataTimeFields = map[string]bool{ - "CreationTime": true, - "AttrChangeTime": true, - "ModificationTime": true, - "AccessTime": true, -} - -var MetadataNotAttrTimeFields = map[string]bool{ - "CreationTime": true, - "ModificationTime": true, - "AccessTime": true, -} - -var MetadataNumWritesField = map[string]bool{ - "NumWrites": true, -} - -func checkMetadata(t *testing.T, md *MetadataStruct, expMd *MetadataStruct, fieldsToCheck map[string]bool, errorPrefix string) { - for field := range fieldsToCheck { - switch field { - - case "InodeType": - value := md.InodeType - expValue := expMd.InodeType - if value != expValue { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "LinkCount": - value := md.LinkCount - expValue := expMd.LinkCount - if value != expValue { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "Size": - value := md.Size - expValue := expMd.Size - if value != expValue { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "CreationTime": - value := md.CreationTime - expValue := expMd.CreationTime - if !value.Equal(expValue) { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "ModificationTime": - value := md.ModificationTime - expValue := expMd.ModificationTime - if !value.Equal(expValue) { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "AccessTime": - value := md.AccessTime - expValue := expMd.AccessTime - if !value.Equal(expValue) { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "AttrChangeTime": - value := md.AttrChangeTime - expValue := expMd.AttrChangeTime - if !value.Equal(expValue) { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "NumWrites": - value := md.NumWrites - expValue := expMd.NumWrites - if value != expValue { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "InodeStreamNameSlice": - value := len(md.InodeStreamNameSlice) - expValue := len(expMd.InodeStreamNameSlice) - if value != expValue { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - for i, streamName := range md.InodeStreamNameSlice { - value := streamName - expValue := expMd.InodeStreamNameSlice[i] - if value != expValue { - t.Fatalf("%s returned %s[%d] %v, expected %v", errorPrefix, - field, i, value, expValue) - } - } - - case "Mode": - value := md.Mode - expValue := expMd.Mode - - // Add the file type to the expected mode - if md.InodeType == DirType { - expValue |= PosixModeDir - } else if md.InodeType == SymlinkType { - expValue |= PosixModeSymlink - } else if md.InodeType == FileType { - expValue |= PosixModeFile - } - if value != expValue { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "UserID": - value := md.UserID - expValue := expMd.UserID - if value != expValue { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - case "GroupID": - value := md.GroupID - expValue := expMd.GroupID - if value != expValue { - t.Fatalf("%s returned %s %v, expected %v", errorPrefix, field, value, expValue) - } - - default: - // catch field name typos - t.Fatalf("%s specified unknown field '%v'", errorPrefix, field) - } - } -} - -func checkMetadataTimeChanges(t *testing.T, oldMd *MetadataStruct, newMd *MetadataStruct, creationTimeShouldChange bool, modificationTimeShouldChange bool, accessTimeShouldChange bool, attrChangeTimeShouldChange bool, errorPrefix string) { - if creationTimeShouldChange { - if oldMd.CreationTime == newMd.CreationTime { - t.Fatalf("%s should have changed CreationTime", errorPrefix) - } - } else { - if oldMd.CreationTime != newMd.CreationTime { - t.Fatalf("%s should not have changed CreationTime", errorPrefix) - } - } - - if modificationTimeShouldChange { - if oldMd.ModificationTime == newMd.ModificationTime { - t.Fatalf("%s should have changed ModificationTime", errorPrefix) - } - } else { - if oldMd.ModificationTime != newMd.ModificationTime { - t.Fatalf("%s should not have changed ModificationTime", errorPrefix) - } - } - - if accessTimeShouldChange { - if oldMd.AccessTime == newMd.AccessTime { - t.Fatalf("%s should have changed AccessTime", errorPrefix) - } - } else { - if oldMd.AccessTime != newMd.AccessTime { - t.Fatalf("%s should not have changed AccessTime", errorPrefix) - } - } - - if attrChangeTimeShouldChange { - if oldMd.AttrChangeTime == newMd.AttrChangeTime { - t.Fatalf("%s should have changed AttrChangeTime", errorPrefix) - } - } else { - if oldMd.AttrChangeTime != newMd.AttrChangeTime { - t.Fatalf("%s should not have changed AttrChangeTime", errorPrefix) - } - } -} - -func TestAPI(t *testing.T) { - var ( - timeBeforeOp time.Time - timeAfterOp time.Time - toDestroyInodeNumber InodeNumber - ) - - testSetup(t, false) - - _, ok := AccountNameToVolumeName("BadAccountName") - if ok { - t.Fatalf("AccountNameToVolumeName(\"BadAccountName\") should have failed") - } - - goodVolumeName, ok := AccountNameToVolumeName("AUTH_test") - if !ok { - t.Fatalf("AccountNameToVolumeName(\"AUTH_test\") should have succeeded") - } - if "TestVolume" != goodVolumeName { - t.Fatalf("AccountNameToVolumeName(\"AUTH_test\") should have returned \"TestVolume\"") - } - - _, ok = VolumeNameToActivePeerPrivateIPAddr("BadVolumeName") - if ok { - t.Fatalf("VolumeNameToActivePeerPrivateIPAddr(\"BadVolumeName\") should have failed") - } - - goodActivePeerPrivateIPAddr, ok := VolumeNameToActivePeerPrivateIPAddr("TestVolume") - if !ok { - t.Fatalf("VolumeNameToActivePeerPrivateIPAddr(\"TestVolume\") should have succeeded") - } - if "127.0.0.1" != goodActivePeerPrivateIPAddr { - t.Fatalf("VolumeNameToActivePeerPrivateIPAddr(\"TestVolume\") should have returned \"127.0.0.1\"") - } - - _, err := FetchVolumeHandle("BadVolumeName") - if nil == err { - t.Fatalf("FetchVolumeHandle(\"BadVolumeName\") should have failed") - } - - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") should have worked - got error: %v", err) - } - - fsid := testVolumeHandle.GetFSID() - if 1 != fsid { - t.Fatalf("GetFSID() returned unexpected FSID") - } - - fileInodeNumber, err := testVolumeHandle.CreateFile(InodeMode(0000), InodeRootUserID, InodeGroupID(0)) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeRootUserID, InodeGroupID(0), nil, F_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,F_OK) after CreateFile() should not have failed") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeRootUserID, InodeGroupID(456), nil, - R_OK|W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,R_OK|W_OK) should have returned true") - } - - // not even root can execute a file without any execute bits set - if testVolumeHandle.Access(fileInodeNumber, InodeRootUserID, InodeGroupID(456), nil, - X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,R_OK|W_OK|X_OK) should have returned false") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(0), nil, - R_OK|W_OK|X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),InodeGroupID(0),,R_OK|W_OK|X_OK) should have returned false") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeRootUserID, InodeGroupID(0), nil, - R_OK|P_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,R_OK|X_OK) should have returned false") - } - - err = testVolumeHandle.SetOwnerUserID(fileInodeNumber, InodeUserID(123)) - if nil != err { - t.Fatalf("SetOwnerUserID(,InodeUserID(123)) failed: %v", err) - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeRootUserID, InodeGroupID(0), nil, P_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeRootUserID,,,P_OK should have returned true") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(0), nil, P_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,P_OK should have returned true") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(0), nil, P_OK, OwnerOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,P_OK,UserOveride should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(789), InodeGroupID(0), nil, P_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(789),,,P_OK should have returned false") - } - - err = testVolumeHandle.SetPermMode(fileInodeNumber, InodeMode(0600)) - if nil != err { - t.Fatalf("SetPermMode(,InodeMode(0600)) failed: %v", err) - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, R_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,R_OK) should have returned true") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,X_OK) should have returned false") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, R_OK|W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,R_OK|W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, - R_OK|W_OK|X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),,,R_OK|W_OK|X_OK) should have returned false") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, - R_OK|W_OK|X_OK, OwnerOverride) { - t.Fatalf("Access(fileInodeNumber,InodeUserID(123),OwnerOverride,,R_OK|W_OK|X_OK) should have returned false") - } - - err = testVolumeHandle.SetOwnerUserIDGroupID(fileInodeNumber, InodeRootUserID, InodeGroupID(456)) - if nil != err { - t.Fatalf("SetOwnerUserIDGroupID(,InodeRootUserID,InodeGroupID(456)) failed: %v", err) - } - - err = testVolumeHandle.SetPermMode(fileInodeNumber, InodeMode(0060)) - if nil != err { - t.Fatalf("SetPermMode(,InodeMode(0060)) failed: %v", err) - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, R_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,InodeGroupID(456),,R_OK) should have returned true") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,InodeGroupID(456),,W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,InodeGroupID(456),,X_OK) should have returned false") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, X_OK, OwnerOverride) { - t.Fatalf("Access(fileInodeNumber,,InodeGroupID(456),,X_OK,OwnerOverride) should have returned false") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, - R_OK|W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,InodeGroupID(456),,R_OK|W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, - R_OK|W_OK|X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,InodeGroupID(456),,R_OK|W_OK|X_OK) should have returned false") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(789), - []InodeGroupID{InodeGroupID(456)}, R_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,[]InodeGroupID{InodeGroupID(456)},R_OK) should have returned true") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(789), - []InodeGroupID{InodeGroupID(456)}, W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,[]InodeGroupID{InodeGroupID(456)},W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(789), - []InodeGroupID{InodeGroupID(456)}, X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,[]InodeGroupID{InodeGroupID(456)},X_OK) should have returned false") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(789), - []InodeGroupID{InodeGroupID(456)}, R_OK|W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,[]InodeGroupID{InodeGroupID(456)},R_OK|W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(789), - []InodeGroupID{InodeGroupID(456)}, R_OK|W_OK|X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,[]InodeGroupID{InodeGroupID(456)},R_OK|W_OK|X_OK) should have returned false") - } - - err = testVolumeHandle.SetPermMode(fileInodeNumber, InodeMode(0006)) - if nil != err { - t.Fatalf("SetPermMode(,InodeMode(0006)) failed: %v", err) - } - err = testVolumeHandle.SetOwnerUserIDGroupID(fileInodeNumber, InodeUserID(456), InodeGroupID(0)) - if nil != err { - t.Fatalf("SetOwnerUserIDGroupID(,InodeRootUserID,InodeGroupID(0)) failed: %v", err) - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, R_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,R_OK) should have returned true") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,X_OK) should have returned false") - } - - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, - R_OK|W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,R_OK|W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(123), InodeGroupID(456), nil, - R_OK|W_OK|X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,R_OK|W_OK|X_OK) should have returned false") - } - - // Test the ability of OwnerOverride to override permissions checks for owner (except for exec) - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(456), InodeGroupID(456), nil, R_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,R_OK) should have returned false") - } - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(456), InodeGroupID(456), nil, R_OK, OwnerOverride) { - t.Fatalf("Access(fileInodeNumber,,,,R_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(456), InodeGroupID(456), nil, W_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,W_OK) should have returned false") - } - if !testVolumeHandle.Access(fileInodeNumber, InodeUserID(456), InodeGroupID(456), nil, W_OK, OwnerOverride) { - t.Fatalf("Access(fileInodeNumber,,,,W_OK) should have returned true") - } - - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(456), InodeGroupID(456), nil, X_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,X_OK) should have returned false") - } - if testVolumeHandle.Access(fileInodeNumber, InodeUserID(456), InodeGroupID(456), nil, X_OK, OwnerOverride) { - t.Fatalf("Access(fileInodeNumber,,,,X_OK) should have returned false") - } - - err = testVolumeHandle.Destroy(fileInodeNumber) - if nil != err { - t.Fatalf("Destroy(fileInodeNumber) failed: %v", err) - } - - if testVolumeHandle.Access(fileInodeNumber, InodeRootUserID, InodeGroupID(0), nil, F_OK, NoOverride) { - t.Fatalf("Access(fileInodeNumber,,,,F_OK) after Destroy() should have failed") - } - - rootDirInodeType, err := testVolumeHandle.GetType(RootDirInodeNumber) - if nil != err { - t.Fatalf("GetType(RootDirInodeNumber) failed: %v", err) - } - if DirType != rootDirInodeType { - t.Fatalf("GetType(RootDirInodeNumber) returned unexpected type: %v", rootDirInodeType) - } - - rootDirLinkCount, err := testVolumeHandle.GetLinkCount(RootDirInodeNumber) - if nil != err { - t.Fatalf("GetLinkCount(RootDirInodeNumber) failed: %v", err) - } - if 2 != rootDirLinkCount { - t.Fatalf("GetLinkCount(RootDirInodeNumber) returned unexpected linkCount: %v", rootDirLinkCount) - } - - numEntries, err := testVolumeHandle.NumDirEntries(RootDirInodeNumber) - - if nil != err { - t.Fatalf("NumDirEntries(RootDirInodeNumber) failed: %v", err) - } - if 2 != numEntries { - t.Fatalf("NumDirEntries(RootDirInodeNumber) should have returned numEntries == 2") - } - - dirEntrySlice, moreEntries, err := testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0) - - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned moreEntries == false") - } - if 2 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[1]") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(-1)) - - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(-1)) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(-1)) should have returned moreEntries == false") - } - if 2 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(-1)) should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(-1)) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(-1)) returned unexpected dirEntrySlice[1]") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0, "") - - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"\") failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"\") should have returned moreEntries == false") - } - if 2 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"\") should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"\") returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"\") returned unexpected dirEntrySlice[1]") - } - - testMetadata := &MetadataStruct{ - InodeType: FileType, - NumWrites: 0, - LinkCount: 0, - Mode: PosixModePerm, - UserID: InodeUserID(123), - GroupID: InodeGroupID(456), - } - - fileInodeNumber, err = testVolumeHandle.CreateFile(testMetadata.Mode, testMetadata.UserID, testMetadata.GroupID) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - fileInodeType, err := testVolumeHandle.GetType(fileInodeNumber) - if nil != err { - t.Fatalf("GetType(fileInodeNumber) failed: %v", err) - } - if FileType != fileInodeType { - t.Fatalf("GetType(fileInodeNumber) returned unexpected type: %v", fileInodeType) - } - - fileLinkCount, err := testVolumeHandle.GetLinkCount(fileInodeNumber) - if nil != err { - t.Fatalf("GetLinkCount(fileInodeNumber) failed: %v", err) - } - if 0 != fileLinkCount { - t.Fatalf("GetLinkCount(fileInodeNumber) returned unexpected linkCount: %v", fileLinkCount) - } - - postMetadata, err := testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - if FileType != postMetadata.InodeType { - t.Fatalf("GetMetadata(fileInodeNumber) returned unexpected InodeType") - } - checkMetadata(t, postMetadata, testMetadata, MetadataLinkCountField, "GetMetadata() after CreateFile()") - - // TODO: Add more tests related to CreateFile(): mode > 0777, etc. - - preMetadata, err := testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - time.Sleep(time.Millisecond) - - testMetadata.Mode = PosixModePerm - 1 - - err = testVolumeHandle.SetPermMode(fileInodeNumber, testMetadata.Mode) - if nil != err { - t.Fatalf("SetPermMode(fileInodeNumber, %v) failed: %v", testMetadata.Mode, err) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataPermFields, "GetMetadata() after SetPermMode()") - checkMetadataTimeChanges(t, preMetadata, postMetadata, false, false, false, true, "SetPermMode()") - - preMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - time.Sleep(time.Millisecond) - - testMetadata.UserID = 9753 - - err = testVolumeHandle.SetOwnerUserID(fileInodeNumber, testMetadata.UserID) - if nil != err { - t.Fatalf("SetOwnerUserID(fileInodeNumber, %v) failed: %v", testMetadata.UserID, err) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataPermFields, "GetMetadata() after SetOwnerUserID()") - checkMetadataTimeChanges(t, preMetadata, postMetadata, false, false, false, true, "SetOwnerUserID()") - - preMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - time.Sleep(time.Millisecond) - - testMetadata.GroupID = 12345678 - - err = testVolumeHandle.SetOwnerGroupID(fileInodeNumber, testMetadata.GroupID) - if nil != err { - t.Fatalf("SetOwnerGroupID(fileInodeNumber, %v) failed: %v", testMetadata.GroupID, err) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataPermFields, "GetMetadata() after SetOwnerGroupID()") - checkMetadataTimeChanges(t, preMetadata, postMetadata, false, false, false, true, "SetOwnerGroupID()") - - preMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - time.Sleep(time.Millisecond) - - testMetadata.UserID = 3579 - testMetadata.GroupID = 87654321 - - err = testVolumeHandle.SetOwnerUserIDGroupID(fileInodeNumber, testMetadata.UserID, testMetadata.GroupID) - if nil != err { - t.Fatalf("SetOwnerUserIDGroupID(fileInodeNumber, %v, %v) failed: %v", testMetadata.UserID, testMetadata.GroupID, err) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataPermFields, "GetMetadata() after SetOwnerUserIDGroupID()") - checkMetadataTimeChanges(t, preMetadata, postMetadata, false, false, false, true, "SetOwnerUserIDGroupID()") - - err = testVolumeHandle.Link(RootDirInodeNumber, "link_1_to_file_inode", fileInodeNumber, false) - if nil != err { - t.Fatalf("Link(RootDirInodeNumber, \"link_1_to_file_inode\", fileInodeNumber, false) failed: %v", err) - } - - numEntries, err = testVolumeHandle.NumDirEntries(RootDirInodeNumber) - - if nil != err { - t.Fatalf("NumDirEntries(RootDirInodeNumber) failed: %v", err) - } - if 3 != numEntries { - t.Fatalf("NumDirEntries(RootDirInodeNumber) should have returned numEntries == 3") - } - - fileLinkCount, err = testVolumeHandle.GetLinkCount(fileInodeNumber) - if nil != err { - t.Fatalf("GetLinkCount(fileInodeNumber) failed: %v", err) - } - if 1 != fileLinkCount { - t.Fatalf("GetLinkCount(fileInodeNumber) returned unexpected linkCount: %v", fileLinkCount) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - testMetadata.LinkCount = 1 - checkMetadata(t, postMetadata, testMetadata, MetadataLinkCountField, "GetMetadata() after Link()") - - err = testVolumeHandle.Link(RootDirInodeNumber, "link_2_to_file_inode", fileInodeNumber, false) - if nil != err { - t.Fatalf("Link(RootDirInodeNumber, \"link_2_to_file_inode\", fileInodeNumber, false) failed: %v", err) - } - - numEntries, err = testVolumeHandle.NumDirEntries(RootDirInodeNumber) - - if nil != err { - t.Fatalf("NumDirEntries(RootDirInodeNumber) failed: %v", err) - } - if 4 != numEntries { - t.Fatalf("NumDirEntries(RootDirInodeNumber) should have returned numEntries == 4") - } - - fileLinkCount, err = testVolumeHandle.GetLinkCount(fileInodeNumber) - if nil != err { - t.Fatalf("GetLinkCount(fileInodeNumber) failed: %v", err) - } - if 2 != fileLinkCount { - t.Fatalf("GetLinkCount(fileInodeNumber) returned unexpected linkCount: %v", fileLinkCount) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - testMetadata.LinkCount = 2 - checkMetadata(t, postMetadata, testMetadata, MetadataLinkCountField, "GetMetadata() after Link()") - - symlinkInodeToLink1ToFileInode, err := testVolumeHandle.CreateSymlink("link_1_to_file_inode", PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateSymlink(\"link_1_to_file_inode\") failed: %v", err) - } - - symlinkInodeToLink1ToFileInodeType, err := testVolumeHandle.GetType(symlinkInodeToLink1ToFileInode) - if nil != err { - t.Fatalf("GetType(symlinkInodeToLink1ToFileInode) failed: %v", err) - } - if SymlinkType != symlinkInodeToLink1ToFileInodeType { - t.Fatalf("GetType(symlinkInodeToLink1ToFileInode) returned unexpected type: %v", symlinkInodeToLink1ToFileInodeType) - } - - symlinkToLink1ToFileInodeLinkCount, err := testVolumeHandle.GetLinkCount(symlinkInodeToLink1ToFileInode) - if nil != err { - t.Fatalf("GetLinkCount(symlinkInodeToLink1ToFileInode) failed: %v", err) - } - if 0 != symlinkToLink1ToFileInodeLinkCount { - t.Fatalf("GetLinkCount(symlinkInodeToLink1ToFileInode) returned unexpected linkCount: %v", symlinkToLink1ToFileInodeLinkCount) - } - - symlinkInodeToLink1ToFileInodeTarget, err := testVolumeHandle.GetSymlink(symlinkInodeToLink1ToFileInode) - if nil != err { - t.Fatalf("GetSymlink(symlinkInodeToLink1ToFileInode) failed: %v", err) - } - if "link_1_to_file_inode" != symlinkInodeToLink1ToFileInodeTarget { - t.Fatalf("GetSymlink(symlinkInodeToLink1ToFileInode) should have returned target == \"%v\"... instead got \"%v\"", "link_1_to_file_inode", symlinkInodeToLink1ToFileInodeTarget) - } - - err = testVolumeHandle.Link(RootDirInodeNumber, "symlink_to_link_1_to_file_inode", symlinkInodeToLink1ToFileInode, false) - if nil != err { - t.Fatalf("Link(RootDirInodeNumber, \"symlink_to_link_1_to_file_inode\", symlinkInodeToLink1ToFileInode, false) failed: %v", err) - } - - numEntries, err = testVolumeHandle.NumDirEntries(RootDirInodeNumber) - - if nil != err { - t.Fatalf("NumDirEntries(RootDirInodeNumber) failed: %v", err) - } - if 5 != numEntries { - t.Fatalf("NumDirEntries(RootDirInodeNumber) should have returned numEntries == 5") - } - - symlinkToLink1ToFileInodeLinkCount, err = testVolumeHandle.GetLinkCount(symlinkInodeToLink1ToFileInode) - if nil != err { - t.Fatalf("GetLinkCount(symlinkInodeToLink1ToFileInode) failed: %v", err) - } - if 1 != symlinkToLink1ToFileInodeLinkCount { - t.Fatalf("GetLinkCount(symlinkInodeToLink1ToFileInode) returned unexpected linkCount: %v", symlinkToLink1ToFileInodeLinkCount) - } - - symlinkInodeToLink1ToFileInodeLookupResult, err := testVolumeHandle.Lookup(RootDirInodeNumber, "symlink_to_link_1_to_file_inode") - if nil != err { - t.Fatalf("Lookup(RootDirInodeNumber, \"symlink_to_link_1_to_file_inode\") failed: %v", err) - } - if symlinkInodeToLink1ToFileInodeLookupResult != symlinkInodeToLink1ToFileInode { - t.Fatalf("Lookup(RootDirInodeNumber, \"symlink_to_link_1_to_file_inode\") returned unexpected InodeNumber") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 1, 0) - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 1, 0) failed: %v", err) - } - if !moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 1, 0) should have returned moreEntries == true") - } - if 1 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 1, 0) should have returned dirEntrySlice with 1 element, got %v elements", len(dirEntrySlice)) - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 1, 0) returned unexpected dirEntrySlice[0]") - } - - // 60 = ".." entry length (8 + (2 + 1) + 2 + 8) + "link_1_to_file_inode" entry length (8 + (20 + 1) + 2 + 8) - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 60, InodeDirLocation(0)) - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, InodeDirLocation(0)) failed: %v", err) - } - if !moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, InodeDirLocation(0)) should have returned moreEntries == true") - } - if 2 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, InodeDirLocation(0)) should have returned dirEntrySlice with 2 elements, got %v elements", len(dirEntrySlice)) - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != "..") || (dirEntrySlice[0].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, InodeDirLocation(0)) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != fileInodeNumber) || (dirEntrySlice[1].Basename != "link_1_to_file_inode") || (dirEntrySlice[1].NextDirLocation != 3) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, InodeDirLocation(0)) returned unexpected dirEntrySlice[1]") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 60, ".") - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, \".\") failed: %v", err) - } - if !moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, \".\") should have returned moreEntries == true") - } - if 2 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, \".\") should have returned dirEntrySlice with 2 elements, got %v elements", len(dirEntrySlice)) - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != "..") || (dirEntrySlice[0].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, \".\") returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != fileInodeNumber) || (dirEntrySlice[1].Basename != "link_1_to_file_inode") || (dirEntrySlice[1].NextDirLocation != 3) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 60, \".\") returned unexpected dirEntrySlice[1]") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(2)) - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(2)) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(2)) should have returned moreEntries == false") - } - if 2 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(2)) should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != fileInodeNumber) || (dirEntrySlice[0].Basename != "link_2_to_file_inode") || (dirEntrySlice[0].NextDirLocation != 4) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(2)) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != symlinkInodeToLink1ToFileInode) || (dirEntrySlice[1].Basename != "symlink_to_link_1_to_file_inode") || (dirEntrySlice[1].NextDirLocation != 5) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, InodeDirLocation(2)) returned unexpected dirEntrySlice[1]") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0, "link_1_to_file_inode") - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"link_1_to_file_inode\") failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"link_1_to_file_inode\") should have returned moreEntries == false") - } - if 2 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"link_1_to_file_inode\") should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != fileInodeNumber) || (dirEntrySlice[0].Basename != "link_2_to_file_inode") || (dirEntrySlice[0].NextDirLocation != 4) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"link_1_to_file_inode\") returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != symlinkInodeToLink1ToFileInode) || (dirEntrySlice[1].Basename != "symlink_to_link_1_to_file_inode") || (dirEntrySlice[1].NextDirLocation != 5) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0, \"link_1_to_file_inode\") returned unexpected dirEntrySlice[1]") - } - - toDestroyInodeNumber, err = testVolumeHandle.Unlink(RootDirInodeNumber, "link_1_to_file_inode", false) - if nil != err { - t.Fatalf("Unlink(RootDirInodeNumber, \"link_1_to_file_inode\", false) failed: %v", err) - } - if InodeNumber(0) != toDestroyInodeNumber { - t.Fatalf("Unlink(RootDirInodeNumber, \"link_1_to_file_inode\", false) should have returned toDestroyInodeNumber == 0") - } - fileLinkCount, err = testVolumeHandle.GetLinkCount(fileInodeNumber) - if nil != err { - t.Fatalf("GetLinkCount(fileInodeNumber) failed: %v", err) - } - if 1 != fileLinkCount { - t.Fatalf("GetLinkCount(fileInodeNumber) returned unexpected linkCount: %v", fileLinkCount) - } - - numEntries, err = testVolumeHandle.NumDirEntries(RootDirInodeNumber) - - if nil != err { - t.Fatalf("NumDirEntries(RootDirInodeNumber) failed: %v", err) - } - if 4 != numEntries { - t.Fatalf("NumDirEntries(RootDirInodeNumber) should have returned numEntries == 4") - } - - toDestroyInodeNumber, err = testVolumeHandle.Unlink(RootDirInodeNumber, "link_2_to_file_inode", false) - if nil != err { - t.Fatalf("Unlink(RootDirInodeNumber, \"link_2_to_file_inode\", false) failed: %v", err) - } - if fileInodeNumber != toDestroyInodeNumber { - t.Fatalf("Unlink(RootDirInodeNumber, \"link_2_to_file_inode\", false) should have returned toDestroyInodeNumber == fileInodeNumber") - } - fileLinkCount, err = testVolumeHandle.GetLinkCount(fileInodeNumber) - if nil != err { - t.Fatalf("GetLinkCount(fileInodeNumber) failed: %v", err) - } - if 0 != fileLinkCount { - t.Fatalf("GetLinkCount(fileInodeNumber) returned unexpected linkCount: %v", fileLinkCount) - } - - numEntries, err = testVolumeHandle.NumDirEntries(RootDirInodeNumber) - - if nil != err { - t.Fatalf("NumDirEntries(RootDirInodeNumber) failed: %v", err) - } - if 3 != numEntries { - t.Fatalf("NumDirEntries(RootDirInodeNumber) should have returned numEntries == 3") - } - - toDestroyInodeNumber, err = testVolumeHandle.Unlink(RootDirInodeNumber, "symlink_to_link_1_to_file_inode", false) - if nil != err { - t.Fatalf("Unlink(RootDirInodeNumber, \"symlink_to_link_1_to_file_inode\", false) failed: %v", err) - } - if symlinkInodeToLink1ToFileInode != toDestroyInodeNumber { - t.Fatalf("Unlink(RootDirInodeNumber, \"symlink_to_link_1_to_file_inode\", false) should have returned toDestroyInodeNumber == symlinkInodeToLink1ToFileInode") - } - - numEntries, err = testVolumeHandle.NumDirEntries(RootDirInodeNumber) - - if nil != err { - t.Fatalf("NumDirEntries(RootDirInodeNumber) failed: %v", err) - } - if 2 != numEntries { - t.Fatalf("NumDirEntries(RootDirInodeNumber) should have returned numEntries == 2") - } - - err = testVolumeHandle.Destroy(symlinkInodeToLink1ToFileInode) - if nil != err { - t.Fatalf("Destroy(symlinkInodeToLink1ToFileInode) failed: %v", err) - } - - err = testVolumeHandle.Flush(fileInodeNumber, false) - if nil != err { - t.Fatalf("Flush(fileInodeNumber, false) failed: %v", err) - } - err = testVolumeHandle.Purge(fileInodeNumber) - if nil != err { - t.Fatalf("Purge(fileInodeNumber) failed: %v", err) - } - - err = testVolumeHandle.PutStream(fileInodeNumber, "test_stream", []byte{0x00, 0x01, 0x02}) - if nil != err { - t.Fatalf("PutStream(fileInodeNumber, \"test_stream\", []byte{0x00, 0x01, 0x02}) failed: %v", err) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - if 1 != len(postMetadata.InodeStreamNameSlice) { - t.Fatalf("GetMetadata(fileInodeNumber) should have returned postMetadata.InodeStreamNameSlice with 1 element") - } - if postMetadata.InodeStreamNameSlice[0] != "test_stream" { - t.Fatalf("GetMetadata(fileInodeNumber) returned unexpected postMetadata.InodeStreamNameSlice[0]") - } - - testStreamData, err := testVolumeHandle.GetStream(fileInodeNumber, "test_stream") - if nil != err { - t.Fatalf("GetStream(fileInodeNumber, \"test_stream\") failed: %v", err) - } - if 0 != bytes.Compare(testStreamData, []byte{0x00, 0x01, 0x02}) { - t.Fatalf("GetStream(fileInodeNumber, \"test_stream\") returned unexpected testStreamData") - } - - err = testVolumeHandle.DeleteStream(fileInodeNumber, "test_stream") - if nil != err { - t.Fatalf("DeleteStream(fileInodeNumber, \"test_stream\") failed: %v", err) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - if 0 != len(postMetadata.InodeStreamNameSlice) { - t.Fatalf("GetMetadata(fileInodeNumber) should have returned postMetadata.InodeStreamNameSlice with 0 elements") - } - - testMetadata.CreationTime = time.Now().Add(time.Hour) - err = testVolumeHandle.SetCreationTime(fileInodeNumber, testMetadata.CreationTime) - if nil != err { - t.Fatalf("SetCreationTime(fileInodeNumber, oneHourFromNow) failed: %v", err) - } - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataCrTimeField, - "GetMetadata() after SetCreationTime() test 1") - - testMetadata.ModificationTime = time.Now().Add(2 * time.Hour) - err = testVolumeHandle.SetModificationTime(fileInodeNumber, testMetadata.ModificationTime) - if nil != err { - t.Fatalf("SetModificationTime(fileInodeNumber, twoHoursFromNow) failed: %v", err) - } - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataModTimeField, "GetMetadata() after SetModificationTime()") - - checkMetadata(t, postMetadata, testMetadata, MetadataNumWritesField, "GetMetadata() before Write()") - err = testVolumeHandle.Write(fileInodeNumber, 0, []byte{0x00, 0x01, 0x02, 0x03, 0x04}, nil) - if nil != err { - t.Fatalf("Write(fileInodeNumber, 0, []byte{0x00, 0x01, 0x02, 0x03, 0x04}) failed: %v", err) - } - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - testMetadata.NumWrites = 1 - checkMetadata(t, postMetadata, testMetadata, MetadataNumWritesField, "GetMetadata() after Write()") - - err = testVolumeHandle.Flush(fileInodeNumber, false) - if err != nil { - t.Fatalf("Flush(fileInodeNumber) failed: %v", err) - } - - fileInodeObjectPath, err := testVolumeHandle.ProvisionObject() - if nil != err { - t.Fatalf("ProvisionObjectPath() failed: %v", err) - } - - accountName, containerName, objectName, err := utils.PathToAcctContObj(fileInodeObjectPath) - if err != nil { - t.Fatalf("couldn't parse %v as object path", fileInodeObjectPath) - } - - putContext, err := swiftclient.ObjectFetchChunkedPutContext(accountName, containerName, objectName, "") - - if err != nil { - t.Fatalf("fetching chunked put context failed") - } - err = putContext.SendChunk([]byte{0x11, 0x22, 0x33}) - if err != nil { - t.Fatalf("sending chunk failed") - } - err = putContext.Close() - if err != nil { - t.Fatalf("closing chunked PUT failed") - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataNumWritesField, "GetMetadata() before Wrote(,,,,,,,true)") - err = testVolumeHandle.Wrote(fileInodeNumber, containerName, objectName, []uint64{1}, []uint64{0}, []uint64{3}, time.Now(), true) - if nil != err { - t.Fatalf("Wrote(fileInodeNumber, containerName, objectName, []uint64{1}, []uint64{0}, []uint64{3}, time.Now(), true) failed: %v", err) - } - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - testMetadata.NumWrites = 2 - checkMetadata(t, postMetadata, testMetadata, MetadataNumWritesField, "GetMetadata() after Wrote(,,,,,,,true)") - - testFileInodeData, err := testVolumeHandle.Read(fileInodeNumber, 0, 5, nil) - if nil != err { - t.Fatalf("Read(fileInodeNumber, 0, 5) failed: %v", err) - } - if 0 != bytes.Compare(testFileInodeData, []byte{0x00, 0x11, 0x22, 0x33, 0x04}) { - t.Fatalf("Read(fileInodeNumber, 0, 5) returned unexpected testFileInodeData") - } - - offset := uint64(0) - length := uint64(5) - testReadPlan, err := testVolumeHandle.GetReadPlan(fileInodeNumber, &offset, &length) - if nil != err { - t.Fatalf("GetReadPlan(fileInodeNumber, 0, 5) failed: %v", err) - } - if 3 != len(testReadPlan) { - t.Fatalf("GetReadPlan(fileInodeNumber, 0, 5) should have returned testReadPlan with 3 elements") - } - if (1 != testReadPlan[0].Length) || (fileInodeObjectPath != testReadPlan[1].ObjectPath) || (0 != testReadPlan[1].Offset) || (3 != testReadPlan[1].Length) || (1 != testReadPlan[2].Length) { - t.Fatalf("GetReadPlan(fileInodeNumber, 0, 5) returned unexpected testReadPlan") - } - - testExtentMapChunk, err := testVolumeHandle.FetchExtentMapChunk(fileInodeNumber, uint64(2), 2, 1) - if nil != err { - t.Fatalf("FetchExtentMap(fileInodeNumber, uint64(2), 2, 1) failed: %v", err) - } - if 0 != testExtentMapChunk.FileOffsetRangeStart { - t.Fatalf("FetchExtentMap(fileInodeNumber, uint64(2), 2, 1) should have returned testExtentMapChunk.FileOffsetRangeStart == 0") - } - if 5 != testExtentMapChunk.FileOffsetRangeEnd { - t.Fatalf("FetchExtentMap(fileInodeNumber, uint64(2), 2, 1) should have returned testExtentMapChunk.FileOffsetRangeEnd == 5") - } - if 5 != testExtentMapChunk.FileSize { - t.Fatalf("FetchExtentMap(fileInodeNumber, uint64(2), 2, 1) should have returned testExtentMapChunk.FileSize == 5") - } - if 3 != len(testExtentMapChunk.ExtentMapEntry) { - t.Fatalf("FetchExtentMap(fileInodeNumber, uint64(2), 2, 1) should have returned testExtentMapChunk.ExtentMapEntry slice with 3 elements") - } - fileInodeObjectPathSplit := strings.Split(fileInodeObjectPath, "/") - fileInodeObjectPathContainerName := fileInodeObjectPathSplit[len(fileInodeObjectPathSplit)-2] - fileInodeObjectPathObjectName := fileInodeObjectPathSplit[len(fileInodeObjectPathSplit)-1] - if (0 != testExtentMapChunk.ExtentMapEntry[0].FileOffset) || - (0 != testExtentMapChunk.ExtentMapEntry[0].LogSegmentOffset) || - (1 != testExtentMapChunk.ExtentMapEntry[0].Length) || - (1 != testExtentMapChunk.ExtentMapEntry[1].FileOffset) || - (0 != testExtentMapChunk.ExtentMapEntry[1].LogSegmentOffset) || - (3 != testExtentMapChunk.ExtentMapEntry[1].Length) || - (fileInodeObjectPathContainerName != testExtentMapChunk.ExtentMapEntry[1].ContainerName) || - (fileInodeObjectPathObjectName != testExtentMapChunk.ExtentMapEntry[1].ObjectName) || - (4 != testExtentMapChunk.ExtentMapEntry[2].FileOffset) || - (4 != testExtentMapChunk.ExtentMapEntry[2].LogSegmentOffset) || - (1 != testExtentMapChunk.ExtentMapEntry[2].Length) { - t.Fatalf("FetchExtentMap(fileInodeNumber, uint64(2), 2, 1) returned unexpected testExtentMapChunk.ExtentMapEntry slice") - } - - // Suffix byte range query, like "Range: bytes=-2" - length = uint64(3) - testSuffixReadPlan, err := testVolumeHandle.GetReadPlan(fileInodeNumber, nil, &length) - if nil != err { - t.Fatalf("GetReadPlan(fileInodeNumber, nil, 3) failed: %v", err) - } - bytesFromReadPlan := []byte{} - for _, rps := range testSuffixReadPlan { - pathParts := strings.SplitN(rps.ObjectPath, "/", 5) - b, err := swiftclient.ObjectGet(pathParts[2], pathParts[3], pathParts[4], rps.Offset, rps.Length) - if err != nil { - t.Fatalf("ObjectGet() returned unexpected error %v", err) - } - bytesFromReadPlan = append(bytesFromReadPlan, b...) - } - if !bytes.Equal(bytesFromReadPlan, []byte{0x22, 0x33, 0x04}) { - t.Fatalf("expected bytes from suffix read plan 0x22, 0x33, 0x04; got %v", bytesFromReadPlan) - } - - // Long suffix byte range query, like "Range: bytes=-200" (for a file of size < 200) - length = uint64(10) // our file has only 5 bytes - testSuffixReadPlan, err = testVolumeHandle.GetReadPlan(fileInodeNumber, nil, &length) - if nil != err { - t.Fatalf("GetReadPlan(fileInodeNumber, nil, 10) failed: %v", err) - } - bytesFromReadPlan = []byte{} - for _, rps := range testSuffixReadPlan { - pathParts := strings.SplitN(rps.ObjectPath, "/", 5) - b, err := swiftclient.ObjectGet(pathParts[2], pathParts[3], pathParts[4], rps.Offset, rps.Length) - if err != nil { - t.Fatalf("ObjectGet() returned unexpected error %v", err) - } - bytesFromReadPlan = append(bytesFromReadPlan, b...) - } - if !bytes.Equal(bytesFromReadPlan, []byte{0x00, 0x11, 0x22, 0x33, 0x04}) { - t.Fatalf("expected bytes from suffix read plan 0x00, 0x11, 0x22, 0x33, 0x04; got %v", bytesFromReadPlan) - } - - // Prefix byte range query, like "Range: bytes=4-" - offset = uint64(3) - testPrefixReadPlan, err := testVolumeHandle.GetReadPlan(fileInodeNumber, &offset, nil) - if nil != err { - t.Fatalf("GetReadPlan(fileInodeNumber, 3, nil) failed: %v", err) - } - bytesFromReadPlan = []byte{} - for _, rps := range testPrefixReadPlan { - pathParts := strings.SplitN(rps.ObjectPath, "/", 5) - b, err := swiftclient.ObjectGet(pathParts[2], pathParts[3], pathParts[4], rps.Offset, rps.Length) - if err != nil { - t.Fatalf("ObjectGet() returned unexpected error %v", err) - } - bytesFromReadPlan = append(bytesFromReadPlan, b...) - } - if !bytes.Equal(bytesFromReadPlan, []byte{0x33, 0x04}) { - t.Fatalf("expected bytes from prefix read plan 0x33, 0x04; got %v", bytesFromReadPlan) - } - - accountName, containerName, objectName, err = utils.PathToAcctContObj(testReadPlan[0].ObjectPath) - if nil != err { - t.Fatalf("expected %q to be a legal object path", testReadPlan[0].ObjectPath) - } - - headSlice, err := swiftclient.ObjectGet(accountName, containerName, objectName, testReadPlan[0].Offset, 1) - if nil != err { - t.Fatalf("ObjectGet() returned unexpected error: %v", err) - } - if 0 != bytes.Compare(headSlice, []byte{0x00}) { - t.Fatalf("expected byte of %v to be 0x00, got %x", testReadPlan[0].ObjectPath, headSlice) - } - - accountName, containerName, objectName, err = utils.PathToAcctContObj(testReadPlan[1].ObjectPath) - if nil != err { - t.Fatalf("expected %q to be a legal object path", testReadPlan[1].ObjectPath) - } - middleSlice, err := swiftclient.ObjectGet(accountName, containerName, objectName, testReadPlan[1].Offset, 3) - if nil != err { - t.Fatalf("ObjectGet() returned unexpected error: %v", err) - } - if 0 != bytes.Compare(middleSlice, []byte{0x11, 0x22, 0x33}) { - t.Fatalf("expected byte of %v to be 0x00, got %x", testReadPlan[0].ObjectPath, headSlice) - } - - accountName, containerName, objectName, err = utils.PathToAcctContObj(testReadPlan[2].ObjectPath) - if nil != err { - t.Fatalf("expected %q to be a legal object path", testReadPlan[2].ObjectPath) - } - tailSlice, err := swiftclient.ObjectGet(accountName, containerName, objectName, testReadPlan[2].Offset, 1) - if nil != err { - t.Fatalf("objectContext.Get() returned unexpected error: %v", err) - } - if 0 != bytes.Compare(tailSlice, []byte{0x04}) { - t.Fatalf("expected byte of %v to be 0x04, got %x", testReadPlan[0].ObjectPath, tailSlice) - } - - fileInodeObjectPath, err = testVolumeHandle.ProvisionObject() - if nil != err { - t.Fatalf("ProvisionObjectPath() failed: %v", err) - } - - accountName, containerName, objectName, err = utils.PathToAcctContObj(fileInodeObjectPath) - if err != nil { - t.Fatalf("couldn't parse %v as object path", fileInodeObjectPath) - } - - putContext, err = swiftclient.ObjectFetchChunkedPutContext(accountName, containerName, objectName, "") - if err != nil { - t.Fatalf("fetching chunked put context failed") - } - err = putContext.SendChunk([]byte{0xFE, 0xFF}) - if err != nil { - t.Fatalf("sending chunk failed") - } - err = putContext.Close() - if err != nil { - t.Fatalf("closing chunked PUT failed") - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataNumWritesField, "GetMetadata() before Wrote(,,,,,,,false)") - err = testVolumeHandle.Wrote(fileInodeNumber, containerName, objectName, []uint64{0}, []uint64{0}, []uint64{2}, time.Now(), false) - if nil != err { - t.Fatalf("Wrote(fileInodeNumber, containerName, objectName, []uint64{0}, []uint64{0}, []uint64{2}, time.Now(), false) failed: %v", err) - } - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - testMetadata.NumWrites = 1 - checkMetadata(t, postMetadata, testMetadata, MetadataNumWritesField, "GetMetadata() after Wrote(,,,,,,,false)") - - testFileInodeData, err = testVolumeHandle.Read(fileInodeNumber, 0, 4, nil) - if nil != err { - t.Fatalf("Read(fileInodeNumber, 0, 4) failed: %v", err) - } - if 0 != bytes.Compare(testFileInodeData, []byte{0xFE, 0xFF}) { - t.Fatalf("Read(fileInodeNumber, 0, 4) returned unexpected testFileInodeData") - } - - preResizeMetadata, err := testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - - testMetadata.Size = 0 - err = testVolumeHandle.SetSize(fileInodeNumber, testMetadata.Size) - if nil != err { - t.Fatalf("SetSize(fileInodeNumber, 0) failed: %v", err) - } - - postMetadata, err = testVolumeHandle.GetMetadata(fileInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(fileInodeNumber) failed: %v", err) - } - checkMetadata(t, postMetadata, testMetadata, MetadataSizeField, "GetMetadata() after SetSize()") - checkMetadataTimeChanges(t, preResizeMetadata, postMetadata, false, true, false, true, "SetSize changed metadata times inappropriately") - - err = testVolumeHandle.Flush(fileInodeNumber, true) - if nil != err { - t.Fatalf("Flush(fileInodeNumber, true) failed: %v", err) - } - - err = testVolumeHandle.Destroy(fileInodeNumber) - if nil != err { - t.Fatalf("Destroy(fileInodeNumber) failed: %v", err) - } - - file1Inode, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - file2Inode, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - err = testVolumeHandle.Link(RootDirInodeNumber, "1stLocation", file1Inode, false) - if nil != err { - t.Fatalf("Link(RootDirInodeNumber, \"1stLocation\", file1Inode, false) failed: %v", err) - } - - err = testVolumeHandle.Link(RootDirInodeNumber, "3rdLocation", file2Inode, false) - if nil != err { - t.Fatalf("Link(RootDirInodeNumber, \"3rdLocation\", file2Inode, false) failed: %v", err) - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0) - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned moreEntries == false") - } - if 4 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[1]") - } - if (dirEntrySlice[2].InodeNumber != file1Inode) || (dirEntrySlice[2].Basename != "1stLocation") || (dirEntrySlice[2].NextDirLocation != 3) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[2]") - } - if (dirEntrySlice[3].InodeNumber != file2Inode) || (dirEntrySlice[3].Basename != "3rdLocation") || (dirEntrySlice[3].NextDirLocation != 4) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[2]") - } - - toDestroyInodeNumber, err = testVolumeHandle.Move(RootDirInodeNumber, "1stLocation", RootDirInodeNumber, "2ndLocation") - if nil != err { - t.Fatalf("Move(RootDirInodeNumber, \"1stLocation\", RootDirInodeNumber, \"2ndLocation\") failed: %v", err) - } - if InodeNumber(0) != toDestroyInodeNumber { - t.Fatalf("Move(RootDirInodeNumber, \"1stLocation\", RootDirInodeNumber, \"2ndLocation\") should have returned toDestroyInodeNumber == 0") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0) - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned moreEntries == false") - } - if 4 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[1]") - } - if (dirEntrySlice[2].InodeNumber != file1Inode) || (dirEntrySlice[2].Basename != "2ndLocation") || (dirEntrySlice[2].NextDirLocation != 3) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[2]") - } - if (dirEntrySlice[3].InodeNumber != file2Inode) || (dirEntrySlice[3].Basename != "3rdLocation") || (dirEntrySlice[3].NextDirLocation != 4) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[2]") - } - - toDestroyInodeNumber, err = testVolumeHandle.Unlink(RootDirInodeNumber, "3rdLocation", false) - if nil != err { - t.Fatalf("Unlink(RootDirInodeNumber, \"3rdLocation\", false) failed: %v", err) - } - if file2Inode != toDestroyInodeNumber { - t.Fatalf("Unlink(RootDirInodeNumber, \"3rdLocation\", false) should have returned toDestroyInodeNumber == file2Inode") - } - toDestroyInodeNumber, err = testVolumeHandle.Move(RootDirInodeNumber, "2ndLocation", RootDirInodeNumber, "3rdLocation") - if nil != err { - t.Fatalf("Move(RootDirInodeNumber, \"2ndLocation\", RootDirInodeNumber, \"3rdLocation\") failed: %v", err) - } - if InodeNumber(0) != toDestroyInodeNumber { - t.Fatalf("Move(RootDirInodeNumber, \"2ndLocation\", RootDirInodeNumber, \"3rdLocation\") should have returned toDestroyInodeNumber == 0") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0) - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned moreEntries == false") - } - if 3 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[1]") - } - if (dirEntrySlice[2].InodeNumber != file1Inode) || (dirEntrySlice[2].Basename != "3rdLocation") || (dirEntrySlice[2].NextDirLocation != 3) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[2]") - } - - err = testVolumeHandle.Destroy(file2Inode) - if nil != err { - t.Fatalf("Destroy(file2Inode) failed: %v", err) - } - - postMetadata, err = testVolumeHandle.GetMetadata(RootDirInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(RootDirInodeNumber) failed: %v", err) - } - testMetadata.LinkCount = 2 - checkMetadata(t, postMetadata, testMetadata, MetadataLinkCountField, "GetMetadata() after multiple calls to Link()") - - subDirInode, err := testVolumeHandle.CreateDir(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateDir() failed: %v", err) - } - - subDirInodeType, err := testVolumeHandle.GetType(subDirInode) - if nil != err { - t.Fatalf("GetType(subDirInode) failed: %v", err) - } - if DirType != subDirInodeType { - t.Fatalf("GetType(subDirInode) returned unexpected type: %v", subDirInodeType) - } - - subDirLinkCount, err := testVolumeHandle.GetLinkCount(subDirInode) - if nil != err { - t.Fatalf("GetLinkCount(subDirInode) failed: %v", err) - } - if 1 != subDirLinkCount { - t.Fatalf("GetLinkCount(subDirInode) returned unexpected linkCount: %v", subDirLinkCount) - } - - err = testVolumeHandle.SetLinkCount(subDirInode, uint64(2)) - if nil != err { - t.Fatalf("SetLinkCount(subDirInode, uint64(2)) failed: %v", err) - } - - subDirLinkCount, err = testVolumeHandle.GetLinkCount(subDirInode) - if nil != err { - t.Fatalf("GetLinkCount(subDirInode) failed: %v", err) - } - if 2 != subDirLinkCount { - t.Fatalf("GetLinkCount(subDirInode) returned unexpected linkCount: %v", subDirLinkCount) - } - - err = testVolumeHandle.SetLinkCount(subDirInode, uint64(1)) - if nil != err { - t.Fatalf("SetLinkCount(subDirInode, uint64(1)) failed: %v", err) - } - - subDirLinkCount, err = testVolumeHandle.GetLinkCount(subDirInode) - if nil != err { - t.Fatalf("GetLinkCount(subDirInode) failed: %v", err) - } - if 1 != subDirLinkCount { - t.Fatalf("GetLinkCount(subDirInode) returned unexpected linkCount: %v", subDirLinkCount) - } - - // it should be illegal to link to a directory - err = testVolumeHandle.Link(RootDirInodeNumber, "subDir", subDirInode, false) - if nil != err { - t.Fatalf("Link(RootDirInodeNumber, \"subDir\", subDirInode, false) failed: %v", err) - } - - rootDirLinkCount, err = testVolumeHandle.GetLinkCount(RootDirInodeNumber) - if nil != err { - t.Fatalf("GetLinkCount(RootDirInodeNumber) failed: %v", err) - } - if 3 != rootDirLinkCount { - t.Fatalf("GetLinkCount(RootDirInodeNumber) returned unexpected linkCount: %v", rootDirLinkCount) - } - - subDirLinkCount, err = testVolumeHandle.GetLinkCount(subDirInode) - if nil != err { - t.Fatalf("GetLinkCount(subDirInode) failed: %v", err) - } - if 2 != subDirLinkCount { - t.Fatalf("GetLinkCount(subDirInode) returned unexpected linkCount: %v", subDirLinkCount) - } - - postMetadata, err = testVolumeHandle.GetMetadata(RootDirInodeNumber) - if nil != err { - t.Fatalf("GetMetadata(RootDirInodeNumber) failed: %v", err) - } - testMetadata.LinkCount = 3 - checkMetadata(t, postMetadata, testMetadata, MetadataLinkCountField, "GetMetadata() after Link()") - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0) - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned moreEntries == false") - } - if 4 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned dirEntrySlice with 2 elements, got %#v, which is of length %v", dirEntrySlice, len(dirEntrySlice)) - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[1]") - } - if (dirEntrySlice[2].InodeNumber != file1Inode) || (dirEntrySlice[2].Basename != "3rdLocation") || (dirEntrySlice[2].NextDirLocation != 3) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[2]") - } - if (dirEntrySlice[3].InodeNumber != subDirInode) || (dirEntrySlice[3].Basename != "subDir") || (dirEntrySlice[3].NextDirLocation != 4) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[3]") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(subDirInode, 0, 0) - if nil != err { - t.Fatalf("ReadDir(subDirInode, 0, 0) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(subDirInode, 0, 0) should have returned moreEntries == false") - } - if 2 != len(dirEntrySlice) { - t.Fatalf("ReadDir(subDirInode, 0, 0) should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != subDirInode) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(subDirInode, 0, 0) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(subDirInode, 0, 0) returned unexpected dirEntrySlice[1]") - } - - toDestroyInodeNumber, err = testVolumeHandle.Move(RootDirInodeNumber, "3rdLocation", subDirInode, "4thLocation") - if nil != err { - t.Fatalf("Move(RootDirInodeNumber, \"3rdLocation\", subDirInode, \"4thLocation\") failed: %v", err) - } - if InodeNumber(0) != toDestroyInodeNumber { - t.Fatalf("Move(RootDirInodeNumber, \"3rdLocation\", subDirInode, \"4thLocation\") should have returned toDestroyInodeNumber == 0") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(RootDirInodeNumber, 0, 0) - if nil != err { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned moreEntries == false") - } - if 3 != len(dirEntrySlice) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) should have returned dirEntrySlice with 2 elements, got %#v", dirEntrySlice) - } - if (dirEntrySlice[0].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[1]") - } - if (dirEntrySlice[2].InodeNumber != subDirInode) || (dirEntrySlice[2].Basename != "subDir") || (dirEntrySlice[2].NextDirLocation != 3) { - t.Fatalf("ReadDir(RootDirInodeNumber, 0, 0) returned unexpected dirEntrySlice[2]") - } - - dirEntrySlice, moreEntries, err = testVolumeHandle.ReadDir(subDirInode, 0, 0) - if nil != err { - t.Fatalf("ReadDir(subDirInode, 0, 0) failed: %v", err) - } - if moreEntries { - t.Fatalf("ReadDir(subDirInode, 0, 0) should have returned moreEntries == false") - } - if 3 != len(dirEntrySlice) { - t.Fatalf("ReadDir(subDirInode, 0, 0) should have returned dirEntrySlice with 2 elements") - } - if (dirEntrySlice[0].InodeNumber != subDirInode) || (dirEntrySlice[0].Basename != ".") || (dirEntrySlice[0].NextDirLocation != 1) { - t.Fatalf("ReadDir(subDirInode, 0, 0) returned unexpected dirEntrySlice[0]") - } - if (dirEntrySlice[1].InodeNumber != RootDirInodeNumber) || (dirEntrySlice[1].Basename != "..") || (dirEntrySlice[1].NextDirLocation != 2) { - t.Fatalf("ReadDir(subDirInode, 0, 0) returned unexpected dirEntrySlice[1]") - } - if (dirEntrySlice[2].InodeNumber != file1Inode) || (dirEntrySlice[2].Basename != "4thLocation") || (dirEntrySlice[2].NextDirLocation != 3) { - t.Fatalf("ReadDir(subDirInode, 0, 0) returned unexpected dirEntrySlice[2]") - } - - toDestroyInodeNumber, err = testVolumeHandle.Unlink(subDirInode, "4thLocation", false) - if nil != err { - t.Fatalf("Unlink(subDirInode, \"4thLocation\", false) failed: %v", err) - } - if file1Inode != toDestroyInodeNumber { - t.Fatalf("Unlink(subDirInode, \"4thLocation\", false) should have returned toDestroyInodeNumber == file1Inode") - } - - err = testVolumeHandle.Destroy(file1Inode) - if nil != err { - t.Fatalf("Destroy(file1Inode) failed: %v", err) - } - - toDestroyInodeNumber, err = testVolumeHandle.Unlink(RootDirInodeNumber, "subDir", false) - if nil != err { - t.Fatalf("Unlink(RootDirInodeNumber, \"subDir\", false) failed: %v", err) - } - if subDirInode != toDestroyInodeNumber { - t.Fatalf("Unlink(RootDirInodeNumber, \"subDir\", false) should have returned toDestroyInodeNumber == subDirInode") - } - - rootDirLinkCount, err = testVolumeHandle.GetLinkCount(RootDirInodeNumber) - if nil != err { - t.Fatalf("GetLinkCount(RootDirInodeNumber) failed: %v", err) - } - if 2 != rootDirLinkCount { - t.Fatalf("GetLinkCount(RootDirInodeNumber) returned unexpected linkCount: %v", rootDirLinkCount) - } - - subDirLinkCount, err = testVolumeHandle.GetLinkCount(subDirInode) - if nil != err { - t.Fatalf("GetLinkCount(subDirInode) failed: %v", err) - } - if 1 != subDirLinkCount { - t.Fatalf("GetLinkCount(subDirInode) returned unexpected linkCount: %v", subDirLinkCount) - } - - err = testVolumeHandle.Destroy(subDirInode) - if nil != err { - t.Fatalf("Destroy(subDirInode) failed: %v", err) - } - - positiveDurationToDelayOrSkew, err := time.ParseDuration("+" + durationToDelayOrSkew) - if nil != err { - t.Fatalf("time.ParseDuration(\"100ms\") failed: %v", err) - } - negativeDurationToDelayOrSkew, err := time.ParseDuration("-" + durationToDelayOrSkew) - if nil != err { - t.Fatalf("time.ParseDuration(\"-100ms\") failed: %v", err) - } - - timeBeforeCreateDir := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - testMetadata.Mode = InodeMode(0744) - testMetadata.UserID = InodeUserID(987) - testMetadata.GroupID = InodeGroupID(654) - dirInode, err := testVolumeHandle.CreateDir(testMetadata.Mode, testMetadata.UserID, testMetadata.GroupID) - if nil != err { - t.Fatalf("CreateDir() failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterCreateDir := time.Now() - - dirInodeMetadataAfterCreateDir, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - if dirInodeMetadataAfterCreateDir.CreationTime.Before(timeBeforeCreateDir) || dirInodeMetadataAfterCreateDir.CreationTime.After(timeAfterCreateDir) { - t.Fatalf("dirInodeMetadataAfterCreateDir.CreationTime unexpected") - } - if !dirInodeMetadataAfterCreateDir.ModificationTime.Equal(dirInodeMetadataAfterCreateDir.CreationTime) { - t.Fatalf("dirInodeMetadataAfterCreateDir.ModificationTime unexpected") - } - if !dirInodeMetadataAfterCreateDir.AccessTime.Equal(dirInodeMetadataAfterCreateDir.CreationTime) { - t.Fatalf("dirInodeMetadataAfterCreateDir.AccessTime unexpected") - } - if !dirInodeMetadataAfterCreateDir.AttrChangeTime.Equal(dirInodeMetadataAfterCreateDir.CreationTime) { - t.Fatalf("dirInodeMetadataAfterCreateDir.AttrChangeTime unexpected") - } - checkMetadata(t, dirInodeMetadataAfterCreateDir, testMetadata, MetadataPermFields, "GetMetadata() after CreateDir()") - - // TODO: Add more tests related to mode/uid/gid and CreateDir(): mode > 0777, etc. - - testMetadata.CreationTime = dirInodeMetadataAfterCreateDir.CreationTime.Add(negativeDurationToDelayOrSkew) - testMetadata.ModificationTime = dirInodeMetadataAfterCreateDir.ModificationTime - testMetadata.AccessTime = dirInodeMetadataAfterCreateDir.AccessTime - testMetadata.AttrChangeTime = dirInodeMetadataAfterCreateDir.AttrChangeTime - - timeBeforeOp = time.Now() - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.SetCreationTime(dirInode, testMetadata.CreationTime) - if nil != err { - t.Fatalf("SetCreationTime(dirInode,) failed: %v", err) - } - time.Sleep(positiveDurationToDelayOrSkew) - timeAfterOp = time.Now() - - dirInodeMetadataAfterSetCreationTime, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - checkMetadata(t, dirInodeMetadataAfterSetCreationTime, testMetadata, MetadataNotAttrTimeFields, - "GetMetadata() after SetCreationTime() test 2") - if dirInodeMetadataAfterSetCreationTime.AttrChangeTime.Before(timeBeforeOp) || - dirInodeMetadataAfterSetCreationTime.AttrChangeTime.After(timeAfterOp) { - t.Fatalf("dirInodeMetadataAfterSetCreationTime.AttrChangeTime unexpected") - } - - testMetadata.ModificationTime = dirInodeMetadataAfterSetCreationTime.ModificationTime.Add(negativeDurationToDelayOrSkew) - timeBeforeOp = time.Now() - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.SetModificationTime(dirInode, testMetadata.ModificationTime) - if nil != err { - t.Fatalf("SetModificationTime(dirInode,) failed: %v", err) - } - time.Sleep(positiveDurationToDelayOrSkew) - timeAfterOp = time.Now() - - dirInodeMetadataAfterSetModificationTime, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - checkMetadata(t, dirInodeMetadataAfterSetModificationTime, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after SetModificationTime()") - if dirInodeMetadataAfterSetModificationTime.AttrChangeTime.Before(timeBeforeOp) || - dirInodeMetadataAfterSetModificationTime.AttrChangeTime.After(timeAfterOp) { - t.Fatalf("dirInodeMetadataAfterSetModificationTime.AttrChangeTime unexpected") - } - - testMetadata.AccessTime = dirInodeMetadataAfterSetModificationTime.AccessTime.Add(negativeDurationToDelayOrSkew) - timeBeforeOp = time.Now() - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.SetAccessTime(dirInode, testMetadata.AccessTime) - if nil != err { - t.Fatalf("SetAccessTime(dirInode,) failed: %v", err) - } - time.Sleep(positiveDurationToDelayOrSkew) - timeAfterOp = time.Now() - - dirInodeMetadataAfterSetAccessTime, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - checkMetadata(t, dirInodeMetadataAfterSetAccessTime, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after testVolumeHandle.SetAccessTime()") - if dirInodeMetadataAfterSetAccessTime.AttrChangeTime.Before(timeBeforeOp) || - dirInodeMetadataAfterSetAccessTime.AttrChangeTime.After(timeAfterOp) { - t.Fatalf("dirInodeMetadataAfterSetAccessTime.AttrChangeTime unexpected") - } - - timeBeforeDirInodePutStream := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.PutStream(dirInode, "stream_name", []byte{}) - if nil != err { - t.Fatalf("PutStream(dirInode,,) failed: %v", err) - } - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterDirInodePutStream := time.Now() - - dirInodeMetadataAfterPutStream, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - testMetadata.CreationTime = dirInodeMetadataAfterSetAccessTime.CreationTime - testMetadata.ModificationTime = dirInodeMetadataAfterSetAccessTime.ModificationTime - testMetadata.AccessTime = dirInodeMetadataAfterSetAccessTime.AccessTime - checkMetadata(t, dirInodeMetadataAfterPutStream, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after PutStream()") - if dirInodeMetadataAfterPutStream.AttrChangeTime.Before(timeBeforeDirInodePutStream) || dirInodeMetadataAfterPutStream.AttrChangeTime.After(timeAfterDirInodePutStream) { - t.Fatalf("dirInodeMetadataAfterPutStream.AttrChangeTime unexpected") - } - - timeBeforeDirInodeDeleteStream := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.DeleteStream(dirInode, "stream_name") - if nil != err { - t.Fatalf("DeleteStream(dirInode,) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterDirInodeDeleteStream := time.Now() - - dirInodeMetadataAfterDeleteStream, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - checkMetadata(t, dirInodeMetadataAfterPutStream, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after PutStream()") - if dirInodeMetadataAfterDeleteStream.AttrChangeTime.Before(timeBeforeDirInodeDeleteStream) || dirInodeMetadataAfterDeleteStream.AttrChangeTime.After(timeAfterDirInodeDeleteStream) { - t.Fatalf("dirInodeMetadataAfterDeleteStream.AttrChangeTime unexpected") - } - - timeBeforeCreateFile := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - fileInode, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterCreateFile := time.Now() - - fileInodeMetadataAfterCreateFile, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - if fileInodeMetadataAfterCreateFile.CreationTime.Before(timeBeforeCreateFile) || fileInodeMetadataAfterCreateFile.CreationTime.After(timeAfterCreateFile) { - t.Fatalf("fileInodeMetadataAfterCreateDir.CreationTime unexpected") - } - if !fileInodeMetadataAfterCreateFile.ModificationTime.Equal(fileInodeMetadataAfterCreateFile.CreationTime) { - t.Fatalf("fileInodeMetadataAfterCreateDir.ModificationTime unexpected") - } - if !fileInodeMetadataAfterCreateFile.AccessTime.Equal(fileInodeMetadataAfterCreateFile.CreationTime) { - t.Fatalf("fileInodeMetadataAfterCreateDir.AccessTime unexpected") - } - if !fileInodeMetadataAfterCreateFile.AttrChangeTime.Equal(fileInodeMetadataAfterCreateFile.CreationTime) { - t.Fatalf("fileInodeMetadataAfterCreateDir.AttrChangeTime unexpected") - } - - testMetadata.CreationTime = fileInodeMetadataAfterCreateFile.CreationTime.Add(negativeDurationToDelayOrSkew) - testMetadata.ModificationTime = fileInodeMetadataAfterCreateFile.ModificationTime - testMetadata.AccessTime = fileInodeMetadataAfterCreateFile.AccessTime - testMetadata.AttrChangeTime = fileInodeMetadataAfterCreateFile.AttrChangeTime - - timeBeforeOp = time.Now() - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.SetCreationTime(fileInode, testMetadata.CreationTime) - if nil != err { - t.Fatalf("SetCreationTime(fileInode,) failed: %v", err) - } - time.Sleep(positiveDurationToDelayOrSkew) - timeAfterOp = time.Now() - - fileInodeMetadataAfterSetCreationTime, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - checkMetadata(t, fileInodeMetadataAfterSetCreationTime, testMetadata, MetadataNotAttrTimeFields, - "GetMetadata() after SetCreationTime() test 3") - if fileInodeMetadataAfterSetCreationTime.AttrChangeTime.Before(timeBeforeOp) || - fileInodeMetadataAfterSetCreationTime.AttrChangeTime.After(timeAfterOp) { - t.Fatalf("fileInodeMetadataAfterSetCreationTime.AttrChangeTime unexpected") - } - - testMetadata.ModificationTime = fileInodeMetadataAfterSetCreationTime.ModificationTime.Add(negativeDurationToDelayOrSkew) - err = testVolumeHandle.SetModificationTime(fileInode, testMetadata.ModificationTime) - if nil != err { - t.Fatalf("SetModificationTime(fileInode,) failed: %v", err) - } - fileInodeMetadataAfterSetModificationTime, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - checkMetadata(t, fileInodeMetadataAfterSetModificationTime, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after SetModificationTime()") - - testMetadata.AccessTime = fileInodeMetadataAfterSetModificationTime.AccessTime.Add(negativeDurationToDelayOrSkew) - err = testVolumeHandle.SetAccessTime(fileInode, testMetadata.AccessTime) - if nil != err { - t.Fatalf("SetAccessTime(fileInode,) failed: %v", err) - } - fileInodeMetadataAfterSetAccessTime, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - checkMetadata(t, fileInodeMetadataAfterSetAccessTime, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after SetAccessTime() test 1") - - timeBeforeFileInodePutStream := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.PutStream(fileInode, "stream_name", []byte{}) - if nil != err { - t.Fatalf("PutStream(fileInode,,) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterFileInodePutStream := time.Now() - - fileInodeMetadataAfterPutStream, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - checkMetadata(t, fileInodeMetadataAfterPutStream, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after PutStream()") - if fileInodeMetadataAfterPutStream.AttrChangeTime.Before(timeBeforeFileInodePutStream) || fileInodeMetadataAfterPutStream.AttrChangeTime.After(timeAfterFileInodePutStream) { - t.Fatalf("fileInodeMetadataAfterPutStream.AttrChangeTime unexpected") - } - - timeBeforeFileInodeDeleteStream := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.DeleteStream(fileInode, "stream_name") - if nil != err { - t.Fatalf("DeleteStream(fileInode,) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterFileInodeDeleteStream := time.Now() - - fileInodeMetadataAfterDeleteStream, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - checkMetadata(t, fileInodeMetadataAfterDeleteStream, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after DeleteStream()") - if fileInodeMetadataAfterDeleteStream.AttrChangeTime.Before(timeBeforeFileInodeDeleteStream) || fileInodeMetadataAfterDeleteStream.AttrChangeTime.After(timeAfterFileInodeDeleteStream) { - t.Fatalf("fileInodeMetadataAfterDeleteStream.AttrChangeTime unexpected") - } - - timeBeforeCreateSymlink := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - testMetadata.Mode = PosixModePerm - testMetadata.UserID = InodeUserID(1111) - testMetadata.GroupID = InodeGroupID(7777) - symlinkInode, err := testVolumeHandle.CreateSymlink("nowhere", testMetadata.Mode, testMetadata.UserID, testMetadata.GroupID) - if nil != err { - t.Fatalf("CreateSymlink(\"nowhere\") failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterCreateSymlink := time.Now() - - symlinkInodeMetadataAfterCreateSymlink, err := testVolumeHandle.GetMetadata(symlinkInode) - if nil != err { - t.Fatalf("GetMetadata(symlinkInode) failed: %v", err) - } - if symlinkInodeMetadataAfterCreateSymlink.CreationTime.Before(timeBeforeCreateSymlink) || symlinkInodeMetadataAfterCreateSymlink.CreationTime.After(timeAfterCreateSymlink) { - t.Fatalf("symlinkInodeMetadataAfterCreateDir.CreationTime unexpected") - } - if !symlinkInodeMetadataAfterCreateSymlink.ModificationTime.Equal(symlinkInodeMetadataAfterCreateSymlink.CreationTime) { - t.Fatalf("symlinkInodeMetadataAfterCreateDir.ModificationTime unexpected") - } - if !symlinkInodeMetadataAfterCreateSymlink.AccessTime.Equal(symlinkInodeMetadataAfterCreateSymlink.CreationTime) { - t.Fatalf("symlinkInodeMetadataAfterCreateDir.AccessTime unexpected") - } - if !symlinkInodeMetadataAfterCreateSymlink.AttrChangeTime.Equal(symlinkInodeMetadataAfterCreateSymlink.CreationTime) { - t.Fatalf("symlinkInodeMetadataAfterCreateDir.AttrChangeTime unexpected") - } - checkMetadata(t, symlinkInodeMetadataAfterCreateSymlink, testMetadata, MetadataPermFields, "GetMetadata() after CreateSymlink()") - - // TODO: Add more tests related to mode/uid/gid and CreateFile(): mode > 0777, etc. - - testMetadata.CreationTime = symlinkInodeMetadataAfterCreateSymlink.CreationTime.Add(negativeDurationToDelayOrSkew) - testMetadata.ModificationTime = symlinkInodeMetadataAfterCreateSymlink.ModificationTime - testMetadata.AccessTime = symlinkInodeMetadataAfterCreateSymlink.AccessTime - testMetadata.AttrChangeTime = symlinkInodeMetadataAfterCreateSymlink.AttrChangeTime - timeBeforeOp = time.Now() - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.SetCreationTime(symlinkInode, testMetadata.CreationTime) - if nil != err { - t.Fatalf("SetCreationTime(symlinkInode,) failed: %v", err) - } - time.Sleep(positiveDurationToDelayOrSkew) - timeAfterOp = time.Now() - - symlinkInodeMetadataAfterSetCreationTime, err := testVolumeHandle.GetMetadata(symlinkInode) - if nil != err { - t.Fatalf("GetMetadata(symlinkInode) failed: %v", err) - } - checkMetadata(t, symlinkInodeMetadataAfterSetCreationTime, testMetadata, MetadataNotAttrTimeFields, - "GetMetadata() after SetCreationTime() test 4") - if symlinkInodeMetadataAfterSetCreationTime.AttrChangeTime.Before(timeBeforeOp) || - symlinkInodeMetadataAfterSetCreationTime.AttrChangeTime.After(timeAfterOp) { - t.Fatalf("symlinkInodeMetadataAfterSetCreationTime.AttrChangeTime unexpected") - } - - testMetadata.ModificationTime = symlinkInodeMetadataAfterSetCreationTime.ModificationTime.Add(negativeDurationToDelayOrSkew) - err = testVolumeHandle.SetModificationTime(symlinkInode, testMetadata.ModificationTime) - if nil != err { - t.Fatalf("SetModificationTime(symlinkInode,) failed: %v", err) - } - symlinkInodeMetadataAfterSetModificationTime, err := testVolumeHandle.GetMetadata(symlinkInode) - if nil != err { - t.Fatalf("GetMetadata(symlinkInode) failed: %v", err) - } - checkMetadata(t, symlinkInodeMetadataAfterSetModificationTime, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after SetModificationTime()") - - testMetadata.AccessTime = symlinkInodeMetadataAfterSetModificationTime.AccessTime.Add(negativeDurationToDelayOrSkew) - err = testVolumeHandle.SetAccessTime(symlinkInode, testMetadata.AccessTime) - if nil != err { - t.Fatalf("SetAccessTime(symlinkInode,) failed: %v", err) - } - symlinkInodeMetadataAfterSetAccessTime, err := testVolumeHandle.GetMetadata(symlinkInode) - if nil != err { - t.Fatalf("GetMetadata(symlinkInode) failed: %v", err) - } - checkMetadata(t, symlinkInodeMetadataAfterSetAccessTime, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after SetAccessTime() test 2") - - timeBeforeSymlinkInodePutStream := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.PutStream(symlinkInode, "stream_name", []byte{}) - if nil != err { - t.Fatalf("PutStream(symlinkInode,,) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterSymlinkInodePutStream := time.Now() - - symlinkInodeMetadataAfterPutStream, err := testVolumeHandle.GetMetadata(symlinkInode) - if nil != err { - t.Fatalf("GetMetadata(symlinkInode) failed: %v", err) - } - checkMetadata(t, symlinkInodeMetadataAfterPutStream, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after PutStream()") - if symlinkInodeMetadataAfterPutStream.AttrChangeTime.Before(timeBeforeSymlinkInodePutStream) || symlinkInodeMetadataAfterPutStream.AttrChangeTime.After(timeAfterSymlinkInodePutStream) { - t.Fatalf("symlinkInodeMetadataAfterPutStream.AttrChangeTime unexpected") - } - - timeBeforeSymlinkInodeDeleteStream := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.DeleteStream(symlinkInode, "stream_name") - if nil != err { - t.Fatalf("DeleteStream(symlinkInode,) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterSymlinkInodeDeleteStream := time.Now() - - symlinkInodeMetadataAfterDeleteStream, err := testVolumeHandle.GetMetadata(symlinkInode) - if nil != err { - t.Fatalf("GetMetadata(symlinkInode) failed: %v", err) - } - checkMetadata(t, symlinkInodeMetadataAfterDeleteStream, testMetadata, MetadataNotAttrTimeFields, "GetMetadata() after DeleteStream()") - if symlinkInodeMetadataAfterDeleteStream.AttrChangeTime.Before(timeBeforeSymlinkInodeDeleteStream) || symlinkInodeMetadataAfterDeleteStream.AttrChangeTime.After(timeAfterSymlinkInodeDeleteStream) { - t.Fatalf("symlinkInodeMetadataAfterDeleteStream.AttrChangeTime unexpected") - } - - timeBeforeLink := time.Now() - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.Link(dirInode, "loc_1", fileInode, false) - if nil != err { - t.Fatalf("Link(dirInode, \"loc_1\", fileInode, false) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterLink := time.Now() - - dirInodeMetadataAfterLink, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - fileInodeMetadataAfterLink, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - - if !dirInodeMetadataAfterLink.CreationTime.Equal(dirInodeMetadataAfterDeleteStream.CreationTime) { - t.Fatalf("dirInodeMetadataAfterLink.CreationTime unexpected") - } - if dirInodeMetadataAfterLink.ModificationTime.Before(timeBeforeLink) || dirInodeMetadataAfterLink.ModificationTime.After(timeAfterLink) { - t.Fatalf("dirInodeMetadataAfterLink.ModificationTime unexpected") - } - if !dirInodeMetadataAfterLink.AccessTime.Equal(dirInodeMetadataAfterDeleteStream.AccessTime) { - t.Fatalf("dirInodeMetadataAfterLink.AccessTime unexpected changed") - } - if dirInodeMetadataAfterLink.AttrChangeTime.Before(timeBeforeLink) || dirInodeMetadataAfterLink.AttrChangeTime.After(timeAfterLink) { - t.Fatalf("dirInodeMetadataAfterLink.AttrChangeTime unexpected") - } - if !dirInodeMetadataAfterLink.AttrChangeTime.Equal(dirInodeMetadataAfterLink.ModificationTime) { - t.Fatalf("dirInodeMetadataAfterLink.AttrChangeTime should equal dirInodeMetadataAfterLink.ModificationTime") - } - if !fileInodeMetadataAfterLink.CreationTime.Equal(fileInodeMetadataAfterDeleteStream.CreationTime) { - t.Fatalf("fileInodeMetadataAfterLink.CreationTime unexpected") - } - if !fileInodeMetadataAfterLink.ModificationTime.Equal(fileInodeMetadataAfterDeleteStream.ModificationTime) { - t.Fatalf("fileInodeMetadataAfterLink.ModificationTime unexpected") - } - if !fileInodeMetadataAfterLink.AccessTime.Equal(fileInodeMetadataAfterDeleteStream.AccessTime) { - t.Fatalf("fileInodeMetadataAfterLink.AccessTime unexpected") - } - if fileInodeMetadataAfterLink.AttrChangeTime.Before(timeBeforeLink) || fileInodeMetadataAfterLink.AttrChangeTime.After(timeAfterLink) { - t.Fatalf("fileInodeMetadataAfterLink.AttrChangeTime unexpected") - } - - time.Sleep(positiveDurationToDelayOrSkew) - - toDestroyInodeNumber, err = testVolumeHandle.Move(dirInode, "loc_1", dirInode, "loc_2") - if nil != err { - t.Fatalf("Move(dirInode, \"loc_1\", dirInode, \"loc_2\") failed: %v", err) - } - if InodeNumber(0) != toDestroyInodeNumber { - t.Fatalf("Move(dirInode, \"loc_1\", dirInode, \"loc_2\") should have returned toDestroyInodeNumber == 0") - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterMove := time.Now() - - dirInodeMetadataAfterMove, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - fileInodeMetadataAfterMove, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - - if !dirInodeMetadataAfterMove.CreationTime.Equal(dirInodeMetadataAfterLink.CreationTime) { - t.Fatalf("dirInodeMetadataAfterMove.CreationTime unexpected") - } - if dirInodeMetadataAfterMove.ModificationTime.Before(timeAfterLink) || dirInodeMetadataAfterMove.ModificationTime.After(timeAfterMove) { - t.Fatalf("dirInodeMetadataAfterMove.ModificationTime unexpected") - } - if !dirInodeMetadataAfterMove.AccessTime.Equal(dirInodeMetadataAfterLink.AccessTime) { - t.Fatalf("dirInodeMetadataAfterMove.AccessTime unexpected change") - } - if dirInodeMetadataAfterMove.AttrChangeTime.Equal(dirInodeMetadataAfterLink.AttrChangeTime) { - t.Fatalf("dirInodeMetadataAfterMove.AttrChangeTime unchanged") - } - if !fileInodeMetadataAfterMove.CreationTime.Equal(fileInodeMetadataAfterLink.CreationTime) { - t.Fatalf("fileInodeMetadataAfterMove.CreationTime unexpected") - } - if !fileInodeMetadataAfterMove.ModificationTime.Equal(fileInodeMetadataAfterLink.ModificationTime) { - t.Fatalf("fileInodeMetadataAfterMove.ModificationTime unexpected") - } - if !fileInodeMetadataAfterMove.AccessTime.Equal(fileInodeMetadataAfterLink.AccessTime) { - t.Fatalf("fileInodeMetadataAfterMove.AccessTime unexpected") - } - if fileInodeMetadataAfterMove.AttrChangeTime.Equal(fileInodeMetadataAfterLink.AttrChangeTime) { - t.Fatalf("fileInodeMetadataAfterMove.AttrChangeTime should change after move") - } - - time.Sleep(positiveDurationToDelayOrSkew) - - toDestroyInodeNumber, err = testVolumeHandle.Unlink(dirInode, "loc_2", false) - if nil != err { - t.Fatalf("Unlink(dirInode, \"loc_2\", false) failed: %v", err) - } - if fileInode != toDestroyInodeNumber { - t.Fatalf("Unlink(dirInode, \"loc_2\", false) should have returned toDestroyInodeNumber == fileInode") - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterUnlink := time.Now() - - dirInodeMetadataAfterUnlink, err := testVolumeHandle.GetMetadata(dirInode) - if nil != err { - t.Fatalf("GetMetadata(dirInode) failed: %v", err) - } - fileInodeMetadataAfterUnlink, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - - if !dirInodeMetadataAfterUnlink.CreationTime.Equal(dirInodeMetadataAfterMove.CreationTime) { - t.Fatalf("dirInodeMetadataAfterUnlink.CreationTime unexpected") - } - if dirInodeMetadataAfterUnlink.ModificationTime.Before(timeAfterMove) || dirInodeMetadataAfterUnlink.ModificationTime.After(timeAfterUnlink) { - t.Fatalf("dirInodeMetadataAfterUnlink.ModificationTime unexpected") - } - if !dirInodeMetadataAfterUnlink.AccessTime.Equal(dirInodeMetadataAfterMove.AccessTime) { - t.Fatalf("dirInodeMetadataAfterUnlink.AccessTime unexpected change") - } - if dirInodeMetadataAfterUnlink.AttrChangeTime.Before(timeAfterMove) || dirInodeMetadataAfterUnlink.AttrChangeTime.After(timeAfterUnlink) { - t.Fatalf("dirInodeMetadataAfterUnlink.AttrChangeTime unexpected") - } - if !fileInodeMetadataAfterUnlink.CreationTime.Equal(fileInodeMetadataAfterMove.CreationTime) { - t.Fatalf("fileInodeMetadataAfterUnlink.CreationTime unexpected") - } - if !fileInodeMetadataAfterUnlink.ModificationTime.Equal(fileInodeMetadataAfterMove.ModificationTime) { - t.Fatalf("fileInodeMetadataAfterUnlink.ModificationTime unexpected") - } - if !fileInodeMetadataAfterUnlink.AccessTime.Equal(fileInodeMetadataAfterMove.AccessTime) { - t.Fatalf("fileInodeMetadataAfterUnlink.AccessTime unexpected") - } - if fileInodeMetadataAfterUnlink.AttrChangeTime.Before(timeAfterMove) || fileInodeMetadataAfterUnlink.AttrChangeTime.After(timeAfterUnlink) { - t.Fatalf("fileInodeMetadataAfterUnlink.AttrChangeTime unexpected") - } - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.Write(fileInode, uint64(0), []byte{0x00}, nil) - if nil != err { - t.Fatalf("Write(fileInode, uint64(0), []byte{0x00}) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterWrite := time.Now() - - fileInodeMetadataAfterWrite, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - - if !fileInodeMetadataAfterWrite.CreationTime.Equal(fileInodeMetadataAfterUnlink.CreationTime) { - t.Fatalf("fileInodeMetadataAfterWrite.CreationTime unexpected") - } - if fileInodeMetadataAfterWrite.ModificationTime.Before(timeAfterUnlink) || fileInodeMetadataAfterWrite.ModificationTime.After(timeAfterWrite) { - t.Fatalf("fileInodeMetadataAfterWrite.ModificationTime unexpected") - } - if !fileInodeMetadataAfterWrite.AttrChangeTime.Equal(fileInodeMetadataAfterWrite.ModificationTime) { - t.Fatalf("fileInodeMetadataAfterWrite.AttrChangeTime unexpected") - } - if !fileInodeMetadataAfterWrite.AccessTime.Equal(fileInodeMetadataAfterUnlink.AccessTime) { - t.Fatalf("fileInodeMetadataAfterWrite.AccessTime unexpected change") - } - - time.Sleep(positiveDurationToDelayOrSkew) - - objectPath, err := testVolumeHandle.ProvisionObject() - if nil != err { - t.Fatalf("ProvisionObject() failed: %v", err) - } - _, containerName, objectName, err = utils.PathToAcctContObj(objectPath) - if err != nil { - t.Fatalf("couldn't parse %v as object path", objectPath) - } - err = testVolumeHandle.Wrote(fileInode, containerName, objectName, []uint64{0}, []uint64{0}, []uint64{2}, time.Now(), false) - if nil != err { - t.Fatalf("Wrote(fileInode, objectPath, []uint64{0}, []uint64{0}, []uint64{2}, time.Now(), false) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterWrote := time.Now() - - fileInodeMetadataAfterWrote, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - - if !fileInodeMetadataAfterWrote.CreationTime.Equal(fileInodeMetadataAfterWrite.CreationTime) { - t.Fatalf("fileInodeMetadataAfterWrote.CreationTime unexpected") - } - if fileInodeMetadataAfterWrote.ModificationTime.Before(timeAfterWrite) || fileInodeMetadataAfterWrote.ModificationTime.After(timeAfterWrote) { - t.Fatalf("fileInodeMetadataAfterWrote.ModificationTime unexpected") - } - if !fileInodeMetadataAfterWrote.AttrChangeTime.Equal(fileInodeMetadataAfterWrote.ModificationTime) { - t.Fatalf("fileInodeMetadataAfterWrote.AttrChangeTime should equal fileInodeMetadataAfterWrote.ModificationTime") - } - if !fileInodeMetadataAfterWrote.AccessTime.Equal(fileInodeMetadataAfterWrite.AccessTime) { - t.Fatalf("fileInodeMetadataAfterWrote.AccessTime unexpected change") - } - - time.Sleep(positiveDurationToDelayOrSkew) - - err = testVolumeHandle.SetSize(fileInode, uint64(0)) - if nil != err { - t.Fatalf("SetSize(fileInode, uint64(0)) failed: %v", err) - } - - time.Sleep(positiveDurationToDelayOrSkew) - - timeAfterSetSize := time.Now() - - fileInodeMetadataAfterSetSize, err := testVolumeHandle.GetMetadata(fileInode) - if nil != err { - t.Fatalf("GetMetadata(fileInode) failed: %v", err) - } - - if !fileInodeMetadataAfterSetSize.CreationTime.Equal(fileInodeMetadataAfterWrote.CreationTime) { - t.Fatalf("fileInodeMetadataAfterSetSize.CreationTime unexpected") - } - if fileInodeMetadataAfterSetSize.ModificationTime.Before(timeAfterWrote) || fileInodeMetadataAfterSetSize.ModificationTime.After(timeAfterSetSize) { - t.Fatalf("fileInodeMetadataAfterSetSize.ModificationTime unexpected") - } - if !fileInodeMetadataAfterSetSize.AttrChangeTime.Equal(fileInodeMetadataAfterSetSize.ModificationTime) { - t.Fatalf("fileInodeMetadataAfterSetsize.AttrChangeTime should equal fileInodeMetadataAfterSetSize.ModificationTime") - } - if !fileInodeMetadataAfterSetSize.AccessTime.Equal(fileInodeMetadataAfterWrote.AccessTime) { - t.Fatalf("fileInodeMetadataAfterSetSize.AccessTime unexpected change") - } - - // TODO: Need to test GetFragmentationReport() - - // TODO: Once implemented, need to test Optimize() - - testTeardown(t) -} - -func TestInodeDiscard(t *testing.T) { - testSetup(t, false) - - assert := assert.New(t) - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") should have worked - got error: %v", err) - } - - // Calculate how many inodes we must create to make sure the inode cache discard - // routine will find something to discard. - vS := testVolumeHandle.(*volumeStruct) - maxBytes := vS.inodeCacheLRUMaxBytes - iSize := globals.inodeSize - entriesNeeded := maxBytes / iSize - entriesNeeded = entriesNeeded * 6 - for i := uint64(0); i < entriesNeeded; i++ { - fileInodeNumber, err := testVolumeHandle.CreateFile(InodeMode(0000), InodeRootUserID, InodeGroupID(0)) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - fName := fmt.Sprintf("file-%v i: %v", fileInodeNumber, i) - err = testVolumeHandle.Link(RootDirInodeNumber, fName, fileInodeNumber, false) - if nil != err { - t.Fatalf("Link(RootDirInodeNumber, \"%v\", file1Inode, false) failed: %v", fName, err) - } - - fileInode, ok, err := vS.fetchInode(fileInodeNumber) - assert.Nil(err, nil, "Unable to fetchInode due to err - even though just created") - assert.True(ok, "fetchInode returned !ok - even though just created") - assert.False(fileInode.dirty, "fetchInode.dirty == true - even though just linked") - } - - discarded, dirty, locked, lruItems := vS.inodeCacheDiscard() - - assert.NotEqual(discarded, uint64(0), "Number of inodes discarded should be non-zero") - assert.Equal(dirty, uint64(0), "Number of inodes dirty should zero") - assert.Equal(locked, uint64(0), "Number of inodes locked should zero") - assert.Equal((lruItems * iSize), (maxBytes/iSize)*iSize, "Number of inodes in cache not same as max") - - testTeardown(t) -} diff --git a/inode/benchmark_test.go b/inode/benchmark_test.go deleted file mode 100644 index 262f602e..00000000 --- a/inode/benchmark_test.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "strconv" - "testing" - - "github.com/NVIDIA/proxyfs/utils" -) - -func writeBenchmarkHelper(b *testing.B, byteSize uint64) { - testVolumeHandle, _ := FetchVolumeHandle("TestVolume") - fileInodeNumber, _ := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - buffer := make([]byte, 4096) - b.ResetTimer() - for i := 0; i < b.N; i++ { - testVolumeHandle.Write(fileInodeNumber, 0, buffer, nil) - } -} - -func Benchmark4KiBWrite(b *testing.B) { - writeBenchmarkHelper(b, 4*1024) -} - -func Benchmark8KiBWrite(b *testing.B) { - writeBenchmarkHelper(b, 8*1024) -} - -func Benchmark16KiBWrite(b *testing.B) { - writeBenchmarkHelper(b, 16*1024) -} - -func Benchmark32KiBWrite(b *testing.B) { - writeBenchmarkHelper(b, 32*1024) -} - -func Benchmark64KiBWrite(b *testing.B) { - writeBenchmarkHelper(b, 64*1024) -} - -func readBenchmarkHelper(b *testing.B, byteSize uint64) { - testVolumeHandle, _ := FetchVolumeHandle("TestVolume") - fileInodeNumber, _ := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - buffer := make([]byte, byteSize) - testVolumeHandle.Write(fileInodeNumber, 0, buffer, nil) - b.ResetTimer() - for i := 0; i < b.N; i++ { - testVolumeHandle.Read(fileInodeNumber, 0, byteSize, nil) - } -} - -func BenchmarkRead4KiB(b *testing.B) { - readBenchmarkHelper(b, 4*1024) -} - -func BenchmarkRead8KiB(b *testing.B) { - readBenchmarkHelper(b, 8*1024) -} - -func BenchmarkRead16KiB(b *testing.B) { - readBenchmarkHelper(b, 16*1024) -} - -func BenchmarkRead32KiB(b *testing.B) { - readBenchmarkHelper(b, 32*1024) -} - -func BenchmarkRead64KiB(b *testing.B) { - readBenchmarkHelper(b, 64*1024) -} - -func getReadPlanBenchmarkHelper(b *testing.B, byteSize uint64) { - testVolumeHandle, _ := FetchVolumeHandle("TestVolume") - fileInodeNumber, _ := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - buffer := make([]byte, byteSize) - testVolumeHandle.Write(fileInodeNumber, 0, buffer, nil) - var zero uint64 - b.ResetTimer() - for i := 0; i < b.N; i++ { - testVolumeHandle.GetReadPlan(fileInodeNumber, &zero, &byteSize) - } -} - -func BenchmarkGetReadPlan4KiB(b *testing.B) { - getReadPlanBenchmarkHelper(b, 4*1024) -} - -func BenchmarkGetReadPlan8KiB(b *testing.B) { - getReadPlanBenchmarkHelper(b, 8*1024) -} - -func BenchmarkGetReadPlan16KiB(b *testing.B) { - getReadPlanBenchmarkHelper(b, 16*1024) -} - -func BenchmarkGetReadPlan32KiB(b *testing.B) { - getReadPlanBenchmarkHelper(b, 32*1024) -} - -func BenchmarkGetReadPlan64KiB(b *testing.B) { - getReadPlanBenchmarkHelper(b, 64*1024) -} - -func readCacheBenchmarkHelper(b *testing.B, byteSize uint64) { - testVolumeHandle, _ := FetchVolumeHandle("TestVolume") - fileInodeNumber, _ := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - buffer := make([]byte, byteSize) - testVolumeHandle.Write(fileInodeNumber, 0, buffer, nil) - testVolumeHandle.Flush(fileInodeNumber, false) - var zero uint64 - zero = 0 - readPlan, _ := testVolumeHandle.GetReadPlan(fileInodeNumber, &zero, &byteSize) - testVolumeHandle.Read(fileInodeNumber, 0, byteSize, nil) - // at this point, the read cache should be populated - - // let's get the log segment number - _, _, objectName, _ := utils.PathToAcctContObj(readPlan[0].ObjectPath) - logSegmentNumber, _ := strconv.ParseUint(objectName, 16, 64) - - volume := testVolumeHandle.(*volumeStruct) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf := []byte{} - readCacheKey := readCacheKeyStruct{volumeName: "TestVolume", logSegmentNumber: logSegmentNumber, cacheLineTag: 0} - volume.volumeGroup.Lock() - readCacheElement, _ := volume.volumeGroup.readCache[readCacheKey] - cacheLine := readCacheElement.cacheLine - buf = append(buf, cacheLine[:byteSize]...) - volume.volumeGroup.Unlock() - } -} - -func BenchmarkReadCache4KiB(b *testing.B) { - readCacheBenchmarkHelper(b, 4*1024) -} - -func BenchmarkReadCache8KiB(b *testing.B) { - readCacheBenchmarkHelper(b, 8*1024) -} - -func BenchmarkReadCache16KiB(b *testing.B) { - readCacheBenchmarkHelper(b, 16*1024) -} - -func BenchmarkReadCache32KiB(b *testing.B) { - readCacheBenchmarkHelper(b, 32*1024) -} - -func BenchmarkReadCache64KiB(b *testing.B) { - readCacheBenchmarkHelper(b, 64*1024) -} diff --git a/inode/cache.go b/inode/cache.go deleted file mode 100644 index 53a0a216..00000000 --- a/inode/cache.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/platform" -) - -func adoptVolumeGroupReadCacheParameters(confMap conf.ConfMap) (err error) { - var ( - readCacheLineCount uint64 - readCacheMemSize uint64 - readCacheQuotaFraction float64 - readCacheTotalSize uint64 - readCacheWeightSum uint64 - totalMemSize uint64 - volumeGroup *volumeGroupStruct - ) - - readCacheWeightSum = 0 - - for _, volumeGroup = range globals.volumeGroupMap { - if 0 < volumeGroup.numServed { - readCacheWeightSum += volumeGroup.readCacheWeight - } - } - - readCacheQuotaFraction, err = confMap.FetchOptionValueFloat64("Peer:"+globals.whoAmI, "ReadCacheQuotaFraction") - if nil != err { - return - } - if (0 > readCacheQuotaFraction) || (1 < readCacheQuotaFraction) { - err = fmt.Errorf("%s.ReadCacheQuotaFraction (%v) must be between 0 and 1", globals.whoAmI, readCacheQuotaFraction) - return - } - - totalMemSize = platform.MemSize() - - readCacheMemSize = uint64(float64(totalMemSize) * readCacheQuotaFraction / platform.GoHeapAllocationMultiplier) - - logger.Infof("Adopting ReadCache Parameters: ReadCacheQuotaFraction(%v) of memSize(0x%016X) totals 0x%016X", - readCacheQuotaFraction, totalMemSize, readCacheMemSize) - - for _, volumeGroup = range globals.volumeGroupMap { - if 0 < volumeGroup.numServed { - readCacheTotalSize = readCacheMemSize * volumeGroup.readCacheWeight / readCacheWeightSum - - readCacheLineCount = readCacheTotalSize / volumeGroup.readCacheLineSize - if 0 == readCacheLineCount { - logger.Infof("Computed 0 ReadCacheLines for Volume Group %v; increasing to 1", - volumeGroup.name) - readCacheLineCount = 1 - } - - volumeGroup.Lock() - volumeGroup.readCacheLineCount = readCacheLineCount - volumeGroup.capReadCacheWhileLocked() - volumeGroup.Unlock() - - logger.Infof("Volume Group %s: %d cache lines (each of size 0x%08X) totalling 0x%016X", - volumeGroup.name, - volumeGroup.readCacheLineCount, volumeGroup.readCacheLineSize, - volumeGroup.readCacheLineCount*volumeGroup.readCacheLineSize) - } - } - - err = nil - return -} - -func startInodeCacheDiscard(confMap conf.ConfMap, volume *volumeStruct, volumeSectionName string) (err error) { - var ( - LRUCacheMaxBytes uint64 - LRUDiscardTimeInterval time.Duration - ) - - LRUCacheMaxBytes, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxBytesInodeCache") - if nil != err { - LRUCacheMaxBytes = 10485760 // TODO - Remove setting a default value - err = nil - } - volume.inodeCacheLRUMaxBytes = LRUCacheMaxBytes - - LRUDiscardTimeInterval, err = confMap.FetchOptionValueDuration(volumeSectionName, "InodeCacheEvictInterval") - if nil != err { - LRUDiscardTimeInterval = 1 * time.Second // TODO - Remove setting a default value - err = nil - } - - if LRUDiscardTimeInterval != 0 { - volume.inodeCacheLRUTickerInterval = LRUDiscardTimeInterval - volume.inodeCacheLRUTicker = time.NewTicker(volume.inodeCacheLRUTickerInterval) - - logger.Infof("Inode cache discard ticker for 'volume: %v' is: %v MaxBytesInodeCache: %v", - volume.volumeName, volume.inodeCacheLRUTickerInterval, volume.inodeCacheLRUMaxBytes) - - // Start ticker for inode cache discard thread - volume.inodeCacheWG.Add(1) - go func() { - for { - select { - case _ = <-volume.inodeCacheLRUTicker.C: - _, _, _, _ = volume.inodeCacheDiscard() - case _, _ = <-volume.inodeCacheStopChan: - volume.inodeCacheWG.Done() - return - } - } - }() - } else { - logger.Infof("Inode cache discard ticker for 'volume: %v' is disabled.", - volume.volumeName) - return - } - - return -} - -func stopInodeCacheDiscard(volume *volumeStruct) { - if volume.inodeCacheLRUTicker != nil { - volume.inodeCacheLRUTicker.Stop() - close(volume.inodeCacheStopChan) - volume.inodeCacheWG.Wait() - logger.Infof("Inode cache discard ticker for 'volume: %v' stopped.", - volume.volumeName) - } -} diff --git a/inode/coalesce_test.go b/inode/coalesce_test.go deleted file mode 100644 index 5dad6c35..00000000 --- a/inode/coalesce_test.go +++ /dev/null @@ -1,441 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "testing" - "time" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/stretchr/testify/assert" -) - -// NB: test setup and such is in api_test.go (look for TestMain function) - -func TestCoalesce(t *testing.T) { - testSetup(t, false) - - // We're going to take some files: - // - // d1/file1a (contents "abcd") - // d1/file1b (contents "efgh") - // d2/file2a (contents "ijkl") - // d2/file2b (contents "mnop") - // d2/file2c (contents "\0\0st\0\0") - // - // and coalesce them into a single file: - // - // d1/combined (contents "abcdefghijklmnop\0\0st\0\0") - // - // This will also unlink the constituent files from their directories. - - assert := assert.New(t) - vh, err := FetchVolumeHandle("TestVolume") - if !assert.Nil(err) { - return - } - - d1InodeNumber, err := vh.CreateDir(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Link(RootDirInodeNumber, "d1", d1InodeNumber, false) - if !assert.Nil(err) { - return - } - - d2InodeNumber, err := vh.CreateDir(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Link(RootDirInodeNumber, "d2", d2InodeNumber, false) - if !assert.Nil(err) { - return - } - - file1aInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file1aInodeNumber, 0, []byte("abcd"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "file1a", file1aInodeNumber, false) - if !assert.Nil(err) { - return - } - - file1bInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file1bInodeNumber, 0, []byte("efgh"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "file1b", file1bInodeNumber, false) - if !assert.Nil(err) { - return - } - - file2aInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file2aInodeNumber, 0, []byte("ijkl"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d2InodeNumber, "file2a", file2aInodeNumber, false) - if !assert.Nil(err) { - return - } - - file2bInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file2bInodeNumber, 0, []byte("mnop"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d2InodeNumber, "file2b", file2bInodeNumber, false) - if !assert.Nil(err) { - return - } - - // Note that this one is sparse: the first 2 bytes are 0, then we have "st", then 2 more 0s - file2cInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file2cInodeNumber, 2, []byte("st"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d2InodeNumber, "file2c", file2cInodeNumber, false) - if !assert.Nil(err) { - return - } - err = vh.SetSize(file2cInodeNumber, 6) - if !assert.Nil(err) { - return - } - - // Now create destination file - combinedInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "combined", combinedInodeNumber, false) - if !assert.Nil(err) { - return - } - - // test setup's done; now we can coalesce things - elements := make([]*CoalesceElement, 0, 5) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1aInodeNumber, - ElementName: "file1a"}) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1bInodeNumber, - ElementName: "file1b"}) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d2InodeNumber, - ElementInodeNumber: file2aInodeNumber, - ElementName: "file2a"}) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d2InodeNumber, - ElementInodeNumber: file2bInodeNumber, - ElementName: "file2b"}) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d2InodeNumber, - ElementInodeNumber: file2cInodeNumber, - ElementName: "file2c"}) - - newMetaData := []byte("The quick brown fox jumped over the lazy dog.") - - // Coalesce the above 5 files and metadata into d1/combined - startTime := time.Now() - attrChangeTime, modificationTime, numWrites, fileSize, err := vh.Coalesce( - combinedInodeNumber, "MetaDataStream", newMetaData, elements) - if !assert.Nil(err) { - return - } - assert.Equal(uint64(22), fileSize) - assert.Equal(uint64(5), numWrites) - assert.Equal(attrChangeTime, modificationTime) - assert.True(attrChangeTime.After(startTime)) - - // The new file has the contents of the old files combined - contents, err := vh.Read(combinedInodeNumber, 0, 22, nil) - if !assert.Nil(err) { - return - } - assert.Equal([]byte("abcdefghijklmnop\x00\x00st\x00\x00"), contents) - - // The old files have ceased to be - _, err = vh.Lookup(d1InodeNumber, "file1a") - assert.True(blunder.Is(err, blunder.NotFoundError)) - _, err = vh.Lookup(d1InodeNumber, "file1b") - assert.True(blunder.Is(err, blunder.NotFoundError)) - _, err = vh.Lookup(d2InodeNumber, "file2a") - assert.True(blunder.Is(err, blunder.NotFoundError)) - _, err = vh.Lookup(d2InodeNumber, "file2b") - assert.True(blunder.Is(err, blunder.NotFoundError)) - _, err = vh.Lookup(d2InodeNumber, "file2c") - assert.True(blunder.Is(err, blunder.NotFoundError)) - - // The new file is linked in at the right spot - foundInodeNumber, err := vh.Lookup(d1InodeNumber, "combined") - if !assert.Nil(err) { - return - } - assert.Equal(combinedInodeNumber, foundInodeNumber) - - // Verify the new file has the new metadata - buf, err := vh.GetStream(combinedInodeNumber, "MetaDataStream") - if assert.Nil(err) { - assert.Equal(buf, newMetaData) - } - - testTeardown(t) -} - -func TestCoalesceDir(t *testing.T) { - testSetup(t, false) - - // We're going to take some files: - // - // d1/file1a (contents "abcd") - // d1/file1b (contents "efgh") - // - // and attempt to coalesce them into a directory: - // - // d1 - - assert := assert.New(t) - vh, err := FetchVolumeHandle("TestVolume") - if !assert.Nil(err) { - return - } - - d1InodeNumber, err := vh.CreateDir(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Link(RootDirInodeNumber, "d1", d1InodeNumber, false) - if !assert.Nil(err) { - return - } - - file1aInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file1aInodeNumber, 0, []byte("abcd"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "file1a", file1aInodeNumber, false) - if !assert.Nil(err) { - return - } - - file1bInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file1bInodeNumber, 0, []byte("efgh"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "file1b", file1bInodeNumber, false) - if !assert.Nil(err) { - return - } - - // test setup's done; now we can coalesce things - elements := make([]*CoalesceElement, 0, 2) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1aInodeNumber, - ElementName: "file1a"}) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1bInodeNumber, - ElementName: "file1b"}) - - // Coalesce the above 2 files into d1 - _, _, _, _, err = vh.Coalesce(d1InodeNumber, "MetaDataStream", nil, elements) - assert.NotNil(err) - assert.True(blunder.Is(err, blunder.PermDeniedError)) - - testTeardown(t) -} - -func TestCoalesceMultipleLinks(t *testing.T) { - testSetup(t, false) - - // We're going to take hard-linked files: - // - // d1/file1a (contents "abcd") - // d1/file1b (hard-linked to d1/file1a) - // - // and attempt to coalesce them into a single file: - // - // d1/combined - - assert := assert.New(t) - vh, err := FetchVolumeHandle("TestVolume") - if !assert.Nil(err) { - return - } - - d1InodeNumber, err := vh.CreateDir(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Link(RootDirInodeNumber, "d1", d1InodeNumber, false) - if !assert.Nil(err) { - return - } - - file1aInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file1aInodeNumber, 0, []byte("abcd"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "file1a", file1aInodeNumber, false) - if !assert.Nil(err) { - return - } - - file1bInodeNumber := file1aInodeNumber - err = vh.Link(d1InodeNumber, "file1b", file1bInodeNumber, false) - if !assert.Nil(err) { - return - } - - // Now create destination file - combinedInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "combined", combinedInodeNumber, false) - if !assert.Nil(err) { - return - } - - // test setup's done; now we can coalesce things - elements := make([]*CoalesceElement, 0, 2) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1aInodeNumber, - ElementName: "file1a"}) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1bInodeNumber, - ElementName: "file1b"}) - - // Coalesce the above 2 files into d1/combined - _, _, _, _, err = vh.Coalesce(combinedInodeNumber, "MetaDataStream", nil, elements) - assert.NotNil(err) - assert.True(blunder.Is(err, blunder.TooManyLinksError)) - - testTeardown(t) -} - -func TestCoalesceDuplicates(t *testing.T) { - testSetup(t, false) - - // We're going to take hard-linked files: - // - // d1/file1a (contents "abcd") - // d1/file1b (contents "efgh") - // d1/file1a (again) - // - // and attempt to coalesce them into a single file: - // - // d1/combined - - assert := assert.New(t) - vh, err := FetchVolumeHandle("TestVolume") - if !assert.Nil(err) { - return - } - - d1InodeNumber, err := vh.CreateDir(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Link(RootDirInodeNumber, "d1", d1InodeNumber, false) - if !assert.Nil(err) { - return - } - - file1aInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file1aInodeNumber, 0, []byte("abcd"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "file1a", file1aInodeNumber, false) - if !assert.Nil(err) { - return - } - - file1bInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Write(file1bInodeNumber, 0, []byte("efgh"), nil) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "file1b", file1bInodeNumber, false) - if !assert.Nil(err) { - return - } - - // Now create destination file - combinedInodeNumber, err := vh.CreateFile(PosixModePerm, 0, 0) - if !assert.Nil(err) { - return - } - err = vh.Link(d1InodeNumber, "combined", combinedInodeNumber, false) - if !assert.Nil(err) { - return - } - - // test setup's done; now we can coalesce things - elements := make([]*CoalesceElement, 0, 3) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1aInodeNumber, - ElementName: "file1a"}) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1bInodeNumber, - ElementName: "file1b"}) - elements = append(elements, &CoalesceElement{ - ContainingDirectoryInodeNumber: d1InodeNumber, - ElementInodeNumber: file1aInodeNumber, - ElementName: "file1a"}) - - // Coalesce the above 3 files into d1/combined - _, _, _, _, err = vh.Coalesce(combinedInodeNumber, "MetaDataStream", nil, elements) - assert.NotNil(err) - assert.True(blunder.Is(err, blunder.InvalidArgError)) - - testTeardown(t) -} diff --git a/inode/config.go b/inode/config.go deleted file mode 100644 index 1536b5e3..00000000 --- a/inode/config.go +++ /dev/null @@ -1,815 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - "sync" - "time" - "unsafe" - - "github.com/NVIDIA/cstruct" - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -type readCacheKeyStruct struct { - volumeName string - logSegmentNumber uint64 - cacheLineTag uint64 // LogSegment offset / readCacheLineSize -} - -type readCacheElementStruct struct { - readCacheKey readCacheKeyStruct - next *readCacheElementStruct // nil if MRU element of volumeGroupStruct.readCache - prev *readCacheElementStruct // nil if LRU element of volumeGroupStruct.readCache - cacheLine []byte -} - -type volumeGroupStruct struct { - trackedlock.Mutex - name string - volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName - numServed uint64 - virtualIPAddr string - activePeerPrivateIPAddr string - readCacheLineSize uint64 - readCacheWeight uint64 - readCacheLineCount uint64 - readCache map[readCacheKeyStruct]*readCacheElementStruct - readCacheMRU *readCacheElementStruct - readCacheLRU *readCacheElementStruct -} - -type physicalContainerLayoutStruct struct { - name string - containerStoragePolicy string // []ContainerStoragePolicy - containerNamePrefix string // []ContainerNamePrefix - containerNameSlice []string // == slice of current PhysicalContainers for this PhysicalContainerLayout - // Note: all prefixed by containerNamePrefix - containersPerPeer uint64 // []ContainersPerPeer - maxObjectsPerContainer uint64 // []MaxObjectsPerContainer - containerNameSliceNextIndex uint64 // == next index in nameSlice - containerNameSliceLoopCount uint64 // == number of times looped through nameSlice - // Note: if 0 == (containerNameSliceLoopCount mod maxObjectsPerContainer) - // then we need to re-provision containerNameSlice -} - -type volumeStruct struct { - trackedlock.Mutex - volumeGroup *volumeGroupStruct - served bool - fsid uint64 - volumeName string - accountName string - maxEntriesPerDirNode uint64 - maxExtentsPerFileNode uint64 - defaultPhysicalContainerLayout *physicalContainerLayoutStruct - maxFlushSize uint64 - headhunterVolumeHandle headhunter.VolumeHandle - inodeCache sortedmap.LLRBTree // key == InodeNumber; value == *inMemoryInodeStruct - inodeCacheStopChan chan struct{} - inodeCacheWG sync.WaitGroup - inodeCacheLRUHead *inMemoryInodeStruct - inodeCacheLRUTail *inMemoryInodeStruct - inodeCacheLRUItems uint64 - inodeCacheLRUMaxBytes uint64 - inodeCacheLRUTicker *time.Ticker - inodeCacheLRUTickerInterval time.Duration - snapShotPolicy *snapShotPolicyStruct -} - -const ( - defaultNoWriteErrno = blunder.NoSpaceError - defaultNoWriteErrnoString = "ENOSPC" - defaultReadOnlyErrno = blunder.ReadOnlyError - defaultReadOnlyErrnoString = "EROFS" -) - -type globalsStruct struct { - trackedlock.Mutex - whoAmI string - myPrivateIPAddr string - dirEntryCache sortedmap.BPlusTreeCache - dirEntryCachePriorCacheHits uint64 - dirEntryCachePriorCacheMisses uint64 - fileExtentMapCache sortedmap.BPlusTreeCache - fileExtentMapCachePriorCacheHits uint64 - fileExtentMapCachePriorCacheMisses uint64 - volumeGroupMap map[string]*volumeGroupStruct // key == volumeGroupStruct.name - volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName - accountMap map[string]*volumeStruct // key == volumeStruct.accountName - fileExtentStructSize uint64 // pre-calculated size of cstruct-packed fileExtentStruct - supportedOnDiskInodeVersions map[Version]struct{} // key == on disk inode version - corruptionDetectedTrueBuf []byte // holds serialized CorruptionDetected == true - corruptionDetectedFalseBuf []byte // holds serialized CorruptionDetected == false - versionV1Buf []byte // holds serialized Version == V1 - inodeRecDefaultPreambleBuf []byte // holds concatenated corruptionDetectedFalseBuf & versionV1Buf - inodeSize uint64 // size of in-memory inode struct - openLogSegmentLRUHead *inFlightLogSegmentStruct - openLogSegmentLRUTail *inFlightLogSegmentStruct - openLogSegmentLRUItems uint64 - noWriteThresholdErrno blunder.FsError // either blunder.NotPermError or blunder.ReadOnlyError or blunder.NoSpaceError - noWriteThresholdErrnoString string // either "EPERM" or "EROFS" or "ENOSPC" - readOnlyThresholdErrno blunder.FsError // either blunder.NotPermError or blunder.ReadOnlyError or blunder.NoSpaceError - readOnlyThresholdErrnoString string // either "EPERM" or "EROFS" or "ENOSPC" - rwMode RWModeType // One of RWMode{Normal|NoWrite|ReadOnly} -} - -var globals globalsStruct - -func init() { - transitions.Register("inode", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - var ( - corruptionDetectedFalse = CorruptionDetected(false) - corruptionDetectedTrue = CorruptionDetected(true) - dirEntryCacheEvictHighLimit uint64 - dirEntryCacheEvictLowLimit uint64 - fileExtentMapEvictHighLimit uint64 - fileExtentMapEvictLowLimit uint64 - ok bool - peerName string - peerNames []string - peerPrivateIPAddr string - peerPrivateIPAddrMap map[string]string - tempInode inMemoryInodeStruct - versionV1 = Version(V1) - ) - - peerPrivateIPAddrMap = make(map[string]string) - - peerNames, err = confMap.FetchOptionValueStringSlice("Cluster", "Peers") - if nil != err { - return - } - - for _, peerName = range peerNames { - peerPrivateIPAddr, err = confMap.FetchOptionValueString("Peer:"+peerName, "PrivateIPAddr") - if nil != err { - return - } - - peerPrivateIPAddrMap[peerName] = peerPrivateIPAddr - } - - globals.whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - return - } - globals.myPrivateIPAddr, ok = peerPrivateIPAddrMap[globals.whoAmI] - if !ok { - err = fmt.Errorf("Cluster.WhoAmI (\"%v\") not in Cluster.Peers list", globals.whoAmI) - return - } - - dirEntryCacheEvictLowLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "DirEntryCacheEvictLowLimit") - if nil != err { - return - } - dirEntryCacheEvictHighLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "DirEntryCacheEvictHighLimit") - if nil != err { - return - } - - globals.dirEntryCache = sortedmap.NewBPlusTreeCache(dirEntryCacheEvictLowLimit, dirEntryCacheEvictHighLimit) - - globals.dirEntryCachePriorCacheHits = 0 - globals.dirEntryCachePriorCacheMisses = 0 - - fileExtentMapEvictLowLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "FileExtentMapEvictLowLimit") - if nil != err { - return - } - fileExtentMapEvictHighLimit, err = confMap.FetchOptionValueUint64("FSGlobals", "FileExtentMapEvictHighLimit") - if nil != err { - return - } - - globals.fileExtentMapCache = sortedmap.NewBPlusTreeCache(fileExtentMapEvictLowLimit, fileExtentMapEvictHighLimit) - - globals.fileExtentMapCachePriorCacheHits = 0 - globals.fileExtentMapCachePriorCacheMisses = 0 - - globals.volumeGroupMap = make(map[string]*volumeGroupStruct) - globals.volumeMap = make(map[string]*volumeStruct) - globals.accountMap = make(map[string]*volumeStruct) - - globals.inodeSize = uint64(unsafe.Sizeof(tempInode)) - - globals.openLogSegmentLRUHead = nil - globals.openLogSegmentLRUTail = nil - globals.openLogSegmentLRUItems = 0 - - globals.fileExtentStructSize, _, err = cstruct.Examine(fileExtentStruct{}) - if nil != err { - return - } - - globals.supportedOnDiskInodeVersions = make(map[Version]struct{}) - - globals.supportedOnDiskInodeVersions[V1] = struct{}{} - - globals.corruptionDetectedTrueBuf, err = cstruct.Pack(corruptionDetectedTrue, cstruct.LittleEndian) - if nil != err { - return - } - globals.corruptionDetectedFalseBuf, err = cstruct.Pack(corruptionDetectedFalse, cstruct.LittleEndian) - if nil != err { - return - } - globals.versionV1Buf, err = cstruct.Pack(versionV1, cstruct.LittleEndian) - if nil != err { - return - } - - globals.inodeRecDefaultPreambleBuf = make([]byte, 0, len(globals.corruptionDetectedFalseBuf)+len(globals.versionV1Buf)) - globals.inodeRecDefaultPreambleBuf = append(globals.inodeRecDefaultPreambleBuf, globals.corruptionDetectedFalseBuf...) - globals.inodeRecDefaultPreambleBuf = append(globals.inodeRecDefaultPreambleBuf, globals.versionV1Buf...) - - swiftclient.SetStarvationCallbackFunc(chunkedPutConnectionPoolStarvationCallback) - - globals.rwMode = RWModeNormal - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - var ( - ok bool - volumeGroup *volumeGroupStruct - volumeGroupSectionName string - ) - - volumeGroup = &volumeGroupStruct{ - name: volumeGroupName, - volumeMap: make(map[string]*volumeStruct), - numServed: 0, - readCacheLineCount: 0, - readCache: make(map[readCacheKeyStruct]*readCacheElementStruct), - readCacheMRU: nil, - readCacheLRU: nil, - } - - volumeGroupSectionName = "VolumeGroup:" + volumeGroupName - - volumeGroup.virtualIPAddr, err = confMap.FetchOptionValueString(volumeGroupSectionName, "VirtualIPAddr") - if nil != err { - if nil == confMap.VerifyOptionValueIsEmpty(volumeGroupSectionName, "VirtualIPAddr") { - volumeGroup.virtualIPAddr = "" - } else { - return - } - } - - if "" == activePeer { - volumeGroup.activePeerPrivateIPAddr = "" - } else { - volumeGroup.activePeerPrivateIPAddr, err = confMap.FetchOptionValueString("Peer:"+activePeer, "PrivateIPAddr") - if nil != err { - return - } - } - - volumeGroup.readCacheLineSize, err = confMap.FetchOptionValueUint64(volumeGroupSectionName, "ReadCacheLineSize") - if nil != err { - return - } - if volumeGroup.readCacheLineSize < 4096 { - logger.Warnf("Section '%s' for VolumeGroup '%s' ReadCacheLineSize %d is < 4096; changing to 4096", - volumeGroupSectionName, volumeGroupName, volumeGroup.readCacheLineSize) - volumeGroup.readCacheLineSize = 4096 - } - - volumeGroup.readCacheWeight, err = confMap.FetchOptionValueUint64(volumeGroupSectionName, "ReadCacheWeight") - if nil != err { - return - } - if volumeGroup.readCacheWeight <= 0 { - logger.Warnf("Section '%s' for VolumeGroup '%s' ReadCacheWeight %d is <= 0; changing to 1", - volumeGroupSectionName, volumeGroupName, volumeGroup.readCacheWeight) - volumeGroup.readCacheWeight = 1 - } - - globals.Lock() - - _, ok = globals.volumeGroupMap[volumeGroupName] - if ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeGroupCreated() called for preexisting VolumeGroup (%s)", volumeGroupName) - return - } - - globals.volumeGroupMap[volumeGroupName] = volumeGroup - - globals.Unlock() - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - var ( - ok bool - volumeGroup *volumeGroupStruct - ) - - globals.Lock() - - volumeGroup, ok = globals.volumeGroupMap[volumeGroupName] - if !ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeGroupMoved() called for nonexistent VolumeGroup (%s)", volumeGroupName) - return - } - - volumeGroup.Lock() - - if "" == activePeer { - volumeGroup.activePeerPrivateIPAddr = "" - } else { - volumeGroup.activePeerPrivateIPAddr, err = confMap.FetchOptionValueString("Peer:"+activePeer, "PrivateIPAddr") - if nil != err { - volumeGroup.Unlock() - globals.Unlock() - return - } - } - - // Note that VirtualIPAddr, ReadCacheLineSize, & ReadCacheWeight are not reloaded - - volumeGroup.Unlock() - globals.Unlock() - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - var ( - ok bool - volumeGroup *volumeGroupStruct - ) - - globals.Lock() - - volumeGroup, ok = globals.volumeGroupMap[volumeGroupName] - if !ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeGroupDestroyed() called for nonexistent VolumeGroup (%s)", volumeGroupName) - return - } - - volumeGroup.Lock() - - if 0 != len(volumeGroup.volumeMap) { - volumeGroup.Unlock() - globals.Unlock() - err = fmt.Errorf("inode.VolumeGroupDestroyed() called for non-empty VolumeGroup (%s)", volumeGroupName) - return - } - - delete(globals.volumeGroupMap, volumeGroupName) - - volumeGroup.Unlock() - globals.Unlock() - - err = nil - return -} - -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - var ( - ok bool - volumeGroup *volumeGroupStruct - volumeSectionName string - ) - - volume := &volumeStruct{volumeName: volumeName, served: false} - - volumeSectionName = "Volume:" + volumeName - - volume.fsid, err = confMap.FetchOptionValueUint64(volumeSectionName, "FSID") - if nil != err { - return - } - - volume.accountName, err = confMap.FetchOptionValueString(volumeSectionName, "AccountName") - if nil != err { - return - } - - globals.Lock() - - _, ok = globals.volumeMap[volumeName] - if ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeCreated() called for preexiting Volume (%s)", volumeName) - return - } - - _, ok = globals.accountMap[volume.accountName] - if ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeCreated() called for preexiting Account (%s)", volume.accountName) - return - } - - volumeGroup, ok = globals.volumeGroupMap[volumeGroupName] - if !ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeCreated() called for Volume (%s) to be added to nonexistent VolumeGroup (%s)", volumeName, volumeGroupName) - return - } - - volumeGroup.Lock() - - _, ok = volumeGroup.volumeMap[volumeName] - if ok { - volumeGroup.Unlock() - globals.Unlock() - err = fmt.Errorf("inode.VolumeCreated() called for preexiting Volume (%s) to be added to VolumeGroup (%s)", volumeName, volumeGroupName) - return - } - - volume.volumeGroup = volumeGroup - volumeGroup.volumeMap[volumeName] = volume - globals.volumeMap[volumeName] = volume - globals.accountMap[volume.accountName] = volume - - volumeGroup.Unlock() - globals.Unlock() - - err = nil - return -} - -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - var ( - newVolumeGroup *volumeGroupStruct - ok bool - oldVolumeGroup *volumeGroupStruct - volume *volumeStruct - ) - - globals.Lock() - - volume, ok = globals.volumeMap[volumeName] - if !ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeMoved() called for nonexistent Volume (%s)", volumeName) - return - } - - if volume.served { - globals.Unlock() - err = fmt.Errorf("inode.VolumeMoved() called for Volume (%s) being actively served", volumeName) - return - } - - newVolumeGroup, ok = globals.volumeGroupMap[volumeGroupName] - if !ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeMoved() called for Volume (%s) to be moved to nonexistent VolumeGroup (%s)", volumeName, volumeGroupName) - return - } - - newVolumeGroup.Lock() - - _, ok = newVolumeGroup.volumeMap[volumeName] - if ok { - newVolumeGroup.Unlock() - globals.Unlock() - err = fmt.Errorf("inode.VolumeMoved() called for Volume (%s) to be moved to VolumeGroup (%s) already containing the Volume", volumeName, volumeGroupName) - return - } - - oldVolumeGroup = volume.volumeGroup - - oldVolumeGroup.Lock() - - delete(oldVolumeGroup.volumeMap, volumeName) - newVolumeGroup.volumeMap[volumeName] = volume - volume.volumeGroup = newVolumeGroup - - // Note that FSID & AccountName are not reloaded - - oldVolumeGroup.Unlock() - newVolumeGroup.Unlock() - globals.Unlock() - - err = nil - return -} - -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - var ( - ok bool - volume *volumeStruct - ) - - globals.Lock() - - volume, ok = globals.volumeMap[volumeName] - if !ok { - globals.Unlock() - err = fmt.Errorf("inode.VolumeDestroyed() called for nonexistent Volume (%s)", volumeName) - return - } - - if volume.served { - globals.Unlock() - err = fmt.Errorf("inode.VolumeDestroyed() called for Volume (%s) being actively served", volumeName) - return - } - - volume.volumeGroup.Lock() - - delete(volume.volumeGroup.volumeMap, volumeName) - delete(globals.volumeMap, volumeName) - delete(globals.accountMap, volume.accountName) - - volume.volumeGroup.Unlock() - globals.Unlock() - - err = nil - return -} - -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - defaultPhysicalContainerLayout *physicalContainerLayoutStruct - defaultPhysicalContainerLayoutName string - defaultPhysicalContainerLayoutSectionName string - ok bool - volume *volumeStruct - volumeSectionName string - ) - - volumeSectionName = "Volume:" + volumeName - - globals.Lock() - - volume, ok = globals.volumeMap[volumeName] - if !ok { - globals.Unlock() - err = fmt.Errorf("inode.ServeVolume() called for nonexistent Volume (%s)", volumeName) - return - } - if volume.served { - globals.Unlock() - err = fmt.Errorf("inode.ServeVolume() called for Volume (%s) already being served", volumeName) - return - } - - volume.maxEntriesPerDirNode, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxEntriesPerDirNode") - if nil != err { - globals.Unlock() - return - } - volume.maxExtentsPerFileNode, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxExtentsPerFileNode") - if nil != err { - globals.Unlock() - return - } - defaultPhysicalContainerLayoutName, err = confMap.FetchOptionValueString(volumeSectionName, "DefaultPhysicalContainerLayout") - if nil != err { - globals.Unlock() - return - } - - defaultPhysicalContainerLayout = &physicalContainerLayoutStruct{ - name: defaultPhysicalContainerLayoutName, - containerNameSliceNextIndex: 0, - containerNameSliceLoopCount: 0, - } - - defaultPhysicalContainerLayoutSectionName = "PhysicalContainerLayout:" + defaultPhysicalContainerLayoutName - - defaultPhysicalContainerLayout.containerStoragePolicy, err = confMap.FetchOptionValueString(defaultPhysicalContainerLayoutSectionName, "ContainerStoragePolicy") - if nil != err { - globals.Unlock() - return - } - defaultPhysicalContainerLayout.containerNamePrefix, err = confMap.FetchOptionValueString(defaultPhysicalContainerLayoutSectionName, "ContainerNamePrefix") - if nil != err { - globals.Unlock() - return - } - defaultPhysicalContainerLayout.containersPerPeer, err = confMap.FetchOptionValueUint64(defaultPhysicalContainerLayoutSectionName, "ContainersPerPeer") - if nil != err { - globals.Unlock() - return - } - defaultPhysicalContainerLayout.maxObjectsPerContainer, err = confMap.FetchOptionValueUint64(defaultPhysicalContainerLayoutSectionName, "MaxObjectsPerContainer") - if nil != err { - globals.Unlock() - return - } - - defaultPhysicalContainerLayout.containerNameSlice = make([]string, defaultPhysicalContainerLayout.containersPerPeer) - - volume.defaultPhysicalContainerLayout = defaultPhysicalContainerLayout - - volume.maxFlushSize, err = confMap.FetchOptionValueUint64(volumeSectionName, "MaxFlushSize") - if nil != err { - globals.Unlock() - return - } - - volume.headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(volume.volumeName) - if nil != err { - globals.Unlock() - return - } - - volume.headhunterVolumeHandle.RegisterForEvents(volume) - - volume.inodeCache = sortedmap.NewLLRBTree(compareInodeNumber, volume) - volume.inodeCacheStopChan = make(chan struct{}, 0) - volume.inodeCacheLRUHead = nil - volume.inodeCacheLRUTail = nil - volume.inodeCacheLRUItems = 0 - - err = startInodeCacheDiscard(confMap, volume, volumeSectionName) - if nil != err { - globals.Unlock() - return - } - - volume.volumeGroup.Lock() - - volume.served = true - volume.volumeGroup.numServed++ - - // temporary value until we can look across all volume groups to compute it - volume.volumeGroup.readCacheLineCount = 1 - - volume.volumeGroup.Unlock() - - globals.Unlock() - - return -} - -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - ok bool - volume *volumeStruct - ) - - globals.Lock() - - volume, ok = globals.volumeMap[volumeName] - if !ok { - globals.Unlock() - err = fmt.Errorf("inode.UnserveVolume() called for nonexistent Volume (%s)", volumeName) - return - } - if !volume.served { - globals.Unlock() - err = fmt.Errorf("inode.UnserveVolume() called for Volume (%s) not being served", volumeName) - return - } - - stopInodeCacheDiscard(volume) - - volume.inodeCache = nil - volume.inodeCacheStopChan = nil - volume.inodeCacheLRUHead = nil - volume.inodeCacheLRUTail = nil - volume.inodeCacheLRUItems = 0 - - volume.headhunterVolumeHandle.UnregisterForEvents(volume) - - volume.volumeGroup.Lock() - - volume.served = false - volume.volumeGroup.numServed-- - - volume.volumeGroup.Unlock() - globals.Unlock() - - return -} - -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - err = nil - return -} - -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - var ( - volume *volumeStruct - ) - - for _, volume = range globals.volumeMap { - if volume.served && (nil != volume.snapShotPolicy) { - volume.snapShotPolicy.down() - volume.snapShotPolicy = nil - } - } - - err = nil - return -} - -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - var ( - swiftReconNoWriteErrno string - swiftReconReadOnlyErrno string - volume *volumeStruct - ) - - // now that the information for all volume groups is available, compute - // the read cache size per volume group - err = adoptVolumeGroupReadCacheParameters(confMap) - if err != nil { - // fatal - return - } - - swiftReconNoWriteErrno, err = confMap.FetchOptionValueString("SwiftClient", "SwiftReconNoWriteErrno") - if nil == err { - switch swiftReconNoWriteErrno { - case "EPERM": - globals.noWriteThresholdErrno = blunder.NotPermError - globals.noWriteThresholdErrnoString = "EPERM" - case "EROFS": - globals.noWriteThresholdErrno = blunder.ReadOnlyError - globals.noWriteThresholdErrnoString = "EROFS" - case "ENOSPC": - globals.noWriteThresholdErrno = blunder.NoSpaceError - globals.noWriteThresholdErrnoString = "ENOSPC" - default: - err = fmt.Errorf("[SwiftClient]SwiftReconReadOnlyErrno must be either EPERM or EROFS or ENOSPC") - return - } - } else { - logger.WarnfWithError(err, "Unable to fetch [SwiftClient]SwiftReconNoWriteErrno... defaulting to %s", defaultNoWriteErrno) - globals.noWriteThresholdErrno = defaultNoWriteErrno - globals.noWriteThresholdErrnoString = defaultNoWriteErrnoString - } - - swiftReconReadOnlyErrno, err = confMap.FetchOptionValueString("SwiftClient", "SwiftReconReadOnlyErrno") - if nil == err { - switch swiftReconReadOnlyErrno { - case "EPERM": - globals.readOnlyThresholdErrno = blunder.NotPermError - globals.readOnlyThresholdErrnoString = "EPERM" - case "EROFS": - globals.readOnlyThresholdErrno = blunder.ReadOnlyError - globals.readOnlyThresholdErrnoString = "EROFS" - case "ENOSPC": - globals.readOnlyThresholdErrno = blunder.NoSpaceError - globals.readOnlyThresholdErrnoString = "ENOSPC" - default: - err = fmt.Errorf("[SwiftClient]SwiftReconReadOnlyErrno must be either EPERM or EROFS or ENOSPC") - return - } - } else { - logger.WarnfWithError(err, "Unable to fetch [SwiftClient]SwiftReconReadOnlyErrno... defaulting to %s", defaultReadOnlyErrno) - globals.readOnlyThresholdErrno = defaultReadOnlyErrno - globals.readOnlyThresholdErrnoString = defaultReadOnlyErrnoString - } - - for _, volume = range globals.volumeMap { - if volume.served { - err = volume.loadSnapShotPolicy(confMap) - if nil != err { - return - } - if nil != volume.snapShotPolicy { - volume.snapShotPolicy.up() - } - } - } - - err = nil - return -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - if 0 != len(globals.volumeGroupMap) { - err = fmt.Errorf("inode.Down() called with 0 != len(globals.volumeGroupMap") - return - } - if 0 != len(globals.volumeMap) { - err = fmt.Errorf("inode.Down() called with 0 != len(globals.volumeMap") - return - } - - err = nil - return -} diff --git a/inode/config_test.go b/inode/config_test.go deleted file mode 100644 index c10a6d9c..00000000 --- a/inode/config_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "testing" - - "github.com/NVIDIA/proxyfs/transitions" - - "github.com/stretchr/testify/assert" -) - -func TestConfig(t *testing.T) { - var err error - - assert := assert.New(t) - - testSetup(t, false) - - // verify that proxyfs doesn't panic for a bad config value - // (remove the volume group we're going to update) - testConfUpdateStrings := []string{ - "FSGlobals.VolumeGroupList=", - } - - err = testConfMap.UpdateFromStrings(testConfUpdateStrings) - assert.Nil(err, "testConfMap.UpdateFromStrings(testConfUpdateStrings) failed") - - err = transitions.Signaled(testConfMap) - assert.Nil(err, "transitions.Signaled failed") - - // now try with bogus ReadCacheWeight - testConfUpdateStrings = []string{ - "FSGlobals.VolumeGroupList=TestVolumeGroup", - "VolumeGroup:TestVolumeGroup.ReadCacheWeight=0", - "VolumeGroup:TestVolumeGroup.ReadCacheLineSize=0", - } - - err = testConfMap.UpdateFromStrings(testConfUpdateStrings) - assert.Nil(err, "testConfMap.UpdateFromStrings(testConfUpdateStrings) failed") - - err = transitions.Signaled(testConfMap) - assert.Nil(err, "transitions.Signaled failed") - - testTeardown(t) -} diff --git a/inode/cron.go b/inode/cron.go deleted file mode 100644 index 0ce54846..00000000 --- a/inode/cron.go +++ /dev/null @@ -1,645 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - "strconv" - "strings" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/logger" -) - -type snapShotScheduleStruct struct { - name string - policy *snapShotPolicyStruct - minuteSpecified bool - minute int // 0-59 - hourSpecified bool - hour int // 0-23 - dayOfMonthSpecified bool - dayOfMonth int // 1-31 - monthSpecified bool - month time.Month // 1-12 - dayOfWeekSpecified bool - dayOfWeek time.Weekday // 0-6 (0 == Sunday) - keep uint64 - count uint64 // computed by scanning each time daemon() awakes -} - -type snapShotPolicyStruct struct { - name string - volume *volumeStruct - schedule []*snapShotScheduleStruct - location *time.Location - stopChan chan struct{} - doneWaitGroup sync.WaitGroup -} - -func (vS *volumeStruct) loadSnapShotPolicy(confMap conf.ConfMap) (err error) { - var ( - cronTabStringSlice []string - dayOfMonthAsU64 uint64 - dayOfWeekAsU64 uint64 - hourAsU64 uint64 - minuteAsU64 uint64 - monthAsU64 uint64 - snapShotPolicy *snapShotPolicyStruct - snapShotPolicyName string - snapShotPolicySectionName string - snapShotSchedule *snapShotScheduleStruct - snapShotScheduleList []string - snapShotScheduleName string - snapShotScheduleSectionName string - timeZone string - volumeSectionName string - ) - - // Default to no snapShotPolicy found - vS.snapShotPolicy = nil - - volumeSectionName = "Volume:" + vS.volumeName - - snapShotPolicyName, err = confMap.FetchOptionValueString(volumeSectionName, "SnapShotPolicy") - if nil != err { - // Default to setting snapShotPolicy to nil and returning success - err = nil - return - } - - snapShotPolicy = &snapShotPolicyStruct{name: snapShotPolicyName, volume: vS} - - snapShotPolicySectionName = "SnapShotPolicy:" + snapShotPolicyName - - snapShotScheduleList, err = confMap.FetchOptionValueStringSlice(snapShotPolicySectionName, "ScheduleList") - if nil != err { - return - } - if 0 == len(snapShotScheduleList) { - // If ScheduleList is empty, set snapShotPolicy to nil and return success - err = nil - return - } - snapShotPolicy.schedule = make([]*snapShotScheduleStruct, 0, len(snapShotScheduleList)) - for _, snapShotScheduleName = range snapShotScheduleList { - snapShotScheduleSectionName = "SnapShotSchedule:" + snapShotScheduleName - - snapShotSchedule = &snapShotScheduleStruct{name: snapShotScheduleName, policy: snapShotPolicy} - - cronTabStringSlice, err = confMap.FetchOptionValueStringSlice(snapShotScheduleSectionName, "CronTab") - if nil != err { - return - } - if 5 != len(cronTabStringSlice) { - err = fmt.Errorf("%v.CronTab must be a 5 element crontab time specification", snapShotScheduleSectionName) - return - } - - if "*" == cronTabStringSlice[0] { - snapShotSchedule.minuteSpecified = false - } else { - snapShotSchedule.minuteSpecified = true - - minuteAsU64, err = strconv.ParseUint(cronTabStringSlice[0], 10, 8) - if nil != err { - return - } - if 59 < minuteAsU64 { - err = fmt.Errorf("%v.CronTab[0] must be valid minute (0-59)", snapShotScheduleSectionName) - return - } - - snapShotSchedule.minute = int(minuteAsU64) - } - - if "*" == cronTabStringSlice[1] { - snapShotSchedule.hourSpecified = false - } else { - snapShotSchedule.hourSpecified = true - - hourAsU64, err = strconv.ParseUint(cronTabStringSlice[1], 10, 8) - if nil != err { - return - } - if 23 < hourAsU64 { - err = fmt.Errorf("%v.CronTab[1] must be valid hour (0-23)", snapShotScheduleSectionName) - return - } - - snapShotSchedule.hour = int(hourAsU64) - } - - if "*" == cronTabStringSlice[2] { - snapShotSchedule.dayOfMonthSpecified = false - } else { - snapShotSchedule.dayOfMonthSpecified = true - - dayOfMonthAsU64, err = strconv.ParseUint(cronTabStringSlice[2], 10, 8) - if nil != err { - return - } - if (0 == dayOfMonthAsU64) || (31 < dayOfMonthAsU64) { - err = fmt.Errorf("%v.CronTab[2] must be valid dayOfMonth (1-31)", snapShotScheduleSectionName) - return - } - - snapShotSchedule.dayOfMonth = int(dayOfMonthAsU64) - } - - if "*" == cronTabStringSlice[3] { - snapShotSchedule.monthSpecified = false - } else { - snapShotSchedule.monthSpecified = true - - monthAsU64, err = strconv.ParseUint(cronTabStringSlice[3], 10, 8) - if nil != err { - return - } - if (0 == monthAsU64) || (12 < monthAsU64) { - err = fmt.Errorf("%v.CronTab[3] must be valid month (1-12)", snapShotScheduleSectionName) - return - } - - snapShotSchedule.month = time.Month(monthAsU64) - } - - if "*" == cronTabStringSlice[4] { - snapShotSchedule.dayOfWeekSpecified = false - } else { - snapShotSchedule.dayOfWeekSpecified = true - - dayOfWeekAsU64, err = strconv.ParseUint(cronTabStringSlice[4], 10, 8) - if nil != err { - return - } - if 6 < dayOfWeekAsU64 { - err = fmt.Errorf("%v.CronTab[4] must be valid dayOfWeek (0-6)", snapShotScheduleSectionName) - return - } - - snapShotSchedule.dayOfWeek = time.Weekday(dayOfWeekAsU64) - } - - snapShotSchedule.keep, err = confMap.FetchOptionValueUint64(snapShotScheduleSectionName, "Keep") - if nil != err { - return - } - - if snapShotSchedule.dayOfWeekSpecified && (snapShotSchedule.dayOfMonthSpecified || snapShotSchedule.monthSpecified) { - err = fmt.Errorf("%v.CronTab must not specify DayOfWeek if DayOfMonth and/or Month are specified", snapShotScheduleSectionName) - return - } - - snapShotPolicy.schedule = append(snapShotPolicy.schedule, snapShotSchedule) - } - - timeZone, err = confMap.FetchOptionValueString(snapShotPolicySectionName, "TimeZone") - - if nil == err { - snapShotPolicy.location, err = time.LoadLocation(timeZone) - if nil != err { - return - } - } else { // nil != err - // If not present, default to UTC - snapShotPolicy.location = time.UTC - } - - // If we reach here, we've successfully loaded the snapShotPolicy - - vS.snapShotPolicy = snapShotPolicy - err = nil - return -} - -func (snapShotPolicy *snapShotPolicyStruct) up() { - snapShotPolicy.stopChan = make(chan struct{}, 1) - snapShotPolicy.doneWaitGroup.Add(1) - go snapShotPolicy.daemon() -} - -func (snapShotPolicy *snapShotPolicyStruct) down() { - snapShotPolicy.stopChan <- struct{}{} - snapShotPolicy.doneWaitGroup.Wait() -} - -func (snapShotPolicy *snapShotPolicyStruct) daemon() { - var ( - err error - nextDuration time.Duration - nextTime time.Time - nextTimePreviously time.Time - snapShotName string - timeNow time.Time - ) - - nextTimePreviously = time.Date(2000, time.January, 1, 0, 0, 0, 0, snapShotPolicy.location) - - for { - timeNow = time.Now().In(snapShotPolicy.location) - nextTime = snapShotPolicy.next(timeNow) - for { - if !nextTime.Equal(nextTimePreviously) { - break - } - // We took the last snapshot so quickly, next() returned the same nextTime - time.Sleep(time.Second) - timeNow = time.Now().In(snapShotPolicy.location) - nextTime = snapShotPolicy.next(timeNow) - } - nextDuration = nextTime.Sub(timeNow) - select { - case _ = <-snapShotPolicy.stopChan: - snapShotPolicy.doneWaitGroup.Done() - return - case <-time.After(nextDuration): - for time.Now().In(snapShotPolicy.location).Before(nextTime) { - // If time.After() returned a bit too soon, loop until it is our time - time.Sleep(100 * time.Millisecond) - } - snapShotName = strings.Replace(nextTime.Format(time.RFC3339), ":", ".", -1) - _, err = snapShotPolicy.volume.SnapShotCreate(snapShotName) - if nil != err { - logger.WarnWithError(err) - } - snapShotPolicy.prune() - } - nextTimePreviously = nextTime - } -} - -func (snapShotPolicy *snapShotPolicyStruct) prune() { - var ( - err error - keep bool - matches bool - matchesAtLeastOne bool - snapShot headhunter.SnapShotStruct - snapShotList []headhunter.SnapShotStruct - snapShotSchedule *snapShotScheduleStruct - snapShotTime time.Time - ) - - // First, zero each snapShotSchedule.count - - for _, snapShotSchedule = range snapShotPolicy.schedule { - snapShotSchedule.count = 0 - } - - // Now fetch the reverse time-ordered list of snapshots - - snapShotList = snapShotPolicy.volume.headhunterVolumeHandle.SnapShotListByTime(true) - - // Now walk snapShotList looking for snapshots to prune - - for _, snapShot = range snapShotList { - snapShotTime, err = time.Parse(time.RFC3339, strings.Replace(snapShot.Name, ".", ":", -1)) - if nil != err { - // SnapShot was not formatted to match a potential SnapShotPolicy/Schedule...skip it - continue - } - - // Compare against each snapShotSchedule - - keep = false - matchesAtLeastOne = false - - for _, snapShotSchedule = range snapShotPolicy.schedule { - matches = snapShotSchedule.compare(snapShotTime) - if matches { - matchesAtLeastOne = true - snapShotSchedule.count++ - if snapShotSchedule.count <= snapShotSchedule.keep { - keep = true - } - } - } - - if matchesAtLeastOne && !keep { - // Although this snapshot "matchesAtLeastOne", - // no snapShotSchedule said "keep" it - - err = snapShotPolicy.volume.SnapShotDelete(snapShot.ID) - if nil != err { - logger.WarnWithError(err) - } - } - } -} - -// thisTime is presumably the snapShotSchedule.policy.location-local parsed snapShotStruct.name -func (snapShotSchedule *snapShotScheduleStruct) compare(thisTime time.Time) (matches bool) { - var ( - dayOfMonth int - dayOfWeek time.Weekday - hour int - minute int - month time.Month - truncatedTime time.Time - year int - ) - - hour, minute, _ = thisTime.Clock() - year, month, dayOfMonth = thisTime.Date() - dayOfWeek = thisTime.Weekday() - - truncatedTime = time.Date(year, month, dayOfMonth, hour, minute, 0, 0, snapShotSchedule.policy.location) - if !truncatedTime.Equal(thisTime) { - matches = false - return - } - - if snapShotSchedule.minuteSpecified { - if snapShotSchedule.minute != minute { - matches = false - return - } - } - - if snapShotSchedule.hourSpecified { - if snapShotSchedule.hour != hour { - matches = false - return - } - } - - if snapShotSchedule.dayOfMonthSpecified { - if snapShotSchedule.dayOfMonth != dayOfMonth { - matches = false - return - } - } - - if snapShotSchedule.monthSpecified { - if snapShotSchedule.month != month { - matches = false - return - } - } - - if snapShotSchedule.dayOfWeekSpecified { - if snapShotSchedule.dayOfWeek != dayOfWeek { - matches = false - return - } - } - - // If we make it this far, thisTime matches snapShotSchedule - - matches = true - return -} - -// Since time.Truncate() only truncates with respect to UTC, it is unsafe - -func truncateToStartOfMinute(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { - var ( - day int - hour int - min int - month time.Month - year int - ) - - hour, min, _ = untruncatedTime.Clock() - year, month, day = untruncatedTime.Date() - - truncatedTime = time.Date(year, month, day, hour, min, 0, 0, loc) - - return -} - -func truncateToStartOfHour(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { - var ( - day int - hour int - month time.Month - year int - ) - - hour, _, _ = untruncatedTime.Clock() - year, month, day = untruncatedTime.Date() - - truncatedTime = time.Date(year, month, day, hour, 0, 0, 0, loc) - - return -} - -func truncateToStartOfDay(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { - var ( - day int - month time.Month - year int - ) - - year, month, day = untruncatedTime.Date() - - truncatedTime = time.Date(year, month, day, 0, 0, 0, 0, loc) - - return -} - -func truncateToStartOfMonth(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { - var ( - month time.Month - year int - ) - - year, month, _ = untruncatedTime.Date() - - truncatedTime = time.Date(year, month, 1, 0, 0, 0, 0, loc) - - return -} - -// timeNow is presumably time.Now() localized to snapShotSchedule.policy.location... -// ...but provided here so that each invocation of the per snapShotSchedule -// within a snapShotPolicy can use the same value -func (snapShotSchedule *snapShotScheduleStruct) next(timeNow time.Time) (nextTime time.Time) { - var ( - dayOfMonth int - dayOfWeek time.Weekday - hour int - minute int - month time.Month - numDaysToAdd int - numHoursToAdd int - numMinutesToAdd int - year int - ) - - // Ensure nextTime is at least at the start of the next minute - nextTime = truncateToStartOfMinute(timeNow, snapShotSchedule.policy.location).Add(time.Minute) - - if snapShotSchedule.minuteSpecified { - minute = nextTime.Minute() - if snapShotSchedule.minute == minute { - // We don't need to advance nextTime - } else { - // No need to (again) truncate nextTime back to the start of the minute - // Now advance nextTime to align with minute - if snapShotSchedule.minute > minute { - numMinutesToAdd = snapShotSchedule.minute - minute - } else { // snapShotSchedule.minute < minute - numMinutesToAdd = snapShotSchedule.minute + 60 - minute - } - nextTime = nextTime.Add(time.Duration(numMinutesToAdd) * time.Minute) - } - } - - if snapShotSchedule.hourSpecified { - hour = nextTime.Hour() - if snapShotSchedule.hour == hour { - // We don't need to advance nextTime - } else { - // First truncate nextTime back to the start of the hour - nextTime = truncateToStartOfHour(nextTime, snapShotSchedule.policy.location) - // Restore minuteSpecified if necessary - if snapShotSchedule.minuteSpecified { - nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) - } - // Now advance nextTime to align with hour - if snapShotSchedule.hour > hour { - numHoursToAdd = snapShotSchedule.hour - hour - } else { // snapShotSchedule.hour < hour - numHoursToAdd = snapShotSchedule.hour + 24 - hour - } - nextTime = nextTime.Add(time.Duration(numHoursToAdd) * time.Hour) - } - } - - if snapShotSchedule.dayOfMonthSpecified { - dayOfMonth = nextTime.Day() - if snapShotSchedule.dayOfMonth == dayOfMonth { - // We don't need to advance nextTime - } else { - // First truncate nextTime back to the start of the day - nextTime = truncateToStartOfDay(nextTime, snapShotSchedule.policy.location) - // Restore minuteSpecified and/or hourSpecified if necessary - if snapShotSchedule.minuteSpecified { - nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) - } - if snapShotSchedule.hourSpecified { - nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) - } - // Now advance nextTime to align with dayOfMonth - // Note: This unfortunately iterative approach avoids complicated - // adjustments for the non-fixed number of days in a month - for { - nextTime = nextTime.Add(24 * time.Hour) - dayOfMonth = nextTime.Day() - if snapShotSchedule.dayOfMonth == dayOfMonth { - break - } - } - } - } - - if snapShotSchedule.monthSpecified { - month = nextTime.Month() - if snapShotSchedule.month == month { - // We don't need to advance nextTime - } else { - // First truncate nextTime back to the start of the month - nextTime = truncateToStartOfMonth(nextTime, snapShotSchedule.policy.location) - // Restore minuteSpecified, hourSpecified, and/or dayOfMonthSpecified if necessary - if snapShotSchedule.minuteSpecified { - nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) - } - if snapShotSchedule.hourSpecified { - nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) - } - if snapShotSchedule.dayOfMonthSpecified { - nextTime = nextTime.Add(time.Duration((snapShotSchedule.dayOfMonth-1)*24) * time.Hour) - } - // Now advance nextTime to align with month - // Note: This unfortunately iterative approach avoids complicated - // adjustments for the non-fixed number of days in a month - hour, minute, _ = nextTime.Clock() - year, month, dayOfMonth = nextTime.Date() - if !snapShotSchedule.dayOfMonthSpecified { - dayOfMonth = 1 - } - for { - if time.December == month { - month = time.January - year++ - } else { - month++ - } - nextTime = time.Date(year, month, dayOfMonth, hour, minute, 0, 0, snapShotSchedule.policy.location) - year, month, dayOfMonth = nextTime.Date() - if snapShotSchedule.dayOfMonthSpecified { - if (snapShotSchedule.month == month) && (snapShotSchedule.dayOfMonth == dayOfMonth) { - break - } else { - dayOfMonth = snapShotSchedule.dayOfMonth - } - } else { - if (snapShotSchedule.month == month) && (1 == dayOfMonth) { - break - } else { - dayOfMonth = 1 - } - } - } - } - } - - if snapShotSchedule.dayOfWeekSpecified { - dayOfWeek = nextTime.Weekday() - if time.Weekday(snapShotSchedule.dayOfWeek) == dayOfWeek { - // We don't need to advance nextTime - } else { - // First truncate nextTime back to the start of the day - nextTime = truncateToStartOfDay(nextTime, snapShotSchedule.policy.location) - // Restore minuteSpecified and/or hourSpecified if necessary - if snapShotSchedule.minuteSpecified { - nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) - } - if snapShotSchedule.hourSpecified { - nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) - } - // Now advance nextTime to align with dayOfWeek - if time.Weekday(snapShotSchedule.dayOfWeek) > dayOfWeek { - numDaysToAdd = int(snapShotSchedule.dayOfWeek) - int(dayOfWeek) - } else { // time.Weekday(snapShotSchedule.dayOfWeek) < dayOfWeek - numDaysToAdd = int(snapShotSchedule.dayOfWeek) + 7 - int(dayOfWeek) - } - nextTime = nextTime.Add(time.Duration(24*numDaysToAdd) * time.Hour) - } - } - - return -} - -// timeNow is presumably time.Now() localized to snapShotPolicy.location... -// ...but provided here primarily to enable easy testing -func (snapShotPolicy *snapShotPolicyStruct) next(timeNow time.Time) (nextTime time.Time) { - var ( - nextTimeForSnapShotSchedule time.Time - nextTimeHasBeenSet bool - snapShotSchedule *snapShotScheduleStruct - ) - - nextTimeHasBeenSet = false - - for _, snapShotSchedule = range snapShotPolicy.schedule { - nextTimeForSnapShotSchedule = snapShotSchedule.next(timeNow) - if nextTimeHasBeenSet { - if nextTimeForSnapShotSchedule.Before(nextTime) { - nextTime = nextTimeForSnapShotSchedule - } - } else { - nextTime = nextTimeForSnapShotSchedule - nextTimeHasBeenSet = true - } - } - - return -} diff --git a/inode/cron_test.go b/inode/cron_test.go deleted file mode 100644 index f4921e83..00000000 --- a/inode/cron_test.go +++ /dev/null @@ -1,728 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "testing" - "time" - - "github.com/NVIDIA/proxyfs/conf" -) - -func TestLoadSnapShotPolicy(t *testing.T) { - var ( - err error - testConfMap conf.ConfMap - testConfMapStrings []string - volume *volumeStruct - ) - - // Case 1 - no SnapShotPolicy - - testConfMapStrings = []string{} - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("Case 1: conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("Case 1: loadSnapShotPolicy() failed: %v", err) - } - - if nil != volume.snapShotPolicy { - t.Fatalf("Case 1: loadSnapShotPolicy() returned non-nil snapShotPolicy") - } - - // Case 2 - SnapShotPolicy with empty ScheduleList and no TimeZone - - testConfMapStrings = []string{ - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("Case 2: conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("Case 2: loadSnapShotPolicy() failed: %v", err) - } - - if nil != volume.snapShotPolicy { - t.Fatalf("Case 2: loadSnapShotPolicy() returned non-nil snapShotPolicy") - } - - // Case 3 - SnapShotPolicy with trivial ScheduleList and no TimeZone - - testConfMapStrings = []string{ - "SnapShotSchedule:MinutelySnapShotSchedule.CronTab=* * * * *", // ==> snapShotPolicy.schedule[0] - "SnapShotSchedule:MinutelySnapShotSchedule.Keep=59", - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=MinutelySnapShotSchedule", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("Case 3: conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("Case 3: loadSnapShotPolicy() failed: %v", err) - } - - if "CommonSnapShotPolicy" != volume.snapShotPolicy.name { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy with unexpected .name") - } - if 1 != len(volume.snapShotPolicy.schedule) { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy with unexpected .schedule") - } - if "MinutelySnapShotSchedule" != volume.snapShotPolicy.schedule[0].name { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .name") - } - if volume.snapShotPolicy.schedule[0].minuteSpecified { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .minuteSpecified") - } - if volume.snapShotPolicy.schedule[0].hourSpecified { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .hourSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfMonthSpecified { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfMonthSpecified") - } - if volume.snapShotPolicy.schedule[0].monthSpecified { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .monthSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfWeekSpecified { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfWeekSpecified") - } - if 59 != volume.snapShotPolicy.schedule[0].keep { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .keep") - } - if "UTC" != volume.snapShotPolicy.location.String() { - t.Fatalf("Case 3: loadSnapShotPolicy() returned snapShotPolicy with unexpected .location") - } - - // Case 4 - SnapShotPolicy with trivial ScheduleList and empty TimeZone - - testConfMapStrings = []string{ - "SnapShotSchedule:MinutelySnapShotSchedule.CronTab=* * * * *", // ==> snapShotPolicy.schedule[0] - "SnapShotSchedule:MinutelySnapShotSchedule.Keep=59", - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=MinutelySnapShotSchedule", - "SnapShotPolicy:CommonSnapShotPolicy.TimeZone=", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("Case 4: conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("Case 4: loadSnapShotPolicy() failed: %v", err) - } - - if "CommonSnapShotPolicy" != volume.snapShotPolicy.name { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy with unexpected .name") - } - if 1 != len(volume.snapShotPolicy.schedule) { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy with unexpected .schedule") - } - if "MinutelySnapShotSchedule" != volume.snapShotPolicy.schedule[0].name { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .name") - } - if volume.snapShotPolicy.schedule[0].minuteSpecified { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .minuteSpecified") - } - if volume.snapShotPolicy.schedule[0].hourSpecified { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .hourSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfMonthSpecified { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfMonthSpecified") - } - if volume.snapShotPolicy.schedule[0].monthSpecified { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .monthSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfWeekSpecified { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfWeekSpecified") - } - if 59 != volume.snapShotPolicy.schedule[0].keep { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .keep") - } - if "UTC" != volume.snapShotPolicy.location.String() { - t.Fatalf("Case 4: loadSnapShotPolicy() returned snapShotPolicy with unexpected .location") - } - - // Case 5 - SnapShotPolicy with trivial ScheduleList and TimeZone of "UTC" - - testConfMapStrings = []string{ - "SnapShotSchedule:MinutelySnapShotSchedule.CronTab=* * * * *", // ==> snapShotPolicy.schedule[0] - "SnapShotSchedule:MinutelySnapShotSchedule.Keep=59", - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=MinutelySnapShotSchedule", - "SnapShotPolicy:CommonSnapShotPolicy.TimeZone=UTC", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("Case 5: conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("Case 5: loadSnapShotPolicy() failed: %v", err) - } - - if "CommonSnapShotPolicy" != volume.snapShotPolicy.name { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy with unexpected .name") - } - if 1 != len(volume.snapShotPolicy.schedule) { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy with unexpected .schedule") - } - if "MinutelySnapShotSchedule" != volume.snapShotPolicy.schedule[0].name { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .name") - } - if volume.snapShotPolicy.schedule[0].minuteSpecified { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .minuteSpecified") - } - if volume.snapShotPolicy.schedule[0].hourSpecified { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .hourSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfMonthSpecified { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfMonthSpecified") - } - if volume.snapShotPolicy.schedule[0].monthSpecified { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .monthSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfWeekSpecified { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfWeekSpecified") - } - if 59 != volume.snapShotPolicy.schedule[0].keep { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .keep") - } - if "UTC" != volume.snapShotPolicy.location.String() { - t.Fatalf("Case 5: loadSnapShotPolicy() returned snapShotPolicy with unexpected .location") - } - - // Case 6 - SnapShotPolicy with trivial ScheduleList and TimeZone of "Local" - - testConfMapStrings = []string{ - "SnapShotSchedule:MinutelySnapShotSchedule.CronTab=* * * * *", // ==> snapShotPolicy.schedule[0] - "SnapShotSchedule:MinutelySnapShotSchedule.Keep=59", - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=MinutelySnapShotSchedule", - "SnapShotPolicy:CommonSnapShotPolicy.TimeZone=Local", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("Case 6: conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("Case 6: loadSnapShotPolicy() failed: %v", err) - } - - if "CommonSnapShotPolicy" != volume.snapShotPolicy.name { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy with unexpected .name") - } - if 1 != len(volume.snapShotPolicy.schedule) { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy with unexpected .schedule") - } - if "MinutelySnapShotSchedule" != volume.snapShotPolicy.schedule[0].name { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .name") - } - if volume.snapShotPolicy.schedule[0].minuteSpecified { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .minuteSpecified") - } - if volume.snapShotPolicy.schedule[0].hourSpecified { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .hourSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfMonthSpecified { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfMonthSpecified") - } - if volume.snapShotPolicy.schedule[0].monthSpecified { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .monthSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfWeekSpecified { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfWeekSpecified") - } - if 59 != volume.snapShotPolicy.schedule[0].keep { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .keep") - } - if "Local" != volume.snapShotPolicy.location.String() { - t.Fatalf("Case 6: loadSnapShotPolicy() returned snapShotPolicy with unexpected .location") - } - - // Case 7 - SnapShotPolicy with exhaustive ScheduleList and a specific TimeZone - - testConfMapStrings = []string{ - "SnapShotSchedule:MinutelySnapShotSchedule.CronTab=* * * * *", // ==> snapShotPolicy.schedule[0] - "SnapShotSchedule:MinutelySnapShotSchedule.Keep=59", - "SnapShotSchedule:HourlySnapShotSchedule.CronTab=0 * * * *", // ==> snapShotPolicy.schedule[1] - "SnapShotSchedule:HourlySnapShotSchedule.Keep=23", - "SnapShotSchedule:DailySnapShotSchedule.CronTab=0 0 * * *", // ==> snapShotPolicy.schedule[2] - "SnapShotSchedule:DailySnapShotSchedule.Keep=6", - "SnapShotSchedule:WeeklySnapShotSchedule.CronTab=0 0 * * 0", // ==> snapShotPolicy.schedule[3] - "SnapShotSchedule:WeeklySnapShotSchedule.Keep=8", - "SnapShotSchedule:MonthlySnapShotSchedule.CronTab=0 0 1 * *", // ==> snapShotPolicy.schedule[4] - "SnapShotSchedule:MonthlySnapShotSchedule.Keep=11", - "SnapShotSchedule:YearlySnapShotSchedule.CronTab=0 0 1 1 *", // ==> snapShotPolicy.schedule[5] - "SnapShotSchedule:YearlySnapShotSchedule.Keep=4", - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=MinutelySnapShotSchedule,HourlySnapShotSchedule,DailySnapShotSchedule,WeeklySnapShotSchedule,MonthlySnapShotSchedule,YearlySnapShotSchedule", - "SnapShotPolicy:CommonSnapShotPolicy.TimeZone=America/Los_Angeles", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("Case 7: conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("Case 7: loadSnapShotPolicy() failed: %v", err) - } - - if "CommonSnapShotPolicy" != volume.snapShotPolicy.name { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy with unexpected .name") - } - - if 6 != len(volume.snapShotPolicy.schedule) { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy with unexpected .schedule") - } - - if "MinutelySnapShotSchedule" != volume.snapShotPolicy.schedule[0].name { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .name") - } - if volume.snapShotPolicy.schedule[0].minuteSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .minuteSpecified") - } - if volume.snapShotPolicy.schedule[0].hourSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .hourSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfMonthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfMonthSpecified") - } - if volume.snapShotPolicy.schedule[0].monthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .monthSpecified") - } - if volume.snapShotPolicy.schedule[0].dayOfWeekSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .dayOfWeekSpecified") - } - if 59 != volume.snapShotPolicy.schedule[0].keep { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[0] with unexpected .keep") - } - - if "HourlySnapShotSchedule" != volume.snapShotPolicy.schedule[1].name { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[1] with unexpected .name") - } - if !volume.snapShotPolicy.schedule[1].minuteSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[1] with unexpected .minuteSpecified") - } - if 0 != volume.snapShotPolicy.schedule[1].minute { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[1] with unexpected .minute") - } - if volume.snapShotPolicy.schedule[1].hourSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[1] with unexpected .hourSpecified") - } - if volume.snapShotPolicy.schedule[1].dayOfMonthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[1] with unexpected .dayOfMonthSpecified") - } - if volume.snapShotPolicy.schedule[1].monthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[1] with unexpected .monthSpecified") - } - if volume.snapShotPolicy.schedule[1].dayOfWeekSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[1] with unexpected .dayOfWeekSpecified") - } - if 23 != volume.snapShotPolicy.schedule[1].keep { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[1] with unexpected .keep") - } - - if "DailySnapShotSchedule" != volume.snapShotPolicy.schedule[2].name { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .name") - } - if !volume.snapShotPolicy.schedule[2].minuteSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .minuteSpecified") - } - if 0 != volume.snapShotPolicy.schedule[2].minute { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .minute") - } - if !volume.snapShotPolicy.schedule[2].hourSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .hourSpecified") - } - if 0 != volume.snapShotPolicy.schedule[2].hour { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .hour") - } - if volume.snapShotPolicy.schedule[2].dayOfMonthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .dayOfMonthSpecified") - } - if volume.snapShotPolicy.schedule[2].monthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .monthSpecified") - } - if volume.snapShotPolicy.schedule[2].dayOfWeekSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .dayOfWeekSpecified") - } - if 6 != volume.snapShotPolicy.schedule[2].keep { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[2] with unexpected .keep") - } - - if "WeeklySnapShotSchedule" != volume.snapShotPolicy.schedule[3].name { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .name") - } - if !volume.snapShotPolicy.schedule[3].minuteSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .minuteSpecified") - } - if 0 != volume.snapShotPolicy.schedule[3].minute { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .minute") - } - if !volume.snapShotPolicy.schedule[3].hourSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .hourSpecified") - } - if 0 != volume.snapShotPolicy.schedule[3].hour { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .hour") - } - if volume.snapShotPolicy.schedule[3].dayOfMonthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .dayOfMonthSpecified") - } - if volume.snapShotPolicy.schedule[3].monthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .monthSpecified") - } - if !volume.snapShotPolicy.schedule[3].dayOfWeekSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .dayOfWeekSpecified") - } - if 0 != volume.snapShotPolicy.schedule[3].dayOfWeek { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .dayOfWeek") - } - if 8 != volume.snapShotPolicy.schedule[3].keep { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[3] with unexpected .keep") - } - - if "MonthlySnapShotSchedule" != volume.snapShotPolicy.schedule[4].name { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .name") - } - if !volume.snapShotPolicy.schedule[4].minuteSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .minuteSpecified") - } - if 0 != volume.snapShotPolicy.schedule[4].minute { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .minute") - } - if !volume.snapShotPolicy.schedule[4].hourSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .hourSpecified") - } - if 0 != volume.snapShotPolicy.schedule[4].hour { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .hour") - } - if !volume.snapShotPolicy.schedule[4].dayOfMonthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .dayOfMonthSpecified") - } - if 1 != volume.snapShotPolicy.schedule[4].dayOfMonth { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .dayOfMonth") - } - if volume.snapShotPolicy.schedule[4].monthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .monthSpecified") - } - if volume.snapShotPolicy.schedule[4].dayOfWeekSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .dayOfWeekSpecified") - } - if 11 != volume.snapShotPolicy.schedule[4].keep { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[4] with unexpected .keep") - } - - if "YearlySnapShotSchedule" != volume.snapShotPolicy.schedule[5].name { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .name") - } - if !volume.snapShotPolicy.schedule[5].minuteSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .minuteSpecified") - } - if 0 != volume.snapShotPolicy.schedule[5].minute { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .minute") - } - if !volume.snapShotPolicy.schedule[5].hourSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .hourSpecified") - } - if 0 != volume.snapShotPolicy.schedule[5].hour { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .hour") - } - if !volume.snapShotPolicy.schedule[5].dayOfMonthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .dayOfMonthSpecified") - } - if 1 != volume.snapShotPolicy.schedule[5].dayOfMonth { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .dayOfMonth") - } - if !volume.snapShotPolicy.schedule[5].monthSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .monthSpecified") - } - if 1 != volume.snapShotPolicy.schedule[5].month { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .month") - } - if volume.snapShotPolicy.schedule[5].dayOfWeekSpecified { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .dayOfWeekSpecified") - } - if 4 != volume.snapShotPolicy.schedule[5].keep { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy.schedule[5] with unexpected .keep") - } - - if "America/Los_Angeles" != volume.snapShotPolicy.location.String() { - t.Fatalf("Case 7: loadSnapShotPolicy() returned snapShotPolicy with unexpected .location") - } -} - -func TestSnapShotScheduleCompare(t *testing.T) { - var ( - err error - matches bool - matchingTime time.Time - mismatchingTime time.Time - testConfMap conf.ConfMap - testConfMapStrings []string - volume *volumeStruct - ) - - testConfMapStrings = []string{ - "SnapShotSchedule:MinutelySnapShotSchedule.CronTab=* * * * *", // ==> snapShotPolicy.schedule[0] - "SnapShotSchedule:MinutelySnapShotSchedule.Keep=59", - "SnapShotSchedule:HourlySnapShotSchedule.CronTab=0 * * * *", // ==> snapShotPolicy.schedule[1] - "SnapShotSchedule:HourlySnapShotSchedule.Keep=23", - "SnapShotSchedule:DailySnapShotSchedule.CronTab=0 0 * * *", // ==> snapShotPolicy.schedule[2] - "SnapShotSchedule:DailySnapShotSchedule.Keep=6", - "SnapShotSchedule:WeeklySnapShotSchedule.CronTab=0 0 * * 0", // ==> snapShotPolicy.schedule[3] - "SnapShotSchedule:WeeklySnapShotSchedule.Keep=8", - "SnapShotSchedule:MonthlySnapShotSchedule.CronTab=0 0 1 * *", // ==> snapShotPolicy.schedule[4] - "SnapShotSchedule:MonthlySnapShotSchedule.Keep=11", - "SnapShotSchedule:YearlySnapShotSchedule.CronTab=0 0 1 1 *", // ==> snapShotPolicy.schedule[5] - "SnapShotSchedule:YearlySnapShotSchedule.Keep=4", - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=MinutelySnapShotSchedule,HourlySnapShotSchedule,DailySnapShotSchedule,WeeklySnapShotSchedule,MonthlySnapShotSchedule,YearlySnapShotSchedule", - "SnapShotPolicy:CommonSnapShotPolicy.TimeZone=America/Los_Angeles", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("loadSnapShotPolicy() failed: %v", err) - } - - matchingTime = time.Date(2017, time.January, 1, 0, 0, 0, 0, volume.snapShotPolicy.location) // A Sunday - - mismatchingTime = time.Date(2017, time.January, 1, 0, 0, 1, 0, volume.snapShotPolicy.location) // +1 second - matches = volume.snapShotPolicy.schedule[0].compare(matchingTime) - if !matches { - t.Fatalf("snapShotPolicy.schedule[0].compare(matchingTime) should have returned true") - } - matches = volume.snapShotPolicy.schedule[0].compare(mismatchingTime) - if matches { - t.Fatalf("snapShotPolicy.schedule[0].compare(mismatchingTime) should have returned false") - } - - mismatchingTime = time.Date(2017, time.January, 1, 0, 1, 0, 0, volume.snapShotPolicy.location) // +1 minute - matches = volume.snapShotPolicy.schedule[1].compare(matchingTime) - if !matches { - t.Fatalf("snapShotPolicy.schedule[1].compare(matchingTime) should have returned true") - } - matches = volume.snapShotPolicy.schedule[1].compare(mismatchingTime) - if matches { - t.Fatalf("snapShotPolicy.schedule[1].compare(mismatchingTime) should have returned false") - } - - mismatchingTime = time.Date(2017, time.January, 1, 1, 0, 0, 0, volume.snapShotPolicy.location) // +1 hour - matches = volume.snapShotPolicy.schedule[2].compare(matchingTime) - if !matches { - t.Fatalf("snapShotPolicy.schedule[2].compare(matchingTime) should have returned true") - } - matches = volume.snapShotPolicy.schedule[2].compare(mismatchingTime) - if matches { - t.Fatalf("snapShotPolicy.schedule[2].compare(mismatchingTime) should have returned false") - } - - mismatchingTime = time.Date(2017, time.January, 2, 0, 0, 0, 0, volume.snapShotPolicy.location) // +1 day - matches = volume.snapShotPolicy.schedule[3].compare(matchingTime) - if !matches { - t.Fatalf("snapShotPolicy.schedule[3].compare(matchingTime) should have returned true") - } - matches = volume.snapShotPolicy.schedule[3].compare(mismatchingTime) - if matches { - t.Fatalf("snapShotPolicy.schedule[3].compare(mismatchingTime) should have returned false") - } - - mismatchingTime = time.Date(2017, time.January, 2, 0, 0, 0, 0, volume.snapShotPolicy.location) // A Monday - matches = volume.snapShotPolicy.schedule[4].compare(matchingTime) - if !matches { - t.Fatalf("snapShotPolicy.schedule[4].compare(matchingTime) should have returned true") - } - matches = volume.snapShotPolicy.schedule[4].compare(mismatchingTime) - if matches { - t.Fatalf("snapShotPolicy.schedule[4].compare(mismatchingTime) should have returned false") - } - - mismatchingTime = time.Date(2017, time.February, 1, 0, 0, 0, 0, volume.snapShotPolicy.location) // +1 month - matches = volume.snapShotPolicy.schedule[5].compare(matchingTime) - if !matches { - t.Fatalf("snapShotPolicy.schedule[5].compare(matchingTime) should have returned true") - } - matches = volume.snapShotPolicy.schedule[5].compare(mismatchingTime) - if matches { - t.Fatalf("snapShotPolicy.schedule[5].compare(mismatchingTime) should have returned false") - } -} - -func TestSnapShotScheduleNext(t *testing.T) { - var ( - err error - nextTime time.Time - testConfMap conf.ConfMap - testConfMapStrings []string - timeNow time.Time - volume *volumeStruct - ) - - testConfMapStrings = []string{ - "SnapShotSchedule:MinutelySnapShotSchedule.CronTab=* * * * *", // ==> snapShotPolicy.schedule[0] - "SnapShotSchedule:MinutelySnapShotSchedule.Keep=59", - "SnapShotSchedule:HourlySnapShotSchedule.CronTab=0 * * * *", // ==> snapShotPolicy.schedule[1] - "SnapShotSchedule:HourlySnapShotSchedule.Keep=23", - "SnapShotSchedule:DailySnapShotSchedule.CronTab=0 0 * * *", // ==> snapShotPolicy.schedule[2] - "SnapShotSchedule:DailySnapShotSchedule.Keep=6", - "SnapShotSchedule:WeeklySnapShotSchedule.CronTab=0 0 * * 0", // ==> snapShotPolicy.schedule[3] - "SnapShotSchedule:WeeklySnapShotSchedule.Keep=8", - "SnapShotSchedule:MonthlySnapShotSchedule.CronTab=0 0 1 * *", // ==> snapShotPolicy.schedule[4] - "SnapShotSchedule:MonthlySnapShotSchedule.Keep=11", - "SnapShotSchedule:YearlySnapShotSchedule.CronTab=0 0 1 1 *", // ==> snapShotPolicy.schedule[5] - "SnapShotSchedule:YearlySnapShotSchedule.Keep=4", - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=MinutelySnapShotSchedule,HourlySnapShotSchedule,DailySnapShotSchedule,WeeklySnapShotSchedule,MonthlySnapShotSchedule,YearlySnapShotSchedule", - "SnapShotPolicy:CommonSnapShotPolicy.TimeZone=America/Los_Angeles", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("loadSnapShotPolicy() failed: %v", err) - } - - timeNow = time.Date(2017, time.January, 1, 0, 0, 0, 0, volume.snapShotPolicy.location) // A Sunday - - nextTime = volume.snapShotPolicy.schedule[0].next(timeNow) - if !nextTime.Equal(time.Date(2017, time.January, 1, 0, 1, 0, 0, volume.snapShotPolicy.location)) { - t.Fatalf("snapShotPolicy.schedule[0].next(timeNow) returned unexpected time: %v", nextTime) - } - - nextTime = volume.snapShotPolicy.schedule[1].next(timeNow) - if !nextTime.Equal(time.Date(2017, time.January, 1, 1, 0, 0, 0, volume.snapShotPolicy.location)) { - t.Fatalf("snapShotPolicy.schedule[1].next(timeNow) returned unexpected time: %v", nextTime) - } - - nextTime = volume.snapShotPolicy.schedule[2].next(timeNow) - if !nextTime.Equal(time.Date(2017, time.January, 2, 0, 0, 0, 0, volume.snapShotPolicy.location)) { - t.Fatalf("snapShotPolicy.schedule[2].next(timeNow) returned unexpected time: %v", nextTime) - } - - nextTime = volume.snapShotPolicy.schedule[3].next(timeNow) - if !nextTime.Equal(time.Date(2017, time.January, 8, 0, 0, 0, 0, volume.snapShotPolicy.location)) { - t.Fatalf("snapShotPolicy.schedule[3].next(timeNow) returned unexpected time: %v", nextTime) - } - - nextTime = volume.snapShotPolicy.schedule[4].next(timeNow) - if !nextTime.Equal(time.Date(2017, time.February, 1, 0, 0, 0, 0, volume.snapShotPolicy.location)) { - t.Fatalf("snapShotPolicy.schedule[4].next(timeNow) returned unexpected time: %v", nextTime) - } - - nextTime = volume.snapShotPolicy.schedule[5].next(timeNow) - if !nextTime.Equal(time.Date(2018, time.January, 1, 0, 0, 0, 0, volume.snapShotPolicy.location)) { - t.Fatalf("snapShotPolicy.schedule[5].next(timeNow) returned unexpected time: %v", nextTime) - } -} - -func TestSnapShotPolicyNext(t *testing.T) { - var ( - err error - testConfMap conf.ConfMap - testConfMapStrings []string - timeMidnight time.Time - timeMidnightNext time.Time - timeOhThirtyAM time.Time - timeOneFortyFiveAM time.Time - timeOneFortyFiveAMNext time.Time - timeTwoAM time.Time - volume *volumeStruct - ) - - testConfMapStrings = []string{ - "SnapShotSchedule:HalfPastTheHourSnapShotSchedule.CronTab=30 * * * *", // ==> snapShotPolicy.schedule[0] - "SnapShotSchedule:HalfPastTheHourSnapShotSchedule.Keep=99", - "SnapShotSchedule:TwoAMSnapShotSchedule.CronTab=0 2 * * *", // ==> snapShotPolicy.schedule[1] - "SnapShotSchedule:TwoAMSnapShotSchedule.Keep=99", - "SnapShotPolicy:CommonSnapShotPolicy.ScheduleList=HalfPastTheHourSnapShotSchedule,TwoAMSnapShotSchedule", - "SnapShotPolicy:CommonSnapShotPolicy.TimeZone=America/Los_Angeles", - "Volume:TestVolume.SnapShotPolicy=CommonSnapShotPolicy", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - volume = &volumeStruct{volumeName: "TestVolume", snapShotPolicy: nil} - - err = volume.loadSnapShotPolicy(testConfMap) - if nil != err { - t.Fatalf("loadSnapShotPolicy() failed: %v", err) - } - - timeMidnight = time.Date(2017, time.January, 1, 0, 0, 0, 0, volume.snapShotPolicy.location) - timeOhThirtyAM = time.Date(2017, time.January, 1, 0, 30, 0, 0, volume.snapShotPolicy.location) - timeOneFortyFiveAM = time.Date(2017, time.January, 1, 1, 45, 0, 0, volume.snapShotPolicy.location) - timeTwoAM = time.Date(2017, time.January, 1, 2, 0, 0, 0, volume.snapShotPolicy.location) - - timeMidnightNext = volume.snapShotPolicy.schedule[0].next(timeMidnight) - timeOneFortyFiveAMNext = volume.snapShotPolicy.schedule[1].next(timeOneFortyFiveAM) - - if timeMidnightNext != timeOhThirtyAM { - t.Fatalf("snapShotPolicy.schedule[0].next(timeMidnight) returned unexpected time: %v", timeMidnightNext) - } - if timeOneFortyFiveAMNext != timeTwoAM { - t.Fatalf("snapShotPolicy.schedule[1].next(timeOneFortyFiveAM) returned unexpected time: %v", timeOneFortyFiveAMNext) - } - - timeMidnightNext = volume.snapShotPolicy.next(timeMidnight) - timeOneFortyFiveAMNext = volume.snapShotPolicy.next(timeOneFortyFiveAM) - - if timeMidnightNext != timeOhThirtyAM { - t.Fatalf("snapShotPolicy.next(timeMidnight) returned unexpected time: %v", timeMidnightNext) - } - if timeOneFortyFiveAMNext != timeTwoAM { - t.Fatalf("snapShotPolicy.next(timeOneFortyFiveAM) returned unexpected time: %v", timeOneFortyFiveAMNext) - } -} diff --git a/inode/dir.go b/inode/dir.go deleted file mode 100644 index f0d59a4b..00000000 --- a/inode/dir.go +++ /dev/null @@ -1,1429 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - "time" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/utils" -) - -func (vS *volumeStruct) createRootOrSubDir(filePerm InodeMode, userID InodeUserID, groupID InodeGroupID, isRootDir bool) (dirInodeNumber InodeNumber, err error) { - // Create file mode out of file permissions plus inode type - fileMode, err := determineMode(filePerm, DirType) - if nil != err { - return - } - - var dirInode *inMemoryInodeStruct - - if isRootDir { - dirInode = vS.makeInMemoryInodeWithThisInodeNumber(DirType, fileMode, userID, groupID, RootDirInodeNumber, true) - dirInodeNumber = RootDirInodeNumber - } else { - dirInode, err = vS.makeInMemoryInode(DirType, fileMode, userID, groupID) - if nil != err { - return - } - dirInodeNumber = dirInode.InodeNumber - } - - dirInode.dirty = true - - // sorted map from directory entry name (a string) to InodeNumber - - dirMapping := - sortedmap.NewBPlusTree( - vS.maxEntriesPerDirNode, - sortedmap.CompareString, - &dirInodeCallbacks{treeNodeLoadable{inode: dirInode}}, - globals.dirEntryCache) - - ok, err := dirMapping.Put(".", dirInode.InodeNumber) - if (nil != err) || (!ok) { - panic(err) - } - - if isRootDir { - ok, err = dirMapping.Put("..", dirInode.InodeNumber) - if (nil != err) || (!ok) { - panic(err) - } - - dirInode.LinkCount = 2 - } else { - dirInode.LinkCount = 1 - } - - dirInode.payload = dirMapping - - if isRootDir { - // If creating RootDir, since this must be atomic, caller already holds vS.Mutex - ok, err = vS.inodeCacheInsertWhileLocked(dirInode) - } else { - ok, err = vS.inodeCacheInsert(dirInode) - } - if nil != err { - return - } - if !ok { - err = fmt.Errorf("inodeCacheInsert(dirInode) failed") - return - } - - // If creating RootDir, force an immediate flush to ensure it is atomically created as well - if isRootDir { - err = vS.flushInode(dirInode) - if nil != err { - logger.ErrorfWithError(err, "createRootOrSubDir() call to flushInode() failed") - return - } - } - - err = nil - return -} - -func (vS *volumeStruct) CreateDir(filePerm InodeMode, userID InodeUserID, groupID InodeGroupID) (dirInodeNumber InodeNumber, err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - stats.IncrementOperations(&stats.DirCreateOps) - - dirInodeNumber, err = vS.createRootOrSubDir(filePerm, userID, groupID, false) - - if err == nil { - stats.IncrementOperations(&stats.DirCreateSuccessOps) - } - - return -} - -func linkInMemory(dirInode *inMemoryInodeStruct, targetInode *inMemoryInodeStruct, basename string) error { - dirInode.dirty = true - targetInode.dirty = true - - dirMapping := dirInode.payload.(sortedmap.BPlusTree) - - ok, err := dirMapping.Put(basename, targetInode.InodeNumber) - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("%s: failed to create link '%v' to inode %v in directory inode %v: entry exists", - utils.GetFnName(), basename, targetInode.InodeNumber, dirInode.InodeNumber) - return blunder.AddError(err, blunder.FileExistsError) - } - - updateTime := time.Now() - - targetInode.LinkCount++ - targetInode.AttrChangeTime = updateTime - - if targetInode.InodeType == DirType && targetInode.InodeNumber != RootDirInodeNumber { - subdirMapping := targetInode.payload.(sortedmap.BPlusTree) - subdirMapping.Put("..", dirInode.InodeNumber) - dirInode.LinkCount++ - } - - dirInode.AttrChangeTime = updateTime - dirInode.ModificationTime = updateTime - - return nil -} - -// Insert-only version of linkInMemory() -func linkInMemoryInsertOnly(dirInode *inMemoryInodeStruct, basename string, targetInodeNumber InodeNumber) (err error) { - dirInode.dirty = true - - dirMapping := dirInode.payload.(sortedmap.BPlusTree) - - ok, err := dirMapping.Put(basename, targetInodeNumber) - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("%s: failed to create link '%v' to inode %v in directory inode %v: entry exists", - utils.GetFnName(), basename, targetInodeNumber, dirInode.InodeNumber) - return blunder.AddError(err, blunder.FileExistsError) - } - - updateTime := time.Now() - - dirInode.AttrChangeTime = updateTime - dirInode.ModificationTime = updateTime - - return nil -} - -// This is used by the link(2), create(2), and mkdir(2) operations -// (mountstruct.Link(), mountstruct.Create(), and mountstruct.Mkdir()) -func (vS *volumeStruct) Link(dirInodeNumber InodeNumber, basename string, targetInodeNumber InodeNumber, insertOnly bool) (err error) { - var ( - dirInode *inMemoryInodeStruct - flushInodeList []*inMemoryInodeStruct - ok bool - snapShotIDType headhunter.SnapShotIDType - targetInode *inMemoryInodeStruct - ) - - err = enforceRWMode(false) - if nil != err { - return - } - - if (RootDirInodeNumber == dirInodeNumber) && (SnapShotDirName == basename) { - err = blunder.NewError(blunder.InvalidArgError, "Link() to /%v not allowed", SnapShotDirName) - return - } - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = blunder.NewError(blunder.InvalidArgError, "Link() on non-LiveView dirInodeNumber not allowed") - return - } - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(targetInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = blunder.NewError(blunder.InvalidArgError, "Link() on non-LiveView targetInodeNumber not allowed") - return - } - - stats.IncrementOperations(&stats.DirLinkOps) - - dirInode, err = vS.fetchInodeType(dirInodeNumber, DirType) - if err != nil { - logger.ErrorfWithError(err, "dirInode error") - return err - } - - if insertOnly { - err = linkInMemoryInsertOnly(dirInode, basename, targetInodeNumber) - if err != nil { - return err - } - - flushInodeList = []*inMemoryInodeStruct{dirInode} - } else { - targetInode, ok, err = vS.fetchInode(targetInodeNumber) - if err != nil { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when re-read from disk) - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: targetInode fetch error", utils.GetFnName()) - return err - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: Link failing request to link to inode %d volume '%s' because it is unallocated", - utils.GetFnName(), targetInode.InodeNumber, vS.volumeName) - err = blunder.AddError(err, blunder.NotFoundError) - logger.ErrorWithError(err) - return err - } - - err = linkInMemory(dirInode, targetInode, basename) - if err != nil { - return err - } - - flushInodeList = []*inMemoryInodeStruct{dirInode, targetInode} - } - - // REVIEW TODO: We think we need to do something more than just return an error here :-) - - err = vS.flushInodes(flushInodeList) - if err != nil { - logger.ErrorWithError(err) - return err - } - - stats.IncrementOperations(&stats.DirLinkSuccessOps) - return nil -} - -// Manipulate a directory to remove an an entry. Like Unlink(), but without any inode loading or flushing. -func unlinkInMemory(dirInode *inMemoryInodeStruct, untargetInode *inMemoryInodeStruct, basename string) (toDestroyInodeNumber InodeNumber) { - var ( - dirMapping sortedmap.BPlusTree - err error - ok bool - updateTime time.Time - ) - - dirMapping = dirInode.payload.(sortedmap.BPlusTree) - - dirInode.dirty = true - untargetInode.dirty = true - - ok, err = dirMapping.DeleteByKey(basename) - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("Unlink(): dirInode DeleteByKey of \"%v\" should have returned ok == true", basename) - panic(err) - } - - untargetInode.LinkCount-- - - if DirType == untargetInode.InodeType { - untargetDirMapping := untargetInode.payload.(sortedmap.BPlusTree) - - ok, err = untargetDirMapping.DeleteByKey("..") - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("Unlink(): untargetInode DeleteByKey of \"..\" should have returned ok == true") - panic(err) - } - - dirInode.LinkCount-- - - if 1 == untargetInode.LinkCount { - toDestroyInodeNumber = untargetInode.InodeNumber - } else { - toDestroyInodeNumber = InodeNumber(0) - } - } else { - if 0 == untargetInode.LinkCount { - toDestroyInodeNumber = untargetInode.InodeNumber - } else { - toDestroyInodeNumber = InodeNumber(0) - } - } - - updateTime = time.Now() - - dirInode.AttrChangeTime = updateTime - dirInode.ModificationTime = updateTime - - untargetInode.AttrChangeTime = updateTime - - return -} - -// Remove-only version of unlinkInMemory() -func unlinkInMemoryRemoveOnly(dirInode *inMemoryInodeStruct, basename string) { - var ( - dirMapping sortedmap.BPlusTree - err error - ok bool - updateTime time.Time - ) - - dirMapping = dirInode.payload.(sortedmap.BPlusTree) - - dirInode.dirty = true - - ok, err = dirMapping.DeleteByKey(basename) - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("Unlink(): dirInode DeleteByKey of \"%v\" should have returned ok == true", basename) - panic(err) - } - - updateTime = time.Now() - - dirInode.AttrChangeTime = updateTime - dirInode.ModificationTime = updateTime -} - -func (vS *volumeStruct) Unlink(dirInodeNumber InodeNumber, basename string, removeOnly bool) (toDestroyInodeNumber InodeNumber, err error) { - var ( - dirInode *inMemoryInodeStruct - flushInodeList []*inMemoryInodeStruct - ok bool - snapShotIDType headhunter.SnapShotIDType - untargetInode *inMemoryInodeStruct - untargetInodeNumber InodeNumber - ) - - err = enforceRWMode(false) - if nil != err { - return - } - - if (RootDirInodeNumber == dirInodeNumber) && (SnapShotDirName == basename) { - err = blunder.NewError(blunder.InvalidArgError, "Unlink() of /%v not allowed", SnapShotDirName) - return - } - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = blunder.NewError(blunder.InvalidArgError, "Unlink() on non-LiveView dirInodeNumber not allowed") - return - } - - stats.IncrementOperations(&stats.DirUnlinkOps) - - dirInode, err = vS.fetchInodeType(dirInodeNumber, DirType) - if nil != err { - return - } - - untargetInodeNumber, err = vS.lookupByDirInodeNumber(dirInodeNumber, basename) - if nil != err { - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - if removeOnly { - unlinkInMemoryRemoveOnly(dirInode, basename) - - toDestroyInodeNumber = InodeNumber(0) - - flushInodeList = []*inMemoryInodeStruct{dirInode} - } else { - untargetInode, ok, err = vS.fetchInode(untargetInodeNumber) - if nil != err { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when re-read from disk) - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of target inode failed", utils.GetFnName()) - return - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request to Unlink inode %d volume '%s' because it is unallocated", - utils.GetFnName(), untargetInode.InodeNumber, vS.volumeName) - err = blunder.AddError(err, blunder.NotFoundError) - logger.ErrorWithError(err) - return - } - - // Pre-flush untargetInode so that no time-based (implicit) flushes will occur during this transaction - err = vS.flushInode(untargetInode) - if err != nil { - logger.ErrorfWithError(err, "Unlink(): untargetInode flush error") - panic(err) - } - - toDestroyInodeNumber = unlinkInMemory(dirInode, untargetInode, basename) - - flushInodeList = []*inMemoryInodeStruct{dirInode, untargetInode} - } - - err = vS.flushInodes(flushInodeList) - if err != nil { - logger.ErrorWithError(err) - return - } - - stats.IncrementOperations(&stats.DirUnlinkSuccessOps) - return -} - -func (vS *volumeStruct) Move(srcDirInodeNumber InodeNumber, srcBasename string, dstDirInodeNumber InodeNumber, dstBasename string) (toDestroyInodeNumber InodeNumber, err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - if (RootDirInodeNumber == srcDirInodeNumber) && (SnapShotDirName == srcBasename) { - err = blunder.NewError(blunder.InvalidArgError, "Move() from /%v not allowed", SnapShotDirName) - return - } - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(srcDirInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = blunder.NewError(blunder.InvalidArgError, "Move() on non-LiveView srcDirInodeNumber not allowed") - return - } - if (RootDirInodeNumber == dstDirInodeNumber) && (SnapShotDirName == dstBasename) { - err = blunder.NewError(blunder.InvalidArgError, "Move() into /%v not allowed", SnapShotDirName) - return - } - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dstDirInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = blunder.NewError(blunder.InvalidArgError, "Move() on non-LiveView dstDirInodeNumber not allowed") - return - } - - stats.IncrementOperations(&stats.DirRenameOps) - - srcDirInode, err := vS.fetchInodeType(srcDirInodeNumber, DirType) - if nil != err { - logger.ErrorfWithError(err, "Move(): srcDirInode fetch error") - panic(err) - } - srcDirMapping := srcDirInode.payload.(sortedmap.BPlusTree) - - var dstDirInode *inMemoryInodeStruct - var dstDirMapping sortedmap.BPlusTree - if srcDirInodeNumber == dstDirInodeNumber { - if srcBasename == dstBasename { - err = fmt.Errorf("%v: Source & Target of Move() cannot be identical: %v/%v", utils.GetFnName(), srcDirInodeNumber, srcBasename) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.FileExistsError) - return - } - dstDirInode = srcDirInode - dstDirMapping = srcDirMapping - } else { - dstDirInode, err = vS.fetchInodeType(dstDirInodeNumber, DirType) - if nil != err { - logger.ErrorfWithError(err, "Move(): dstDirInode fetch error") - panic(err) - } - dstDirMapping = dstDirInode.payload.(sortedmap.BPlusTree) - } - - srcInodeNumberAsValue, ok, err := srcDirMapping.GetByKey(srcBasename) - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("%v: unable to find basename %v in dirInode %v", utils.GetFnName(), srcBasename, srcDirInodeNumber) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - srcInodeNumber := srcInodeNumberAsValue.(InodeNumber) - - srcInode, ok, err := vS.fetchInode(srcInodeNumber) - if nil != err { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when re-read from disk) - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of src inode failed", utils.GetFnName()) - return - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request because src inode %d volume '%s' is unallocated", - utils.GetFnName(), srcInode.InodeNumber, vS.volumeName) - err = blunder.AddError(err, blunder.NotDirError) - logger.ErrorWithError(err) - return - } - - var dstInodeNumber InodeNumber - var dstInode *inMemoryInodeStruct - dstInodeNumberAsValue, ok, err := dstDirMapping.GetByKey(dstBasename) - if nil != err { - // this indicates disk corruption or software bug - logger.ErrorfWithError(err, "%s: dstDirInode GetByKey(%s) error inode %d volume '%s'", - utils.GetFnName(), dstBasename, dstDirInode.InodeNumber, vS.volumeName) - panic(err) - } - if ok { - dstInodeNumber = dstInodeNumberAsValue.(InodeNumber) - - dstInode, ok, err = vS.fetchInode(dstInodeNumber) - if nil != err { - // this indicates disk corruption or software bug - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: dstInode fetch error", utils.GetFnName()) - panic(err) - } - if !ok { - // disk corruption or software bug - err = fmt.Errorf("%s: dstInode inode %d volume '%s' is unallocated", - utils.GetFnName(), dstInode.InodeNumber, vS.volumeName) - err = blunder.AddError(err, blunder.NotFoundError) - logger.ErrorWithError(err) - panic(err) - } - } else { - dstInodeNumber = InodeNumber(0) - dstInode = nil - } - - // I believe this is allowed so long at the dstInode is empty --craig - if (nil != dstInode) && (DirType == dstInode.InodeType) { - err = fmt.Errorf("%v: Target of Move() is an existing directory: %v/%v", utils.GetFnName(), dstDirInodeNumber, dstBasename) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.FileExistsError) - return - } - - // All set to proceed - - if FileType == srcInode.InodeType { - // Pre-flush srcInode so that no time-based (implicit) flushes will occur during this transaction - err = vS.flushInode(srcInode) - if err != nil { - logger.ErrorfWithError(err, "Move(): srcInode flush error") - panic(err) - } - } - if (nil != dstInode) && (FileType == dstInode.InodeType) { - // Pre-flush dstInode so that no time-based (implicit) flushes will occur during this transaction - err = vS.flushInode(dstInode) - if err != nil { - logger.ErrorfWithError(err, "Move(): dstInode flush error") - panic(err) - } - } - - updateTime := time.Now() - - inodes := make([]*inMemoryInodeStruct, 0, 4) - - srcDirInode.dirty = true - srcDirInode.AttrChangeTime = updateTime - srcDirInode.ModificationTime = updateTime - inodes = append(inodes, srcDirInode) - - if srcDirInodeNumber != dstDirInodeNumber { - dstDirInode.dirty = true - dstDirInode.AttrChangeTime = updateTime - dstDirInode.ModificationTime = updateTime - inodes = append(inodes, dstDirInode) - - if DirType == srcInode.InodeType { - srcDirInode.LinkCount-- - dstDirInode.LinkCount++ - - srcInodeAsDirMapping := srcInode.payload.(sortedmap.BPlusTree) - ok, err = srcInodeAsDirMapping.PatchByKey("..", dstDirInodeNumber) - if nil != err { - logger.ErrorfWithError(err, "Move(): srcInode PatchByKey error") - panic(err) - } - if !ok { - err = fmt.Errorf("Should have found \"..\" entry") - logger.ErrorfWithError(err, "Move(): srcInode PatchByKey error") - panic(err) - } - } - } - - srcInode.dirty = true - srcInode.AttrChangeTime = updateTime - inodes = append(inodes, srcInode) - - ok, err = srcDirMapping.DeleteByKey(srcBasename) - if nil != err { - logger.ErrorfWithError(err, "Move(): srcDirInode DeleteByKey error") - panic(err) - } - if !ok { - err = fmt.Errorf("Should have found \"%v\" entry", srcBasename) - logger.ErrorfWithError(err, "Move(): srcDirInode DeleteByKey error") - panic(err) - } - - if nil == dstInode { - ok, err = dstDirMapping.Put(dstBasename, srcInodeNumber) - if nil != err { - logger.ErrorfWithError(err, "Move(): dstDirInode Put error") - panic(err) - } - if !ok { - err = fmt.Errorf("Should have been able to PUT \"%v\" entry", dstBasename) - logger.ErrorfWithError(err, "Move(): dstDirInode Put error") - panic(err) - } - } else { - dstInode.dirty = true - dstInode.AttrChangeTime = updateTime - inodes = append(inodes, dstInode) - - dstInode.LinkCount-- - - ok, err = dstDirMapping.PatchByKey(dstBasename, srcInodeNumber) - if nil != err { - logger.ErrorfWithError(err, "Move(): dstDirInode PatchByKey error") - panic(err) - } - if !ok { - err = fmt.Errorf("Should have been able to PatchByKey \"%v\" entry", dstBasename) - logger.ErrorfWithError(err, "Move(): dstDirInode PatchByKey error") - panic(err) - } - } - - // Flush the multi-inode transaction - - err = vS.flushInodes(inodes) - if err != nil { - logger.ErrorfWithError(err, "flushInodes(%v) error", inodes) - panic(err) - } - - // Finally, if we decremented dstInode.LinkCount to zero, indicate dstInode should be destroyed as well - - if (nil != dstInode) && (0 == dstInode.LinkCount) { - toDestroyInodeNumber = dstInode.InodeNumber - } else { - toDestroyInodeNumber = InodeNumber(0) - } - - stats.IncrementOperations(&stats.DirRenameSuccessOps) - - err = nil - return -} - -func (vS *volumeStruct) lookupByDirInode(dirInode *inMemoryInodeStruct, basename string) (targetInodeNumber InodeNumber, err error) { - var ( - dirInodeSnapShotID uint64 - dirMapping sortedmap.BPlusTree - ok bool - value sortedmap.Value - ) - - _, dirInodeSnapShotID, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirInode.InodeNumber)) - - dirMapping = dirInode.payload.(sortedmap.BPlusTree) - value, ok, err = dirMapping.GetByKey(basename) - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("unable to find basename %v in dirInode @ %p", basename, dirInode) - // There are cases where failing to find an inode is not an error. - // Not logging any errors here; let the caller decide if this is log-worthy - err = blunder.AddError(err, blunder.NotFoundError) - return - } - targetInodeNumber, ok = value.(InodeNumber) - if !ok { - err = fmt.Errorf("dirMapping for basename %v in dirInode @ %p not an InodeNumber", basename, dirInode) - panic(err) - } - targetInodeNumber = InodeNumber(vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(dirInodeSnapShotID, uint64(targetInodeNumber))) - - return -} - -func (vS *volumeStruct) lookupByDirInodeNumber(dirInodeNumber InodeNumber, basename string) (targetInodeNumber InodeNumber, err error) { - var ( - dirInode *inMemoryInodeStruct - dirInodeNonce uint64 - dirInodeSnapShotID uint64 - dirInodeSnapShotIDType headhunter.SnapShotIDType - dirMapping sortedmap.BPlusTree - ok bool - value sortedmap.Value - snapShot headhunter.SnapShotStruct - ) - - dirInodeSnapShotIDType, dirInodeSnapShotID, dirInodeNonce = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirInodeNumber)) - - switch dirInodeSnapShotIDType { - case headhunter.SnapShotIDTypeLive: - if (uint64(RootDirInodeNumber) == dirInodeNonce) && - (SnapShotDirName == basename) && - (0 < vS.headhunterVolumeHandle.SnapShotCount()) { - // Lookup should only succeed there are any active SnapShot's - - targetInodeNumber = InodeNumber(vS.headhunterVolumeHandle.SnapShotTypeDotSnapShotAndNonceEncode(dirInodeNonce)) - err = nil - - return - } - // Let normal processing perform the lookup below (SnapShotID == 0, so InodeNumber "adornment" is a no-op) - case headhunter.SnapShotIDTypeSnapShot: - // Let normal processing perform the lookup below but "adorn" resultant InodeNumber's with the dirSnapShotID - case headhunter.SnapShotIDTypeDotSnapShot: - // Currently, this is only supported in RootDirInodeNumber - - if uint64(RootDirInodeNumber) != dirInodeNonce { - err = blunder.AddError(fmt.Errorf("%v other than in '/' not supported", SnapShotDirName), blunder.NotFoundError) - return - } - - switch basename { - case ".": - targetInodeNumber = dirInodeNumber - err = nil - case "..": - targetInodeNumber = RootDirInodeNumber - err = nil - default: - // See if basename is the name for an active SnapShot - - snapShot, ok = vS.headhunterVolumeHandle.SnapShotLookupByName(basename) - - if ok { - targetInodeNumber = InodeNumber(vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(snapShot.ID, dirInodeNonce)) - err = nil - } else { - err = blunder.AddError(fmt.Errorf("/%v/%v SnapShot not found", SnapShotDirName, basename), blunder.NotFoundError) - } - } - - return - } - - dirInode, err = vS.fetchInodeType(dirInodeNumber, DirType) - if nil != err { - logger.ErrorWithError(err) - return - } - - dirMapping = dirInode.payload.(sortedmap.BPlusTree) - value, ok, err = dirMapping.GetByKey(basename) - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("unable to find basename %v in dirInode %v", basename, dirInodeNumber) - // There are cases where failing to find an inode is not an error. - // Not logging any errors here; let the caller decide if this is log-worthy - err = blunder.AddError(err, blunder.NotFoundError) - return - } - targetInodeNumber, ok = value.(InodeNumber) - if !ok { - err = fmt.Errorf("dirMapping for basename %v in dirInode %v not an InodeNumber", basename, dirInodeNumber) - panic(err) - } - targetInodeNumber = InodeNumber(vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(dirInodeSnapShotID, uint64(targetInodeNumber))) - - return -} - -func (vS *volumeStruct) Lookup(dirInodeNumber InodeNumber, basename string) (targetInodeNumber InodeNumber, err error) { - stats.IncrementOperations(&stats.DirLookupOps) - - targetInodeNumber, err = vS.lookupByDirInodeNumber(dirInodeNumber, basename) - - return -} - -func (vS *volumeStruct) NumDirEntries(dirInodeNumber InodeNumber) (numEntries uint64, err error) { - var ( - adjustNumEntriesForSnapShotSubDirInRootDirInode bool - dirMapping sortedmap.BPlusTree - dirMappingLen int - inode *inMemoryInodeStruct - snapShotCount uint64 - snapShotIDType headhunter.SnapShotIDType - ) - - if RootDirInodeNumber == dirInodeNumber { - // Account for .. in / if any SnapShot's exist - snapShotCount = vS.headhunterVolumeHandle.SnapShotCount() - adjustNumEntriesForSnapShotSubDirInRootDirInode = (0 != snapShotCount) - } else { - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirInodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - // numEntries == 1 ('.') + 1 ('..') + # SnapShot's - snapShotCount = vS.headhunterVolumeHandle.SnapShotCount() - numEntries = 1 + 1 + snapShotCount - err = nil - return - } - adjustNumEntriesForSnapShotSubDirInRootDirInode = false - } - - inode, err = vS.fetchInodeType(dirInodeNumber, DirType) - if nil != err { - return - } - - dirMapping = inode.payload.(sortedmap.BPlusTree) - - dirMappingLen, err = dirMapping.Len() - if nil != err { - err = blunder.AddError(err, blunder.IOError) - return - } - - if adjustNumEntriesForSnapShotSubDirInRootDirInode { - numEntries = uint64(dirMappingLen) + 1 - } else { - numEntries = uint64(dirMappingLen) - } - - return -} - -// A maxEntries or maxBufSize argument of zero is interpreted to mean "no maximum". -func (vS *volumeStruct) ReadDir(dirInodeNumber InodeNumber, maxEntries uint64, maxBufSize uint64, prevReturned ...interface{}) (dirEntries []DirEntry, moreEntries bool, err error) { - var ( - bufSize uint64 - dirEntryBasename string - dirEntryBasenameAsKey sortedmap.Key - dirEntryInodeNumber InodeNumber - dirEntryInodeNumberAsValue sortedmap.Value - dirIndex int - dirMapping sortedmap.BPlusTree - dirMappingLen int // If snapShotDirToBeInserted, this is 1 + dirMapping.Len() - dotDotInodeNumberReplacement InodeNumber // If == 0, do not replace ..'s InodeNumber - foundPrevReturned bool - inode *inMemoryInodeStruct - key sortedmap.Key - nextEntry DirEntry - nonce uint64 - okGetByIndex bool - okKeyAsString bool - okPayloadBPlusTree bool - okPutToSnapShotListSorted bool - okValueAsInodeNumber bool - snapShotDirFound bool - snapShotDirIndex int // If snapShotDirToBeInserted, this is the index where it goes - snapShotDirToBeInserted bool // Only true in / - snapShotID uint64 - snapShotIDType headhunter.SnapShotIDType - snapShotList []headhunter.SnapShotStruct - snapShotListElement headhunter.SnapShotStruct - snapShotListSorted sortedmap.LLRBTree // Will also include '.' & '..' - snapShotListSortedLen int - value sortedmap.Value - ) - - // The following defer'd func() is useful for debugging... so leaving it in here as a comment - /* - defer func() { - logger.Errorf("Executed inode.ReadDir()...") - logger.Errorf(" dirInodeNumber: 0x%016X", dirInodeNumber) - logger.Errorf(" maxEntries: 0x%016X", maxEntries) - logger.Errorf(" maxBufSize: 0x%016X", maxBufSize) - switch len(prevReturned) { - case 0: - // Nothing - case 1: - logger.Errorf(" prevReturned: %v", prevReturned[0]) - default: - logger.Errorf(" len(prevReturned) [%v] should have been 0 or 1", len(prevReturned)) - } - if nil == err { - for dirEntriesIndex, dirEntry := range dirEntries { - logger.Errorf(" dirEntries[%v]:", dirEntriesIndex) - logger.Errorf(" InodeNumber: 0x%016X", dirEntry.InodeNumber) - logger.Errorf(" Basename: %s", dirEntry.Basename) - logger.Errorf(" InodeType: %v", dirEntry.Type) - logger.Errorf(" NextDirLocation: %v", dirEntry.NextDirLocation) - } - } - logger.Errorf(" moreEntries: %v", moreEntries) - logger.Errorf(" err: %v", err) - }() - */ - - stats.IncrementOperations(&stats.DirReaddirOps) - - dirEntries = make([]DirEntry, 0, int(maxEntries)) - moreEntries = false - - snapShotIDType, snapShotID, nonce = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirInodeNumber)) - - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - if uint64(RootDirInodeNumber) != nonce { - err = fmt.Errorf("ReadDir() to %v not in '/' not supported", SnapShotDirName) - err = blunder.AddError(err, blunder.NotSupportedError) - return - } - - snapShotList = vS.headhunterVolumeHandle.SnapShotListByName(false) - - snapShotListSorted = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - okPutToSnapShotListSorted, err = snapShotListSorted.Put(".", dirInodeNumber) - if nil != err { - err = fmt.Errorf("ReadDir() encountered error populating SnapShotListSorted: %v", err) - err = blunder.AddError(err, blunder.IOError) - return - } - if !okPutToSnapShotListSorted { - err = fmt.Errorf("ReadDir() encountered !ok populating SnapShotListSorted") - err = blunder.AddError(err, blunder.IOError) - return - } - - okPutToSnapShotListSorted, err = snapShotListSorted.Put("..", RootDirInodeNumber) - if nil != err { - err = fmt.Errorf("ReadDir() encountered error populating SnapShotListSorted: %v", err) - err = blunder.AddError(err, blunder.IOError) - return - } - if !okPutToSnapShotListSorted { - err = fmt.Errorf("ReadDir() encountered !ok populating SnapShotListSorted") - err = blunder.AddError(err, blunder.IOError) - return - } - - for _, snapShotListElement = range snapShotList { - okPutToSnapShotListSorted, err = snapShotListSorted.Put(snapShotListElement.Name, InodeNumber(vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(snapShotListElement.ID, uint64(RootDirInodeNumber)))) - if nil != err { - err = fmt.Errorf("ReadDir() encountered error populating SnapShotListSorted: %v", err) - err = blunder.AddError(err, blunder.IOError) - return - } - if !okPutToSnapShotListSorted { - err = fmt.Errorf("ReadDir() encountered !ok populating SnapShotListSorted") - err = blunder.AddError(err, blunder.IOError) - return - } - } - - switch len(prevReturned) { - case 0: - dirIndex = int(0) - case 1: - var ( - okAsInodeDirLocation bool - okAsString bool - prevReturnedAsInodeDirLocation InodeDirLocation - prevReturnedAsString string - ) - - prevReturnedAsInodeDirLocation, okAsInodeDirLocation = prevReturned[0].(InodeDirLocation) - prevReturnedAsString, okAsString = prevReturned[0].(string) - - if okAsInodeDirLocation { - if 0 > prevReturnedAsInodeDirLocation { - dirIndex = int(0) - } else { - dirIndex = int(prevReturnedAsInodeDirLocation + 1) - } - } else if okAsString { - dirIndex, foundPrevReturned, err = snapShotListSorted.BisectRight(prevReturnedAsString) - if nil != err { - err = fmt.Errorf("ReadDir() encountered error bisecting SnapShotListSorted: %v", err) - } - if foundPrevReturned { - dirIndex++ - } - } else { - err = fmt.Errorf("ReadDir() accepts only zero or one (InodeDirLocation or string) trailing prevReturned argument") - err = blunder.AddError(err, blunder.NotSupportedError) - return - } - default: - err = fmt.Errorf("ReadDir() accepts only zero or one (InodeDirLocation or string) trailing prevReturned argument") - err = blunder.AddError(err, blunder.NotSupportedError) - return - } - - bufSize = 0 - - snapShotListSortedLen, err = snapShotListSorted.Len() - if nil != err { - err = fmt.Errorf("ReadDir() encountered error accessing SnapShotListSorted: %v", err) - err = blunder.AddError(err, blunder.IOError) - return - } - - for { - if snapShotListSortedLen <= dirIndex { - break - } - - dirEntryBasenameAsKey, dirEntryInodeNumberAsValue, okGetByIndex, err = snapShotListSorted.GetByIndex(dirIndex) - if nil != err { - err = fmt.Errorf("ReadDir() encountered error accessing SnapShotListSorted: %v", err) - err = blunder.AddError(err, blunder.IOError) - return - } - if !okGetByIndex { - err = fmt.Errorf("ReadDir() encountered !ok accessing SnapShotListSorted: %v", err) - err = blunder.AddError(err, blunder.IOError) - return - } - - dirEntryBasename, okKeyAsString = dirEntryBasenameAsKey.(string) - if !okKeyAsString { - err = fmt.Errorf("ReadDir() encountered !ok accessing SnapShotListSorted: %v", err) - err = blunder.AddError(err, blunder.IOError) - return - } - - dirEntryInodeNumber, okValueAsInodeNumber = dirEntryInodeNumberAsValue.(InodeNumber) - if !okValueAsInodeNumber { - err = fmt.Errorf("ReadDir() encountered !ok accessing SnapShotListSorted: %v", err) - err = blunder.AddError(err, blunder.IOError) - return - } - - nextEntry = DirEntry{ - InodeNumber: dirEntryInodeNumber, - Basename: dirEntryBasename, - NextDirLocation: InodeDirLocation(dirIndex) + 1, - } - - if (0 != maxEntries) && (uint64(len(dirEntries)+1) > maxEntries) { - break - } - if (0 != maxBufSize) && ((bufSize + uint64(nextEntry.Size())) > maxBufSize) { - break - } - - dirEntries = append(dirEntries, nextEntry) - bufSize += uint64(nextEntry.Size()) - dirIndex++ - } - - moreEntries = dirIndex < dirMappingLen - - stats.IncrementOperationsEntriesAndBytes(stats.DirRead, uint64(len(dirEntries)), bufSize) - - err = nil - return - } - - // If we reach here, snapShotIDType is one of headhunter.SnapShotIDType{Live|SnapShotIDTypeSnapShot} - - inode, err = vS.fetchInodeType(dirInodeNumber, DirType) - if nil != err { - return - } - - dirMapping, okPayloadBPlusTree = inode.payload.(sortedmap.BPlusTree) - if !okPayloadBPlusTree { - err = fmt.Errorf("ReadDir() found unexpected Inode Payload") - err = blunder.AddError(err, blunder.IOError) - return - } - - dirMappingLen, err = dirMapping.Len() - if nil != err { - err = blunder.AddError(err, blunder.IOError) - return - } - - snapShotDirToBeInserted = false // By default, SnapShotDirName not to be inserted - dotDotInodeNumberReplacement = InodeNumber(0) // By default, do not replace ..'s InodeNumber - - if headhunter.SnapShotIDTypeLive == snapShotIDType { - if RootDirInodeNumber == dirInodeNumber { - if uint64(0) < vS.headhunterVolumeHandle.SnapShotCount() { - // Need to find a spot to insert SnapShotDirName - - snapShotDirIndex, snapShotDirFound, err = dirMapping.BisectRight(SnapShotDirName) - if nil != err { - err = blunder.AddError(err, blunder.IOError) - return - } - if snapShotDirFound { - err = fmt.Errorf("ReadDir() encountered pre-existing /%v dirEntry", SnapShotDirName) - err = blunder.AddError(err, blunder.IOError) - return - } - - snapShotDirToBeInserted = true - dirMappingLen++ - } - } - } else { - if uint64(RootDirInodeNumber) == nonce { - dotDotInodeNumberReplacement = InodeNumber(vS.headhunterVolumeHandle.SnapShotTypeDotSnapShotAndNonceEncode(uint64(RootDirInodeNumber))) - } - } - - switch len(prevReturned) { - case 0: - dirIndex = int(0) - case 1: - var ( - foundDoingBisectRight bool - okAsInodeDirLocation bool - okAsString bool - prevReturnedAsInodeDirLocation InodeDirLocation - prevReturnedAsString string - ) - - prevReturnedAsInodeDirLocation, okAsInodeDirLocation = prevReturned[0].(InodeDirLocation) - prevReturnedAsString, okAsString = prevReturned[0].(string) - - if okAsInodeDirLocation { - if 0 > prevReturnedAsInodeDirLocation { - dirIndex = int(0) - } else { - dirIndex = int(prevReturnedAsInodeDirLocation + 1) - } - } else if okAsString { - if snapShotDirToBeInserted { - if SnapShotDirName == prevReturnedAsString { - dirIndex = snapShotDirIndex + 1 - } else { - dirIndex, foundDoingBisectRight, err = dirMapping.BisectRight(prevReturnedAsString) - if nil != err { - err = blunder.AddError(err, blunder.IOError) - return - } - if dirIndex >= snapShotDirIndex { - dirIndex++ - } - if foundDoingBisectRight { - dirIndex++ - } - } - } else { - dirIndex, foundDoingBisectRight, err = dirMapping.BisectRight(prevReturnedAsString) - if nil != err { - err = blunder.AddError(err, blunder.IOError) - return - } - if foundDoingBisectRight { - dirIndex++ - } - } - } else { - err = fmt.Errorf("ReadDir() accepts only zero or one (InodeDirLocation or string) trailing prevReturned argument") - err = blunder.AddError(err, blunder.NotSupportedError) - return - } - default: - err = fmt.Errorf("ReadDir() accepts only zero or one (InodeDirLocation or string) trailing prevReturned argument") - err = blunder.AddError(err, blunder.NotSupportedError) - return - } - - bufSize = 0 - - for { - if snapShotDirToBeInserted && (dirIndex == snapShotDirIndex) { - dirEntryBasename = SnapShotDirName - dirEntryInodeNumber = InodeNumber(vS.headhunterVolumeHandle.SnapShotTypeDotSnapShotAndNonceEncode(uint64(RootDirInodeNumber))) - } else { - if snapShotDirToBeInserted { - if dirIndex < snapShotDirIndex { - key, value, okGetByIndex, err = dirMapping.GetByIndex(dirIndex) - } else { - key, value, okGetByIndex, err = dirMapping.GetByIndex(dirIndex - 1) - } - } else { - key, value, okGetByIndex, err = dirMapping.GetByIndex(dirIndex) - } - if nil != err { - err = blunder.AddError(err, blunder.IOError) - return - } - if !okGetByIndex { - break - } - - dirEntryBasename, okKeyAsString = key.(string) - if !okKeyAsString { - err = fmt.Errorf("ReadDir() encountered dirEntry with non-string Key") - err = blunder.AddError(err, blunder.IOError) - return - } - - if (InodeNumber(0) != dotDotInodeNumberReplacement) && (".." == dirEntryBasename) { - dirEntryInodeNumber = dotDotInodeNumberReplacement - } else { - dirEntryInodeNumber = InodeNumber(vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(snapShotID, uint64(value.(InodeNumber)))) - } - } - - nextEntry = DirEntry{ - InodeNumber: dirEntryInodeNumber, - Basename: dirEntryBasename, - NextDirLocation: InodeDirLocation(dirIndex) + 1, - } - - if (0 != maxEntries) && (uint64(len(dirEntries)+1) > maxEntries) { - break - } - if (0 != maxBufSize) && ((bufSize + uint64(nextEntry.Size())) > maxBufSize) { - break - } - - dirEntries = append(dirEntries, nextEntry) - bufSize += uint64(nextEntry.Size()) - - dirIndex++ // Consumed one dirEntry either manufactured (/) or from dirMapping - } - - moreEntries = dirIndex < dirMappingLen - - stats.IncrementOperationsEntriesAndBytes(stats.DirRead, uint64(len(dirEntries)), bufSize) - - err = nil - return -} - -func (vS *volumeStruct) AddDirEntry(dirInodeNumber InodeNumber, dirEntryName string, dirEntryInodeNumber InodeNumber, skipDirLinkCountIncrementOnSubDirEntry bool, skipSettingDotDotOnSubDirEntry bool, skipDirEntryLinkCountIncrementOnNonSubDirEntry bool) (err error) { - var ( - dirEntryInode *inMemoryInodeStruct - dirInode *inMemoryInodeStruct - dirMapping sortedmap.BPlusTree - ok bool - snapShotIDType headhunter.SnapShotIDType - ) - - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("AddDirEntry(dirInodeNumber==0x%016X,,,,,) not allowed for snapShotIDType == %v", dirInodeNumber, snapShotIDType) - return - } - if "" == dirEntryName { - err = fmt.Errorf("AddDirEntry(,dirEntryName==\"\",,,,) not allowed") - return - } - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirEntryInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("AddDirEntry(,,dirEntryInodeNumber==0x%016X,,,) not allowed for snapShotIDType == %v", dirEntryInodeNumber, snapShotIDType) - return - } - - dirInode, err = vS.fetchInodeType(dirInodeNumber, DirType) - if nil != err { - err = fmt.Errorf("AddDirEntry(dirInodeNumber==0x%016X,,,,,) failed to fetch dirInode: %v", dirInodeNumber, err) - return - } - dirEntryInode, _, err = vS.fetchInode(dirEntryInodeNumber) - if nil != err { - err = fmt.Errorf("AddDirEntry(,,dirEntryInodeNumber==0x%016X,,,) failed to fetch dirEntryInode: %v", dirEntryInodeNumber, err) - return - } - - dirMapping = dirInode.payload.(sortedmap.BPlusTree) - - _, ok, err = dirMapping.GetByKey(dirEntryName) - if nil != err { - panic(err) - } - if ok { - err = fmt.Errorf("AddDirEntry(,dirEntryName==\"%s\",,,,) collided with pre-existing dirEntryName (case 1)", dirEntryName) - return - } - - ok, err = dirMapping.Put(dirEntryName, dirEntryInodeNumber) - if nil != err { - panic(err) - } - if !ok { - err = fmt.Errorf("AddDirEntry(,dirEntryName==\"%s\",,,,) collided with pre-existing dirEntryName (case 2)", dirEntryName) - return - } - - if !skipDirLinkCountIncrementOnSubDirEntry && (DirType == dirEntryInode.InodeType) { - dirInode.onDiskInodeV1Struct.LinkCount++ - } - - if !skipSettingDotDotOnSubDirEntry && (DirType == dirEntryInode.InodeType) { - dirMapping = dirEntryInode.payload.(sortedmap.BPlusTree) - - ok, err = dirMapping.PatchByKey("..", dirInodeNumber) - if nil != err { - panic(err) - } - if !ok { - _, err = dirMapping.Put("..", dirInodeNumber) - if nil != err { - panic(err) - } - } - } - - if !skipDirEntryLinkCountIncrementOnNonSubDirEntry && (DirType != dirEntryInode.InodeType) { - dirEntryInode.onDiskInodeV1Struct.LinkCount++ - } - - dirInode.dirty = true - dirEntryInode.dirty = true - - err = vS.flushInodes([]*inMemoryInodeStruct{dirInode, dirEntryInode}) - - return -} - -func (vS *volumeStruct) ReplaceDirEntries(parentDirInodeNumber InodeNumber, parentDirEntryBasename string, dirInodeNumber InodeNumber, dirEntryInodeNumbers []InodeNumber) (err error) { - var ( - dirEntryBasename string - dirEntryInodeNumber InodeNumber - dirEntryIndex int - dirEntryInode *inMemoryInodeStruct - dirEntryInodes []*inMemoryInodeStruct - dirInode *inMemoryInodeStruct - dirLinkCount uint64 - dirMapping sortedmap.BPlusTree - ok bool - parentDirEntryInodeNumber InodeNumber - parentDirInode *inMemoryInodeStruct - snapShotIDType headhunter.SnapShotIDType - ) - - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(parentDirInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("ReplaceDirEntries(parentDirInodeNumber==0x%016X,,) not allowed for snapShotIDType == %v", parentDirInodeNumber, snapShotIDType) - return - } - - parentDirInode, err = vS.fetchInodeType(parentDirInodeNumber, DirType) - if nil != err { - err = fmt.Errorf("ReplaceDirEntries() cannot find parentDirInodeNumber==0x%016X: %v", parentDirInodeNumber, err) - return - } - - parentDirEntryInodeNumber, err = vS.lookupByDirInode(parentDirInode, parentDirEntryBasename) - if nil != err { - err = fmt.Errorf("ReplaceDirEntries() lookup in parentDirInodeNumber==0x%016X at basename=\"%s\" failed: %v", parentDirInodeNumber, parentDirEntryBasename, err) - return - } - if parentDirEntryInodeNumber != dirInodeNumber { - err = fmt.Errorf("ReplaceDirEntries() lookup in parentDirInodeNumber==0x%016X at basename=\"%s\" returned unexpected parentDirEntryInodeNumber (0x%016X)... expected dirInodeNumber (0x%016X)", parentDirInodeNumber, parentDirEntryBasename, parentDirEntryInodeNumber, dirInodeNumber) - return - } - - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("ReplaceDirEntries(,dirInodeNumber==0x%016X,) not allowed for snapShotIDType == %v", dirInodeNumber, snapShotIDType) - return - } - - dirInode, err = vS.fetchInodeType(dirInodeNumber, DirType) - if nil != err { - err = fmt.Errorf("ReplaceDirEntries() cannot find dirInodeNumber==0x%016X: %v", dirInodeNumber, err) - return - } - - dirEntryInodes = make([]*inMemoryInodeStruct, len(dirEntryInodeNumbers)) - - for dirEntryIndex, dirEntryInodeNumber = range dirEntryInodeNumbers { - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(dirEntryInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("ReplaceDirEntries(,,dirEntryInodeNumbers[%v]==0x%016X) not allowed for snapShotIDType == %v", dirEntryIndex, dirEntryInodeNumber, snapShotIDType) - return - } - - dirEntryInodes[dirEntryIndex], ok, err = vS.fetchInode(dirEntryInodeNumber) - if (nil != err) || !ok { - err = fmt.Errorf("ReplaceDirEntries() cannot find dirEntryInodeNumbers[%v]==0x%016X: %v", dirEntryIndex, dirInodeNumber, err) - return - } - } - - dirMapping = sortedmap.NewBPlusTree(vS.maxEntriesPerDirNode, sortedmap.CompareString, &dirInodeCallbacks{treeNodeLoadable{inode: dirInode}}, globals.dirEntryCache) - - ok, err = dirMapping.Put(".", dirInodeNumber) - if (nil != err) || !ok { - panic(err) - } - - ok, err = dirMapping.Put("..", parentDirInodeNumber) - if (nil != err) || !ok { - panic(err) - } - - dirLinkCount = 2 - - for _, dirEntryInode = range dirEntryInodes { - dirEntryBasename = fmt.Sprintf("%016X", dirEntryInode.InodeNumber) - - ok, err = dirMapping.Put(dirEntryBasename, dirEntryInode.InodeNumber) - if (nil != err) || !ok { - panic(err) - } - - if DirType == dirEntryInode.InodeType { - dirLinkCount++ - } - } - - dirInode.payload = dirMapping - dirInode.onDiskInodeV1Struct.LinkCount = dirLinkCount - - dirInode.dirty = true - - err = vS.flushInode(dirInode) - - return -} diff --git a/inode/dir_stress_test.go b/inode/dir_stress_test.go deleted file mode 100644 index a92af896..00000000 --- a/inode/dir_stress_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - "sync" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/trackedlock" -) - -const ( - testDirStressBasenamePrefix = "__TestDirStress_" // To be appended with Nonce + "_" + ThreadIndex + "_" + LinkIndex (for this Thread) - - testDirStressDelayBetweenLinkAndUnlink time.Duration = 1 * time.Microsecond - testDirStressDelayBetweenThreadStarts time.Duration = 1 * time.Microsecond - - testDirStressNumIterations uint64 = 3 - testDirStressNumLinksPerIteration uint64 = 3 - testDirStressNumThreads uint64 = 30 - - testDirStressVolumeName = "TestVolume" -) - -type testDirStressGlobalsStruct struct { - trackedlock.Mutex // Used to emulate the exclusive lock used in package fs on RootDirInodeNumber - nonce uint64 - waitGroup sync.WaitGroup - err []error -} - -var testDirStressGlobals = &testDirStressGlobalsStruct{} - -func TestDirStressWhileStarved(t *testing.T) { - testDirStress(t, true) -} - -func TestDirStressWhileNotStarved(t *testing.T) { - testDirStress(t, false) -} - -func testDirStress(t *testing.T, starvationMode bool) { - var ( - err error - headhunterVolumeHandle headhunter.VolumeHandle - threadIndex uint64 - ) - - testSetup(t, starvationMode) - - headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(testDirStressVolumeName) - if nil != err { - t.Fatalf("headhunter.FetchVolumeHandle(\"%s\") failed: %v", testDirStressVolumeName, err) - } - - testDirStressGlobals.nonce = headhunterVolumeHandle.FetchNonce() - - testDirStressGlobals.waitGroup.Add(int(testDirStressNumThreads)) - - testDirStressGlobals.err = make([]error, testDirStressNumThreads) - - for threadIndex = uint64(0); threadIndex < testDirStressNumThreads; threadIndex++ { - time.Sleep(testDirStressDelayBetweenThreadStarts) - - go testDirStressThread(threadIndex) - } - - testDirStressGlobals.waitGroup.Wait() - - for _, err = range testDirStressGlobals.err { - if nil != err { - t.Fatal(err) - } - } - - testTeardown(t) -} - -func testDirStressThread(threadIndex uint64) { - var ( - basename string - err error - fileInodeNumber InodeNumber - iteration uint64 - linkIndex uint64 - toDestroyInodeNumber InodeNumber - volumeHandle VolumeHandle - ) - - defer testDirStressGlobals.waitGroup.Done() - - volumeHandle, err = FetchVolumeHandle(testDirStressVolumeName) - if nil != err { - testDirStressGlobals.Lock() - testDirStressGlobals.err[threadIndex] = fmt.Errorf("FetchVolumeHandle(\"%s\") failed: %v", testDirStressVolumeName, err) - testDirStressGlobals.Unlock() - return - } - - for iteration = uint64(0); iteration < testDirStressNumIterations; iteration++ { - fileInodeNumber, err = volumeHandle.CreateFile(PosixModePerm, InodeUserID(0), InodeGroupID(0)) - if nil != err { - testDirStressGlobals.Lock() - testDirStressGlobals.err[threadIndex] = fmt.Errorf("volumeHandle.CreateFile(PosixModePerm, InodeUserID(0), InodeGroupID(0)) failed: %v", err) - testDirStressGlobals.Unlock() - return - } - - for linkIndex = uint64(0); linkIndex < testDirStressNumLinksPerIteration; linkIndex++ { - basename = fmt.Sprintf("%s%016X_%016X_%016X", testDirStressBasenamePrefix, testDirStressGlobals.nonce, threadIndex, linkIndex) - - testDirStressGlobals.Lock() - err = volumeHandle.Link(RootDirInodeNumber, basename, fileInodeNumber, false) - if nil != err { - testDirStressGlobals.err[threadIndex] = fmt.Errorf("volumeHandle.Link(RootDirInodeNumber, \"%s\", fileInodeNumber, false) failed: %v", basename, err) - testDirStressGlobals.Unlock() - return - } - testDirStressGlobals.Unlock() - - time.Sleep(testDirStressDelayBetweenLinkAndUnlink) - - testDirStressGlobals.Lock() - toDestroyInodeNumber, err = volumeHandle.Unlink(RootDirInodeNumber, basename, false) - if nil != err { - testDirStressGlobals.err[threadIndex] = fmt.Errorf("volumeHandle.Unlink(RootDirInodeNumber, \"%s\", false) failed: %v", basename, err) - testDirStressGlobals.Unlock() - return - } - // Note that the originally created fileInode is not in any Directory, so unlinking the only dirEntry to it drops LinkCount to zero - if fileInodeNumber != toDestroyInodeNumber { - testDirStressGlobals.err[threadIndex] = fmt.Errorf("volumeHandle.Unlink(RootDirInodeNumber, \"%s\", false) should have returned toDestroyInodeNumber == fileInodeNumber", basename) - testDirStressGlobals.Unlock() - return - } - testDirStressGlobals.Unlock() - } - - err = volumeHandle.Destroy(fileInodeNumber) - if nil != err { - testDirStressGlobals.Lock() - testDirStressGlobals.err[threadIndex] = fmt.Errorf("volumeHandle.Destroy(fileInodeNumber) failed: %v", err) - testDirStressGlobals.Unlock() - return - } - } - - testDirStressGlobals.Lock() - testDirStressGlobals.err[threadIndex] = nil - testDirStressGlobals.Unlock() -} diff --git a/inode/file.go b/inode/file.go deleted file mode 100644 index ac77bbef..00000000 --- a/inode/file.go +++ /dev/null @@ -1,1345 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - "strconv" - "time" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/utils" -) - -type fileExtentStruct struct { - FileOffset uint64 - Length uint64 - LogSegmentNumber uint64 - LogSegmentOffset uint64 -} - -func (vS *volumeStruct) CreateFile(filePerm InodeMode, userID InodeUserID, groupID InodeGroupID) (fileInodeNumber InodeNumber, err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - fileInode, err := vS.createFileInode(filePerm, userID, groupID) - if err != nil { - return 0, err - } - return fileInode.InodeNumber, nil -} - -// REVIEW TODO: Should one of these (and its siblings, CreateDir & CreateSymlink) be doing flush here? - -func (vS *volumeStruct) createFileInode(filePerm InodeMode, userID InodeUserID, groupID InodeGroupID) (fileInode *inMemoryInodeStruct, err error) { - stats.IncrementOperations(&stats.FileCreateOps) - - // Create file mode out of file permissions plus inode type - fileMode, err := determineMode(filePerm, FileType) - if err != nil { - return nil, err - } - - fileInode, err = vS.makeInMemoryInode(FileType, fileMode, userID, groupID) - if err != nil { - return nil, err - } - - fileInode.dirty = true - - // The payload of a file inode is a B+-tree map whose keys are uint64 file - // offsets and whose values are `fileExtent`s. - - extents := - sortedmap.NewBPlusTree( - vS.maxExtentsPerFileNode, - sortedmap.CompareUint64, - &fileInodeCallbacks{treeNodeLoadable{inode: fileInode}}, - globals.fileExtentMapCache) - - fileInode.payload = extents - - ok, err := vS.inodeCacheInsert(fileInode) - if nil != err { - return - } - if !ok { - err = fmt.Errorf("inodeCacheInsert(fileInode) failed") - return - } - - stats.IncrementOperations(&stats.FileCreateSuccessOps) - return -} - -func fileLen(extents sortedmap.BPlusTree) uint64 { - numExtents, err := extents.Len() - if nil != err { - panic(err) - } - if numExtents == 0 { - return 0 - } - - _, value, ok, err := extents.GetByIndex(numExtents - 1) - if nil != err { - panic(err) - } - if !ok { - panic("couldn't find last extent of nonempty file") - } - - lastFileExtent := value.(*fileExtentStruct) - - return lastFileExtent.FileOffset + lastFileExtent.Length -} - -// Update a file inode to have a new size, zero-padding as necessary. -// -// Doesn't flush anything. -func setSizeInMemory(fileInode *inMemoryInodeStruct, size uint64) (err error) { - extents := fileInode.payload.(sortedmap.BPlusTree) - extentIndex, found, err := extents.BisectLeft(size) - if nil != err { - panic(err) - } - - if !found { - if 0 <= extentIndex { - // Potentially trim preceeding extent - _, extentValue, ok, getByIndexErr := extents.GetByIndex(extentIndex) - if nil != getByIndexErr { - panic(getByIndexErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected extents indexing problem") - panic(unexpectedErr) - } - extent := extentValue.(*fileExtentStruct) - if (extent.FileOffset + extent.Length) > size { - // Yes, we need to trim the preceeding extent - trimSize := extent.Length - (size - extent.FileOffset) - extent.Length -= trimSize - ok, patchByIndexErr := extents.PatchByIndex(extentIndex, extent) - if nil != patchByIndexErr { - panic(patchByIndexErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected extents indexing problem") - panic(unexpectedErr) - } - decrementLogSegmentMapFileData(fileInode, extent.LogSegmentNumber, trimSize) - } - } - - extentIndex++ // Step to next extent entry (beyond potentially truncated preceeding extent) - } - - for { - _, extentValue, ok, getByIndexErr := extents.GetByIndex(extentIndex) - if nil != getByIndexErr { - panic(getByIndexErr) - } - if !ok { - break - } - extent := extentValue.(*fileExtentStruct) - decrementLogSegmentMapFileData(fileInode, extent.LogSegmentNumber, extent.Length) - ok, deleteByIndexErr := extents.DeleteByIndex(extentIndex) - if nil != deleteByIndexErr { - panic(deleteByIndexErr) - } - if !ok { - deleteByIndexErr = fmt.Errorf("extent just indexed should have been delete-able") - panic(deleteByIndexErr) - } - } - - fileInode.dirty = true - fileInode.Size = size - - updateTime := time.Now() - fileInode.ModificationTime = updateTime - fileInode.AttrChangeTime = updateTime - return -} - -func (vS *volumeStruct) Read(fileInodeNumber InodeNumber, offset uint64, length uint64, profiler *utils.Profiler) (buf []byte, err error) { - var ( - fileInode *inMemoryInodeStruct - readPlan []ReadPlanStep - readPlanBytes uint64 - snapShotID uint64 - ) - - fileInode, err = vS.fetchInodeType(fileInodeNumber, FileType) - if nil != err { - logger.ErrorWithError(err) - return - } - - _, snapShotID, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(fileInodeNumber)) - - readPlan, readPlanBytes, err = vS.getReadPlanHelper(snapShotID, fileInode, &offset, &length) - if nil != err { - logger.WarnWithError(err) - return - } - - buf, err = vS.doReadPlan(fileInode, readPlan, readPlanBytes) - if nil != err { - logger.WarnWithError(err) - return - } - - stats.IncrementOperationsAndBucketedBytes(stats.FileRead, uint64(len(buf))) - - err = nil - return -} - -func (vS *volumeStruct) GetReadPlan(fileInodeNumber InodeNumber, offset *uint64, length *uint64) (readPlan []ReadPlanStep, err error) { - var ( - readPlanBytes uint64 - snapShotID uint64 - ) - - fileInode, err := vS.fetchInodeType(fileInodeNumber, FileType) - if nil != err { - logger.ErrorWithError(err) - return - } - - if fileInode.dirty { - err = flush(fileInode, false) - if nil != err { - logger.ErrorWithError(err) - return - } - } - - _, snapShotID, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(fileInodeNumber)) - - readPlan, readPlanBytes, err = vS.getReadPlanHelper(snapShotID, fileInode, offset, length) - if nil != err { - logger.ErrorWithError(err) - return - } - - stats.IncrementOperationsBucketedEntriesAndBucketedBytes(stats.FileReadplan, uint64(len(readPlan)), readPlanBytes) - return -} - -func (vS *volumeStruct) getReadPlanHelper(snapShotID uint64, fileInode *inMemoryInodeStruct, requestedOffset *uint64, requestedLength *uint64) (readPlan []ReadPlanStep, readPlanBytes uint64, err error) { - var ( - offset uint64 - ) - - readPlan = make([]ReadPlanStep, 0) - - if requestedOffset == nil && requestedLength == nil { - err = fmt.Errorf("requestedOffset and requestedLength cannot both be nil") - return - } else if requestedOffset == nil { - // Suffix request, e.g. "bytes=-10", the last 10 bytes of the file - if fileInode.Size > *requestedLength { - offset = fileInode.Size - *requestedLength - readPlanBytes = *requestedLength - } else { - // A suffix request for more bytes than the file has must get the whole file. - offset = 0 - readPlanBytes = fileInode.Size - } - } else if requestedLength == nil { - // Prefix request, e.g. "bytes=25-", from byte 25 to the end - offset = *requestedOffset - readPlanBytes = fileInode.Size - *requestedOffset - } else { - offset = *requestedOffset - readPlanBytes = *requestedLength - } - - if (offset + readPlanBytes) > fileInode.Size { - if fileInode.Size > offset { - readPlanBytes = fileInode.Size - offset - } else { - readPlanBytes = 0 - } - } - - if 0 == readPlanBytes { - return - } - - extents := fileInode.payload.(sortedmap.BPlusTree) - - curOffset := offset - terminalOffset := offset + readPlanBytes - - curExtentIndex, _, err := extents.BisectLeft(offset) - if nil != err { - panic(err) - } - - for { - if 0 > curExtentIndex { - // Skip to next (0th) extent - curExtentIndex++ - } - - // Fetch curExtent - _, curExtentValue, ok, getByIndexErr := extents.GetByIndex(curExtentIndex) - if nil != getByIndexErr { - panic(getByIndexErr) - } - if !ok { - // We have reached the end of extents - break - } - curExtent := curExtentValue.(*fileExtentStruct) - - if curExtent.FileOffset >= terminalOffset { - // [curOffset:terminalOffset) ends before curExtent... so we need look no further - break - } - - if (curExtent.FileOffset + curExtent.Length) <= curOffset { - // [curOffset:terminalOffset) starts beyond curExtent... so move to the next extent - curExtentIndex++ - continue - } - - // At this point, we know [curOffset:terminalOffset) intersects curExtent - - if curOffset < curExtent.FileOffset { - // [curOffset:terminalOffset) starts before curExtent... so insert a zero-fill ReadPlanStep to get to curExtent - step := ReadPlanStep{ - LogSegmentNumber: 0, - Offset: 0, - Length: curExtent.FileOffset - curOffset, - AccountName: "", - ContainerName: "", - ObjectName: "", - ObjectPath: "", - } - readPlan = append(readPlan, step) - curOffset = curExtent.FileOffset - } - - skipSize := uint64(0) - - if curOffset > curExtent.FileOffset { - // [curOffset:terminalOffset) starts after curExtent... so update skipSize appropriately - skipSize = curOffset - curExtent.FileOffset - } - - if terminalOffset <= (curExtent.FileOffset + curExtent.Length) { - // [curOffset:terminalOffset) is completed by some or all of curExtent - step := ReadPlanStep{ - LogSegmentNumber: vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(snapShotID, curExtent.LogSegmentNumber), - Offset: curExtent.LogSegmentOffset + skipSize, - Length: terminalOffset - curOffset, - AccountName: vS.accountName, - } - step.ContainerName, step.ObjectName, step.ObjectPath, err = vS.getObjectLocationFromLogSegmentNumber(step.LogSegmentNumber) - if nil != err { - return - } - readPlan = append(readPlan, step) - curOffset = terminalOffset - break - } else { - // [curOffset:terminalOffset) extends beyond curExtent - step := ReadPlanStep{ - LogSegmentNumber: vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(snapShotID, curExtent.LogSegmentNumber), - Offset: curExtent.LogSegmentOffset + skipSize, - Length: curExtent.Length - skipSize, - AccountName: vS.accountName, - } - step.ContainerName, step.ObjectName, step.ObjectPath, err = vS.getObjectLocationFromLogSegmentNumber(step.LogSegmentNumber) - if nil != err { - return - } - readPlan = append(readPlan, step) - curOffset += step.Length - curExtentIndex++ - } - } - - // Append trailing zero-fill ReadPlanStep if necessary - - if curOffset < terminalOffset { - // We need a trailing zero-fill ReadPlanStep - step := ReadPlanStep{ - LogSegmentNumber: 0, - Offset: 0, - Length: terminalOffset - curOffset, - AccountName: "", - ContainerName: "", - ObjectName: "", - ObjectPath: "", - } - - readPlan = append(readPlan, step) - } - - err = nil - return -} - -func (vS *volumeStruct) FetchExtentMapChunk(fileInodeNumber InodeNumber, fileOffset uint64, maxEntriesFromFileOffset int64, maxEntriesBeforeFileOffset int64) (extentMapChunk *ExtentMapChunkStruct, err error) { - var ( - containerName string - encodedLogSegmentNumber uint64 - extentMap sortedmap.BPlusTree - extentMapLen int - extentMapIndex int - extentMapIndexAtOffset int - extentMapIndexAtOffsetFound bool - extentMapIndexEnd int - extentMapIndexStart int - fileExtent *fileExtentStruct - fileExtentAsValue sortedmap.Value - fileInode *inMemoryInodeStruct - objectName string - snapShotID uint64 - ) - - // Validate args - - if maxEntriesFromFileOffset < 1 { - err = fmt.Errorf("inode.FetchExtentMap() requires maxEntriesFromOffset (%d) >= 1", maxEntriesFromFileOffset) - return - } - if maxEntriesBeforeFileOffset < 0 { - err = fmt.Errorf("inode.FetchExtentMap() requires maxEntriesBeforeOffset (%d) >= 0", maxEntriesBeforeFileOffset) - return - } - - fileInode, err = vS.fetchInodeType(fileInodeNumber, FileType) - if nil != err { - return - } - - // Ensure in-flight LogSegments are flushed - - if fileInode.dirty { - err = flush(fileInode, false) - if nil != err { - logger.ErrorWithError(err) - return - } - } - - // Locate extent that either contains fileOffset, - // or if no extent does, that we select the one just after fileOffset - - extentMap = fileInode.payload.(sortedmap.BPlusTree) - - extentMapLen, err = extentMap.Len() - if nil != err { - panic(err) - } - - if 0 == extentMapLen { - // In the absence of any extents, just describe entire fileInode as zero-filled - - extentMapChunk = &ExtentMapChunkStruct{ - FileOffsetRangeStart: 0, - FileOffsetRangeEnd: fileInode.Size, - FileSize: fileInode.Size, - ExtentMapEntry: make([]ExtentMapEntryStruct, 0), - } - return - } - - extentMapIndexAtOffset, extentMapIndexAtOffsetFound, err = extentMap.BisectLeft(fileOffset) - if nil != err { - panic(err) - } - - if !extentMapIndexAtOffsetFound { - if extentMapIndexAtOffset >= 0 { - _, fileExtentAsValue, _, err = extentMap.GetByIndex(extentMapIndexAtOffset) - if nil != err { - panic(err) - } - - fileExtent = fileExtentAsValue.(*fileExtentStruct) - - if (fileExtent.FileOffset + fileExtent.Length) <= fileOffset { - extentMapIndexAtOffset++ - } - } else { - extentMapIndexAtOffset = 0 - } - } - - // Compute extent indices surrounding fileOffset that are also requested - - if int64(extentMapIndexAtOffset) > maxEntriesBeforeFileOffset { - extentMapIndexStart = extentMapIndexAtOffset - int(maxEntriesBeforeFileOffset) - } else { - extentMapIndexStart = 0 - } - - if int64(extentMapLen-extentMapIndexAtOffset) <= maxEntriesFromFileOffset { - extentMapIndexEnd = extentMapLen - 1 - } else { - extentMapIndexEnd = extentMapIndexAtOffset + int(maxEntriesFromFileOffset) - 1 - } - - // Populate extentMapChunk with selected extents - - _, snapShotID, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(fileInodeNumber)) - - extentMapChunk = &ExtentMapChunkStruct{ - FileSize: fileInode.Size, - ExtentMapEntry: make([]ExtentMapEntryStruct, 0, extentMapIndexEnd-extentMapIndexStart+1), - } - - // Fill in FileOffsetRangeStart to include zero-filled (non-)extent just before first returned extent - - if extentMapIndexStart > 0 { - _, fileExtentAsValue, _, err = extentMap.GetByIndex(extentMapIndexStart - 1) - if nil != err { - panic(err) - } - - fileExtent = fileExtentAsValue.(*fileExtentStruct) - - extentMapChunk.FileOffsetRangeStart = fileExtent.FileOffset + fileExtent.Length - } else { - extentMapChunk.FileOffsetRangeStart = 0 - } - - for extentMapIndex = extentMapIndexStart; extentMapIndex <= extentMapIndexEnd; extentMapIndex++ { - _, fileExtentAsValue, _, err = extentMap.GetByIndex(extentMapIndex) - if nil != err { - panic(err) - } - - fileExtent = fileExtentAsValue.(*fileExtentStruct) - - encodedLogSegmentNumber = vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(snapShotID, fileExtent.LogSegmentNumber) - - containerName, objectName, _, err = vS.getObjectLocationFromLogSegmentNumber(encodedLogSegmentNumber) - if nil != err { - panic(err) - } - - extentMapChunk.ExtentMapEntry = append(extentMapChunk.ExtentMapEntry, ExtentMapEntryStruct{ - FileOffset: fileExtent.FileOffset, - LogSegmentOffset: fileExtent.LogSegmentOffset, - Length: fileExtent.Length, - ContainerName: containerName, - ObjectName: objectName, - }) - } - - // Fill in FileOffsetRangeEnd to included zero-filled (non-)extent just after last returned extent - - if (extentMapIndexEnd + 1) == extentMapLen { - extentMapChunk.FileOffsetRangeEnd = fileInode.Size - } else { - _, fileExtentAsValue, _, err = extentMap.GetByIndex(extentMapIndexEnd + 1) - if nil != err { - panic(err) - } - - fileExtent = fileExtentAsValue.(*fileExtentStruct) - - extentMapChunk.FileOffsetRangeEnd = fileExtent.FileOffset - } - - return -} - -func incrementLogSegmentMapFileData(fileInode *inMemoryInodeStruct, logSegmentNumber uint64, incrementAmount uint64) { - logSegmentRecord, ok := fileInode.LogSegmentMap[logSegmentNumber] - if ok { - logSegmentRecord += incrementAmount - fileInode.LogSegmentMap[logSegmentNumber] = logSegmentRecord - } else { - fileInode.LogSegmentMap[logSegmentNumber] = incrementAmount - } -} - -func decrementLogSegmentMapFileData(fileInode *inMemoryInodeStruct, logSegmentNumber uint64, decrementAmount uint64) { - logSegmentRecord, ok := fileInode.LogSegmentMap[logSegmentNumber] - if ok { - if decrementAmount > logSegmentRecord { - err := fmt.Errorf("Unexpected decrementLogSegmentMapFileData() call would make FileData \"go negative\"") - panic(err) - } - logSegmentRecord -= decrementAmount - fileInode.LogSegmentMap[logSegmentNumber] = logSegmentRecord - } else { - err := fmt.Errorf("Unexpected decrementLogSegmentMapFileData() call referenced non-existent logSegmentNumber") - panic(err) - } -} - -// `recordWrite` is called by `Write` and `Wrote` to update the file inode -// payload's record of the extents that compose the file. -func recordWrite(fileInode *inMemoryInodeStruct, fileOffset uint64, length uint64, logSegmentNumber uint64, logSegmentOffset uint64) (err error) { - extents := fileInode.payload.(sortedmap.BPlusTree) - - // First we need to eliminate extents or portions thereof that overlap the specified write - - extentIndex, found, err := extents.BisectLeft(fileOffset) - if nil != err { - panic(err) - } - - if !found { - if 0 <= extentIndex { - // Potentially split preceeding extent - _, extentValue, ok, getByIndexErr := extents.GetByIndex(extentIndex) - if nil != getByIndexErr { - panic(getByIndexErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected extents indexing problem") - panic(unexpectedErr) - } - leftExtent := extentValue.(*fileExtentStruct) - if (leftExtent.FileOffset + leftExtent.Length) > fileOffset { - // Yes, we need to split the preceeding extent - splitOutSize := (leftExtent.FileOffset + leftExtent.Length) - fileOffset - leftExtent.Length -= splitOutSize - ok, patchByIndexErr := extents.PatchByIndex(extentIndex, leftExtent) - if nil != patchByIndexErr { - panic(patchByIndexErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected extents indexing problem") - panic(unexpectedErr) - } - rightExtent := &fileExtentStruct{ - FileOffset: fileOffset, - Length: splitOutSize, - LogSegmentNumber: leftExtent.LogSegmentNumber, - LogSegmentOffset: leftExtent.LogSegmentOffset + leftExtent.Length, - } - ok, putErr := extents.Put(rightExtent.FileOffset, rightExtent) - if nil != putErr { - panic(putErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected extents key problem") - panic(unexpectedErr) - } - } - } - - extentIndex++ // Step to next extent entry (beyond potentially truncated preceeding extent) - } - - for { - _, extentValue, ok, getByIndexErr := extents.GetByIndex(extentIndex) - if nil != getByIndexErr { - panic(getByIndexErr) - } - if !ok { - // We have reached the end of extents - break - } - leftExtent := extentValue.(*fileExtentStruct) - if leftExtent.FileOffset >= (fileOffset + length) { - // We are done pruning extents - break - } - if (fileOffset + length) >= (leftExtent.FileOffset + leftExtent.Length) { - // This extent entirely overwritten... just delete it - decrementLogSegmentMapFileData(fileInode, leftExtent.LogSegmentNumber, leftExtent.Length) - ok, deleteByIndexErr := extents.DeleteByIndex(extentIndex) - if nil != deleteByIndexErr { - panic(deleteByIndexErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected extents indexing problem") - panic(unexpectedErr) - } - } else { - // This extent partially overwritten... trim it from the front and we will be done trimming - ok, deleteByIndexErr := extents.DeleteByIndex(extentIndex) - if nil != deleteByIndexErr { - panic(deleteByIndexErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected extents indexing problem") - panic(unexpectedErr) - } - overlapSize := (fileOffset + length) - leftExtent.FileOffset - rightExtent := &fileExtentStruct{ - FileOffset: leftExtent.FileOffset + overlapSize, - Length: leftExtent.Length - overlapSize, - LogSegmentNumber: leftExtent.LogSegmentNumber, - LogSegmentOffset: leftExtent.LogSegmentOffset + overlapSize, - } - ok, putErr := extents.Put(rightExtent.FileOffset, rightExtent) - if nil != putErr { - panic(putErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected extents key problem") - panic(unexpectedErr) - } - decrementLogSegmentMapFileData(fileInode, leftExtent.LogSegmentNumber, overlapSize) - break - } - } - - // Now that there will be no overlap, see if we can append to the preceding fileExtent - - prevIndex, found, err := extents.BisectLeft(fileOffset) - if nil != err { - panic(err) - } - if found { - unexpectedErr := fmt.Errorf("unexpected to find fileOffset in extents at this point)") - panic(unexpectedErr) - } - - var prevExtent *fileExtentStruct - - prevExtent = nil - - if 0 <= prevIndex { - _, prevExtentValue, ok, getByIndexErr := extents.GetByIndex(prevIndex) - if nil != getByIndexErr { - panic(getByIndexErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected to not find predecessor in extents at this point") - panic(unexpectedErr) - } - - prevExtent = prevExtentValue.(*fileExtentStruct) - } - - if (nil != prevExtent) && (prevExtent.LogSegmentNumber == logSegmentNumber) && ((prevExtent.FileOffset + prevExtent.Length) == fileOffset) && ((prevExtent.LogSegmentOffset + prevExtent.Length) == logSegmentOffset) { - // APPEND Case: We are able to simply lengthen prevExtent - - prevExtent.Length += length - ok, patchByIndexErr := extents.PatchByIndex(prevIndex, prevExtent) - if nil != patchByIndexErr { - panic(patchByIndexErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected to not be able to PATCH at this point") - panic(unexpectedErr) - } - } else { - // Non-APPEND Case: We need to insert a new extent - - newExtent := &fileExtentStruct{ - FileOffset: fileOffset, - Length: length, - LogSegmentNumber: logSegmentNumber, - LogSegmentOffset: logSegmentOffset, - } - - ok, putErr := extents.Put(newExtent.FileOffset, newExtent) - if nil != putErr { - panic(putErr) - } - if !ok { - unexpectedErr := fmt.Errorf("unexpected to not be able to PUT at this point") - panic(unexpectedErr) - } - } - - if (fileOffset + length) > fileInode.Size { - fileInode.Size = fileOffset + length - } - - incrementLogSegmentMapFileData(fileInode, logSegmentNumber, length) - - return nil -} - -func (vS *volumeStruct) Write(fileInodeNumber InodeNumber, offset uint64, buf []byte, profiler *utils.Profiler) (err error) { - err = enforceRWMode(true) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(fileInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("Write() on non-LiveView fileInodeNumber not allowed") - return - } - - fileInode, err := vS.fetchInodeType(fileInodeNumber, FileType) - if nil != err { - logger.ErrorWithError(err) - return - } - - // writes of length 0 succeed but do not change mtime - if len(buf) == 0 { - return - } - - fileInode.dirty = true - - logSegmentNumber, logSegmentOffset, err := vS.doSendChunk(fileInode, buf) - if nil != err { - logger.ErrorWithError(err) - return - } - - length := uint64(len(buf)) - startingSize := fileInode.Size - - err = recordWrite(fileInode, offset, length, logSegmentNumber, logSegmentOffset) - if nil != err { - logger.ErrorWithError(err) - return - } - - appendedBytes := fileInode.Size - startingSize - overwrittenBytes := length - appendedBytes - - stats.IncrementOperationsBucketedBytesAndAppendedOverwritten(stats.FileWrite, length, appendedBytes, overwrittenBytes) - - updateTime := time.Now() - fileInode.AttrChangeTime = updateTime - fileInode.ModificationTime = updateTime - fileInode.NumWrites++ - - return -} - -func (vS *volumeStruct) Wrote(fileInodeNumber InodeNumber, containerName string, objectName string, fileOffset []uint64, objectOffset []uint64, length []uint64, wroteTime time.Time, patchOnly bool) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - if (len(fileOffset) != len(objectOffset)) || (len(objectOffset) != len(length)) { - err = fmt.Errorf("Wrote() called with unequal # of fileOffset's (%d), objectOffset's (%d), and length's (%d)", len(fileOffset), len(objectOffset), len(length)) - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(fileInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("Wrote() on non-LiveView fileInodeNumber not allowed") - return - } - - fileInode, err := vS.fetchInodeType(fileInodeNumber, FileType) - if err != nil { - logger.ErrorWithError(err) - return err - } - - if fileInode.dirty { - err = flush(fileInode, false) - if nil != err { - logger.ErrorWithError(err) - return - } - } - - logSegmentNumber, err := strconv.ParseUint(objectName, 16, 64) - if err != nil { - return - } - - err = fileInode.volume.setLogSegmentContainer(logSegmentNumber, containerName) - if nil != err { - return - } - - fileInode.dirty = true - - if !patchOnly { - err = setSizeInMemory(fileInode, 0) - if err != nil { - logger.ErrorWithError(err) - return - } - } - - bytesWritten := uint64(0) - - for i, thisLength := range length { - if 0 < thisLength { - err = recordWrite(fileInode, fileOffset[i], thisLength, logSegmentNumber, objectOffset[i]) - if err != nil { - logger.ErrorWithError(err) - return - } - - fileInode.NumWrites++ - bytesWritten += thisLength - } - } - - if !patchOnly { - // For this case only, make it appear we did precisely one write - - fileInode.NumWrites = 1 - } - - fileInode.AttrChangeTime = wroteTime - fileInode.ModificationTime = wroteTime - - err = fileInode.volume.flushInode(fileInode) - if err != nil { - logger.ErrorWithError(err) - return - } - - stats.IncrementOperationsAndBucketedBytes(stats.FileWrote, bytesWritten) - - return -} - -func (vS *volumeStruct) SetSize(fileInodeNumber InodeNumber, size uint64) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(fileInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetSize() on non-LiveView fileInodeNumber not allowed") - return - } - - fileInode, err := vS.fetchInodeType(fileInodeNumber, FileType) - if nil != err { - return err - } - - fileInode.dirty = true - - err = setSizeInMemory(fileInode, size) - if nil != err { - logger.ErrorWithError(err) - return - } - - // changing the file's size is just like a write - fileInode.NumWrites++ - - err = fileInode.volume.flushInode(fileInode) - if nil != err { - logger.ErrorWithError(err) - return err - } - - stats.IncrementOperations(&stats.DirSetsizeOps) - - return -} - -func (vS *volumeStruct) Flush(fileInodeNumber InodeNumber, andPurge bool) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - fileInode, ok, err := vS.fetchInode(fileInodeNumber) - if nil != err { - // this indicates disk corruption or software bug - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: request to flush inode %d volume '%s' failed", - utils.GetFnName(), fileInodeNumber, vS.volumeName) - return - } - if !ok { - // this can happen if background flush loses a race with unlink() - logger.Infof("%s: request to flush free inode %d volume '%s' ignored", - utils.GetFnName(), fileInodeNumber, vS.volumeName) - return - } - if fileInode.InodeType != FileType { - // this should never happen unless there's disk corruption - logger.Errorf("%s: request to flush inode %d volume '%s' type '%v' ignored", - utils.GetFnName(), fileInodeNumber, vS.volumeName, fileInode.InodeType) - return - } - - if fileInode.dirty { - err = flush(fileInode, andPurge) - if nil != err { - logger.ErrorWithError(err) - return - } - } - - stats.IncrementOperations(&stats.FileFlushOps) - - return -} - -func flush(fileInode *inMemoryInodeStruct, andPurge bool) (err error) { - vS := fileInode.volume - err = vS.flushInode(fileInode) - if nil != err { - logger.ErrorfWithError(err, "flushInode(fileInode) failed") - } - - if andPurge { - var ok bool - ok, err = vS.inodeCacheDrop(fileInode) - if nil != err { - return - } - if !ok { - err = fmt.Errorf("inodeCacheDrop(fileInode) failed") - return - } - } - - return -} - -func (vS *volumeStruct) resetFileInodeInMemory(fileInode *inMemoryInodeStruct) (err error) { - var ( - fileInodeExtentMap sortedmap.BPlusTree - ok bool - ) - - fileInode.dirty = true - fileInodeExtentMap = fileInode.payload.(sortedmap.BPlusTree) - - ok = true - for ok { - ok, err = fileInodeExtentMap.DeleteByIndex(0) - if nil != err { - err = fmt.Errorf("resetFileInodeInMemory() on Inode# 0x%016X failed: %v", fileInode.InodeNumber, err) - return - } - } - - fileInode.LogSegmentMap = make(map[uint64]uint64) - fileInode.Size = 0 - fileInode.NumWrites = 0 - - err = nil - return -} - -func (vS *volumeStruct) Coalesce(destInodeNumber InodeNumber, metaDataName string, metaData []byte, elements []*CoalesceElement) (attrChangeTime time.Time, modificationTime time.Time, numWrites uint64, fileSize uint64, err error) { - var ( - alreadyInInodeMap bool - coalesceTime time.Time - destInode *inMemoryInodeStruct - destInodeExtentMap sortedmap.BPlusTree - destInodeOffsetBeforeElementAppend uint64 - dirEntryInodeNumber InodeNumber - element *CoalesceElement - elementInode *inMemoryInodeStruct - elementInodeExtent *fileExtentStruct - elementInodeExtentAsValue sortedmap.Value - elementInodeExtentMap sortedmap.BPlusTree - elementInodeExtentMapIndex int - elementInodeExtentMapLen int - inodeList []*inMemoryInodeStruct - inodeMap map[InodeNumber]*inMemoryInodeStruct - localErr error - logSegmentReferencedBytes uint64 - ok bool - snapShotIDType headhunter.SnapShotIDType - toDestroyInodeNumber InodeNumber - ) - - err = enforceRWMode(false) - if nil != err { - return - } - - // Validate all referenced {Dir|File}Inodes - - inodeMap = make(map[InodeNumber]*inMemoryInodeStruct) - inodeList = make([]*inMemoryInodeStruct, 0, 1+len(elements)) - - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(destInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = blunder.NewError(blunder.PermDeniedError, "Coalesce into non-LiveView destInodeNumber 0x%016X not allowed", destInodeNumber) - return - } - - destInode, ok, err = vS.fetchInode(destInodeNumber) - if nil != err { - err = blunder.NewError(blunder.BadFileError, "Coalesce() couldn't fetch destInodeNumber 0x%016X: %v", destInodeNumber, err) - return - } - if !ok { - err = blunder.NewError(blunder.NotFoundError, "Coalesce() couldn't find destInodeNumber 0x%16X", destInodeNumber) - return - } - if destInode.InodeType != FileType { - err = blunder.NewError(blunder.PermDeniedError, "Coalesce() called for destInodeNumber 0x%016X that is not a FileInode", destInodeNumber) - return - } - - inodeMap[destInodeNumber] = destInode - inodeList = append(inodeList, destInode) - - for _, element = range elements { - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(element.ElementInodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = blunder.NewError(blunder.PermDeniedError, "Coalesce() from non-LiveView element.ElementInodeNumber (0x%016X) not allowed", element.ElementInodeNumber) - return - } - - _, alreadyInInodeMap = inodeMap[element.ElementInodeNumber] - if alreadyInInodeMap { - err = blunder.NewError(blunder.InvalidArgError, "Coalesce() called with duplicate Element Inode 0x%016X", element.ElementInodeNumber) - return - } - - elementInode, ok, err = vS.fetchInode(element.ElementInodeNumber) - if nil != err { - err = blunder.NewError(blunder.BadFileError, "Coalesce() couldn't fetch ElementInodeNumber 0x%016X: %v", element.ElementInodeNumber, err) - return - } - - inodeMap[element.ElementInodeNumber] = elementInode - inodeList = append(inodeList, elementInode) - - if !ok { - err = blunder.NewError(blunder.NotFoundError, "Coalesce() couldn't find ElementInodeNumber 0x%16X", element.ElementInodeNumber) - return - } - if elementInode.InodeType != FileType { - err = blunder.NewError(blunder.PermDeniedError, "Coalesce() called for ElementInodeNumber 0x%016X that is not a FileInode", element.ElementInodeNumber) - return - } - if elementInode.LinkCount != 1 { - err = blunder.NewError(blunder.TooManyLinksError, "Coalesce() called for ElementInodeNumber 0x%016X with LinkCount not == 1 (%v)", element.ElementInodeNumber, elementInode.LinkCount) - return - } - - dirEntryInodeNumber, err = vS.lookupByDirInodeNumber(element.ContainingDirectoryInodeNumber, element.ElementName) - if nil != err { - err = blunder.NewError(blunder.InvalidArgError, "Coalesce() called for ElementName %s not found in ContainingDir 0x%016X: %v", element.ElementName, element.ContainingDirectoryInodeNumber, err) - return - } - if dirEntryInodeNumber != element.ElementInodeNumber { - err = blunder.NewError(blunder.InvalidArgError, "Coalesce() called for ElementName %s in ContainingDir 0x%016X had mismatched InodeNumber", element.ElementName, element.ContainingDirectoryInodeNumber) - return - } - } - - // Ensure all referenced FileInodes are pre-flushed - - err = vS.flushInodes(inodeList) - if nil != err { - err = blunder.NewError(blunder.InvalidArgError, "Coalesce() unable to flush inodeList: %v", err) - return - } - - // Now "append" each Element's extents to destInode (creating duplicate references to LogSegments for now) - - destInodeExtentMap = destInode.payload.(sortedmap.BPlusTree) - - destInode.dirty = true - - destInodeOffsetBeforeElementAppend = fileLen(destInodeExtentMap) - - for _, element = range elements { - elementInode = inodeMap[element.ElementInodeNumber] - destInode.NumWrites++ - elementInodeExtentMap = elementInode.payload.(sortedmap.BPlusTree) - elementInodeExtentMapLen, err = elementInodeExtentMap.Len() - for elementInodeExtentMapIndex = 0; elementInodeExtentMapIndex < elementInodeExtentMapLen; elementInodeExtentMapIndex++ { - _, elementInodeExtentAsValue, ok, err = elementInodeExtentMap.GetByIndex(elementInodeExtentMapIndex) - if nil != err { - localErr = vS.resetFileInodeInMemory(destInode) - if nil != localErr { - logger.Fatalf("Coalesce() doing resetFileInodeInMemory(destInode) failed: %v", localErr) - } - localErr = vS.flushInode(destInode) - if nil != localErr { - logger.Errorf("Coalesce() doing flushInode(destInode) failed: %v", localErr) - } - err = blunder.NewError(blunder.InvalidArgError, "Coalesce() unable to fetch fileExtentStruct from ExtentMap: %v", err) - return - } - elementInodeExtent = elementInodeExtentAsValue.(*fileExtentStruct) - elementInodeExtent.FileOffset += destInodeOffsetBeforeElementAppend - _, err = destInodeExtentMap.Put(elementInodeExtent.FileOffset, elementInodeExtent) - if nil != err { - localErr = vS.resetFileInodeInMemory(destInode) - if nil != localErr { - logger.Fatalf("Coalesce() doing resetFileInodeInMemory(destInode) failed: %v", localErr) - } - localErr = vS.flushInode(destInode) - if nil != localErr { - logger.Errorf("Coalesce() doing flushInode(destInode) failed: %v", localErr) - } - err = blunder.NewError(blunder.InvalidArgError, "Coalesce() unable to append elementInodeExtent to destInodeExtentMap: %v", err) - return - } - logSegmentReferencedBytes, ok = destInode.LogSegmentMap[elementInodeExtent.LogSegmentNumber] - if ok { - destInode.LogSegmentMap[elementInodeExtent.LogSegmentNumber] = elementInodeExtent.Length + logSegmentReferencedBytes - } else { - destInode.LogSegmentMap[elementInodeExtent.LogSegmentNumber] = elementInodeExtent.Length - } - } - destInodeOffsetBeforeElementAppend += elementInode.Size - err = setSizeInMemory(destInode, destInodeOffsetBeforeElementAppend) - if nil != err { - localErr = vS.resetFileInodeInMemory(destInode) - if nil != localErr { - logger.Fatalf("Coalesce() doing resetFileInodeInMemory(destInode) failed: %v", localErr) - } - localErr = vS.flushInode(destInode) - if nil != localErr { - logger.Errorf("Coalesce() doing flushInode(destInode) failed: %v", localErr) - } - err = blunder.NewError(blunder.InvalidArgError, "Coalesce() unable to setSize() destInodeNumber 0x%016X: %v", destInodeNumber, err) - return - } - } - - // Now, destInode is fully assembled... update its metadata & assemble remaining results - coalesceTime = time.Now() - destInode.CreationTime = coalesceTime - destInode.AttrChangeTime = coalesceTime - destInode.ModificationTime = coalesceTime - - // attach new middleware headers (why are we making a copy?) - inodeStreamBuf := make([]byte, len(metaData)) - copy(inodeStreamBuf, metaData) - destInode.StreamMap[metaDataName] = inodeStreamBuf - - // collect the NumberOfWrites value while locked (important for Etag) - numWrites = destInode.NumWrites - fileSize = destInode.Size - attrChangeTime = destInode.AttrChangeTime - modificationTime = destInode.ModificationTime - - // Now, destInode is fully assembled... need to remove all elements references to currently shared LogSegments - - for _, element = range elements { - localErr = vS.resetFileInodeInMemory(inodeMap[element.ElementInodeNumber]) - if nil != err { - logger.Fatalf("Coalesce() doing resetFileInodeInMemory(inodeMap[element.ElementInodeNumber]) failed: %v", localErr) - } - } - - // Time to flush all affected FileInodes - - err = vS.flushInodes(inodeList) - if nil != err { - err = fmt.Errorf("Coalesce() doing flushInodes(inodeList) failed: %v", err) - return - } - - // Now we can Unlink and Destroy each element - - for _, element = range elements { - toDestroyInodeNumber, err = vS.Unlink(element.ContainingDirectoryInodeNumber, element.ElementName, false) - if nil != err { - err = fmt.Errorf("Coalesce() doing Unlink(element.ContainingDirectoryInodeNumber, element.ElementName, false) failed: %v", err) - return - } - if toDestroyInodeNumber != element.ElementInodeNumber { - logger.Fatalf("Coalesce() doing Unlink(element.ContainingDirectoryInodeNumber, element.ElementName, false) was expected to return toDestroyInodeNumber == element.ElementInodeNumber") - } - err = vS.Destroy(element.ElementInodeNumber) - if nil != err { - err = fmt.Errorf("Coalesce() doing Destroy(element.ElementInodeNumber) failed: %v", err) - return - } - } - - // All done - - err = nil - return -} - -func (vS *volumeStruct) DefragmentFile(fileInodeNumber InodeNumber, startingFileOffset uint64, chunkSize uint64) (nextFileOffset uint64, eofReached bool, err error) { - var ( - chunk []byte - chunkSizeCapped uint64 - fileInode *inMemoryInodeStruct - ) - - err = enforceRWMode(false) - if nil != err { - return - } - - fileInode, err = vS.fetchInodeType(fileInodeNumber, FileType) - if nil != err { - return - } - - if startingFileOffset >= fileInode.Size { - nextFileOffset = fileInode.Size - eofReached = true - err = nil - return - } - - if (startingFileOffset + chunkSize) >= fileInode.Size { - chunkSizeCapped = fileInode.Size - startingFileOffset - eofReached = true - } else { - chunkSizeCapped = chunkSize - eofReached = false - } - - nextFileOffset = startingFileOffset + chunkSizeCapped - - chunk, err = vS.Read(fileInodeNumber, startingFileOffset, chunkSizeCapped, nil) - if nil != err { - return - } - - err = vS.Write(fileInodeNumber, startingFileOffset, chunk, nil) - - return // err as returned by Write() is sufficient -} - -func (vS *volumeStruct) setLogSegmentContainer(logSegmentNumber uint64, containerName string) (err error) { - containerNameAsByteSlice := utils.StringToByteSlice(containerName) - err = vS.headhunterVolumeHandle.PutLogSegmentRec(logSegmentNumber, containerNameAsByteSlice) - return -} - -func (vS *volumeStruct) getLogSegmentContainer(logSegmentNumber uint64) (containerName string, err error) { - containerNameAsByteSlice, err := vS.headhunterVolumeHandle.GetLogSegmentRec(logSegmentNumber) - if nil != err { - return - } - containerName = utils.ByteSliceToString(containerNameAsByteSlice) - return -} - -func (vS *volumeStruct) getObjectLocationFromLogSegmentNumber(logSegmentNumber uint64) (containerName string, objectName string, objectPath string, err error) { - var ( - nonce uint64 - ) - - containerName, err = vS.getLogSegmentContainer(logSegmentNumber) - if nil != err { - return - } - - _, _, nonce = vS.headhunterVolumeHandle.SnapShotU64Decode(logSegmentNumber) - - objectName = fmt.Sprintf("%016X", nonce) - objectPath = fmt.Sprintf("/v1/%s/%s/%016X", vS.accountName, containerName, nonce) - return -} diff --git a/inode/file_extent_test.go b/inode/file_extent_test.go deleted file mode 100644 index 1882422a..00000000 --- a/inode/file_extent_test.go +++ /dev/null @@ -1,640 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "bytes" - cryptoRand "crypto/rand" - "fmt" - "math/big" - mathRand "math/rand" - "testing" - - "github.com/NVIDIA/sortedmap" -) - -const ( - sizeOfTestFile uint64 = 0x01000000 // 16 MiB - maxExtentLength uint64 = 0x00001000 // 4 KiB - numExtentOverwrites uint64 = 0x00001000 // 4 Ki - numZeroLengthReads uint64 = 0x00001000 // 4 Ki - - pseudoRandom = false - pseudoRandomSeed = int64(0) -) - -var ( - randSource *mathRand.Rand // A source for pseudo-random numbers (if selected) - inMemoryFileInodeContents []byte // A value of 0x00 means the byte has never been written - byteToWrite = byte(0x01) // Incremented for each write, wrapping from 0xFF to 0x01 per above - curExtentOverwrites = uint64(0) // During write phase, loop until == numExtentOverwrites - curFileSize = uint64(0) // Tracks the highest offset actually written -) - -func testChooseUint64(t *testing.T, mustBeLessThan uint64) (u64 uint64) { - if pseudoRandom { - if nil == randSource { - randSource = mathRand.New(mathRand.NewSource(pseudoRandomSeed)) - } - - u64 = uint64(randSource.Int63n(int64(mustBeLessThan))) - } else { - mustBeLessThanBigIntPtr := big.NewInt(int64(mustBeLessThan)) - u64BigIntPtr, err := cryptoRand.Int(cryptoRand.Reader, mustBeLessThanBigIntPtr) - if nil != err { - t.Fatalf("rand.Int(rand.Reader, mustBeLessThanBigIntPtr) returned error == \"%v\"", err) - } - - u64 = u64BigIntPtr.Uint64() - } - - return -} - -func testChooseExtentStart(t *testing.T) (offset uint64) { - offset = testChooseUint64(t, sizeOfTestFile) - - return -} - -func testChooseExtentSize(t *testing.T) (length uint64) { - length = testChooseUint64(t, maxExtentLength) + 1 - - return -} - -func testChooseExtent(t *testing.T) (offset uint64, length uint64) { - offset = testChooseExtentStart(t) - length = testChooseExtentSize(t) - - if (offset + length) > sizeOfTestFile { - length = sizeOfTestFile - offset - } - - return -} - -func testPopulateFileInode(t *testing.T, testVolumeHandle VolumeHandle, fileInodeNumber InodeNumber) { - for curExtentOverwrites < numExtentOverwrites { - offset, length := testChooseExtent(t) - - overwriteOccurred := false - - for i := offset; i < (offset + length); i++ { - if byte(0x00) != inMemoryFileInodeContents[i] { - overwriteOccurred = true - } - - inMemoryFileInodeContents[i] = byteToWrite - } - if overwriteOccurred { - curExtentOverwrites++ - } - - err := testVolumeHandle.Write(fileInodeNumber, offset, inMemoryFileInodeContents[offset:(offset+length)], nil) - if nil != err { - t.Fatalf("Write(fileInodeNumber, offset, inMemoryFileInodeContents[offset:(offset+length)]) failed: %v", err) - } - - if (offset + length) > curFileSize { - curFileSize = offset + length - } - - if byte(0xFF) == byteToWrite { - byteToWrite = byte(0x01) - } else { - byteToWrite++ - } - } -} - -func testValidateFileInodeBPlusTree(t *testing.T, testVolumeHandle VolumeHandle, fileInodeNumber InodeNumber) { - // Fetch actual fileInode - - fileInode, err := (testVolumeHandle.(*volumeStruct)).fetchInodeType(fileInodeNumber, FileType) - if nil != err { - t.Fatalf("testVolumeHandle.fetchInodeType(fileInodeNumber, FileType) failed: %v", err) - } - - // Fetch extents B+Tree - - extents := fileInode.payload.(sortedmap.BPlusTree) - extentsLen, err := extents.Len() - if nil != err { - t.Fatalf("extents.Len() failed: %v", err) - } - - // Walk extents B+Tree (by index) ensuring coherence with: - // 1 - All keys (fileOffset) are monotonically increasing - // 2 - All keys agree with the fileOffset field of their values - // 3 - All values (extents) don't overlap - // 4 - GetByKey(fileOffset) returns the same extent - - if 0 == extentsLen { - return - } - - var lastExtentTerminationFileOffset uint64 - - for extentIndex := 0; extentIndex < extentsLen; extentIndex++ { - // Fetch current extent - keyByIndex, valueByIndex, ok, err := extents.GetByIndex(extentIndex) - if nil != err { - t.Fatalf("extents.GetByIndex(extentIndex) failed: %v", err) - } - if !ok { - err = fmt.Errorf("extents.GetByIndex(extentIndex) unexpectedly returned !ok") - t.Fatalf(err.Error()) - } - fileOffsetByIndex, ok := keyByIndex.(uint64) - if !ok { - err = fmt.Errorf("keyByIndex.(uint64) unexpectedly returned !ok") - t.Fatalf(err.Error()) - } - extentByIndex, ok := valueByIndex.(*fileExtentStruct) - if !ok { - err = fmt.Errorf("valueByIndex.(*fileExtent) unexpectedly returned !ok") - t.Fatalf(err.Error()) - } - - // Validate extent - if fileOffsetByIndex != extentByIndex.FileOffset { - err = fmt.Errorf("fileOffsetByIndex != extentByIndex.fileOffset") - t.Fatalf(err.Error()) - } - if 0 == extentByIndex.Length { - err = fmt.Errorf("0 == extentByIndex.length") - t.Fatalf(err.Error()) - } - - valueByKey, ok, err := extents.GetByKey(fileOffsetByIndex) - if nil != err { - t.Fatalf("extents.GetByKey(fileOffsetByIndex) failed: %v", err) - } - if !ok { - err = fmt.Errorf("extents.GetByKey(fileOffsetByIndex) unexpectedly returned !ok") - t.Fatalf(err.Error()) - } - extentByKey, ok := valueByKey.(*fileExtentStruct) - if !ok { - err = fmt.Errorf("valueByKey.(*fileExtent) unexpectedly returned !ok") - t.Fatalf(err.Error()) - } - if fileOffsetByIndex != extentByKey.FileOffset { - err = fmt.Errorf("fileOffsetByIndex != extentByKey.fileOffset") - t.Fatalf(err.Error()) - } - - if 0 < extentIndex { - // Ensure this extent strictly follows prior extent - if lastExtentTerminationFileOffset > fileOffsetByIndex { - err = fmt.Errorf("(lastExtentFileOffset + lastExtentLength) > fileOffsetByIndex") - t.Fatalf(err.Error()) - } - } - - // Save this extent's lastExtentTerminationFileOffset for next iteration - lastExtentTerminationFileOffset = fileOffsetByIndex + extentByIndex.Length - } - - // If we reach here, extents is valid -} - -func testVerifyFileInodeBPlusTree(t *testing.T, testVolumeHandle VolumeHandle, fileInodeNumber InodeNumber) { - // Fetch actual fileInode - - fileInode, err := (testVolumeHandle.(*volumeStruct)).fetchInodeType(fileInodeNumber, FileType) - if nil != err { - t.Fatalf("testVolumeHandle.fetchInodeType(fileInodeNumber, FileType) failed: %v", err) - } - - // Fetch extents B+Tree - - extents := fileInode.payload.(sortedmap.BPlusTree) - extentsLen, err := extents.Len() - if nil != err { - t.Fatalf("extents.Len() failed: %v", err) - } - - if 0 == extentsLen { - // Verify entire inMemoryFileInodeContents is all zeroes - for _, byteValue := range inMemoryFileInodeContents { - if 0 != byteValue { - err = fmt.Errorf("0 != byteValue [case 1]") - t.Fatalf(err.Error()) - } - } - - // If we reach here, inMemoryFileInodeContents was all zeroes and (this trivial) extents is verified - return - } - - var lastExtentTerminationFileOffset uint64 - - for extentIndex := 0; extentIndex < extentsLen; extentIndex++ { - // Fetch current extent - key, value, ok, err := extents.GetByIndex(extentIndex) - if nil != err { - t.Fatalf("extents.GetByIndex(extentIndex) failed: %v", err) - } - if !ok { - err = fmt.Errorf("extents.GetByIndex(extentIndex) unexpectedly returned !ok") - t.Fatalf(err.Error()) - } - fileOffset, ok := key.(uint64) - if !ok { - err = fmt.Errorf("key.(uint64) unexpectedly returned !ok") - t.Fatalf(err.Error()) - } - extent, ok := value.(*fileExtentStruct) - if !ok { - err = fmt.Errorf("value.(*fileExtent) unexpectedly returned !ok") - t.Fatalf(err.Error()) - } - - // Verify preceeding hole (if any) in extents matches all zeroes in inMemoryFileInodeContents - for _, byteValue := range inMemoryFileInodeContents[lastExtentTerminationFileOffset:fileOffset] { - if 0 != byteValue { - err = fmt.Errorf("0 != byteValue [case 2]") - t.Fatalf(err.Error()) - } - } - - // Update lastExtentTerminationFileOffset for next iteration but used for non-zero check below as well - lastExtentTerminationFileOffset = fileOffset + extent.Length - - // Verify extent matches non-zeroes in inMemoryFileInodeContents - for _, byteValue := range inMemoryFileInodeContents[fileOffset:lastExtentTerminationFileOffset] { - if 0 == byteValue { - err = fmt.Errorf("0 == byteValue") - t.Fatalf(err.Error()) - } - } - } - - // Verify inMemoryFileInodeContents agrees that lastExtentTerminationFileOffset is EOF - for _, byteValue := range inMemoryFileInodeContents[lastExtentTerminationFileOffset:] { - if 0 != byteValue { - err = fmt.Errorf("0 != byteValue [case 3]") - t.Fatalf(err.Error()) - } - } - - // If we reach here, extents is verified -} - -func testCondenseByteSlice(buf []byte) (condensedString string) { - type countValueTupleStruct struct { - count uint64 - value byte - } - - var countValueTupleSlice []*countValueTupleStruct - - for _, value := range buf { - countValueTupleSliceLen := len(countValueTupleSlice) - if (0 == countValueTupleSliceLen) || (value != countValueTupleSlice[countValueTupleSliceLen-1].value) { - countValueTupleSlice = append(countValueTupleSlice, &countValueTupleStruct{count: 1, value: value}) - } else { - countValueTupleSlice[countValueTupleSliceLen-1].count++ - } - } - - var condensedByteSlice []byte - - condensedByteSlice = append(condensedByteSlice, '[') - - for countValueTupleIndex, countValueTuple := range countValueTupleSlice { - if 0 < countValueTupleIndex { - condensedByteSlice = append(condensedByteSlice, ',', ' ') - } - - valueAsString := fmt.Sprintf("0x%02x", countValueTuple.value) - - if 1 == countValueTuple.count { - condensedByteSlice = append(condensedByteSlice, []byte(valueAsString)...) - } else { - countAsString := fmt.Sprintf("0x%x", countValueTuple.count) - condensedByteSlice = append(condensedByteSlice, []byte(countAsString+"("+valueAsString+")")...) - } - } - - condensedByteSlice = append(condensedByteSlice, ']') - - condensedString = string(condensedByteSlice[:]) - - return -} - -func testVerifyFileInodeContents(t *testing.T, testVolumeHandle VolumeHandle, fileInodeNumber InodeNumber) { - // Verify written sections of fileInode match inMemoryFileInodeContents (i.e. non-0x00 bytes all match) - - offset := uint64(0) - length := uint64(0) - currentlyScanningWrittenBytes := false - - for (offset + length) < curFileSize { - if currentlyScanningWrittenBytes { - if byte(0x00) == inMemoryFileInodeContents[offset+length] { - readBuf, err := testVolumeHandle.Read(fileInodeNumber, offset, length, nil) - if nil != err { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 1] failed: %v", err) - } - - if bytes.Compare(readBuf, inMemoryFileInodeContents[offset:(offset+length)]) != 0 { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 1] returned unexpected []byte:\n expected %v\n got %v", testCondenseByteSlice(inMemoryFileInodeContents[offset:(offset+length)]), testCondenseByteSlice(readBuf)) - } - - offset += length - length = uint64(0) - currentlyScanningWrittenBytes = false - } else { - length++ - } - } else { - if byte(0x00) == inMemoryFileInodeContents[offset+length] { - offset++ - } else { - length = uint64(1) - currentlyScanningWrittenBytes = true - } - } - } - - if currentlyScanningWrittenBytes { - readBuf, err := testVolumeHandle.Read(fileInodeNumber, offset, length, nil) - if nil != err { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 2] failed: %v", err) - } - - if 0 != bytes.Compare(readBuf, inMemoryFileInodeContents[offset:(offset+length)]) { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 2] returned unexpected []byte:\n expected %v\n got %v", testCondenseByteSlice(inMemoryFileInodeContents[offset:(offset+length)]), testCondenseByteSlice(readBuf)) - } - } - - // Walk through entire fileInode verifying entire inMemoryFileInodeContents - - offset = uint64(0) - - for offset < curFileSize { - length = testChooseExtentSize(t) - - if offset+length > curFileSize { - length = curFileSize - offset - } - - readBuf, err := testVolumeHandle.Read(fileInodeNumber, offset, length, nil) - if nil != err { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 3] failed: %v", err) - } - - if 0 != bytes.Compare(readBuf, inMemoryFileInodeContents[offset:(offset+length)]) { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 3] returned unexpected []byte:\n expected %v\n got %v", testCondenseByteSlice(inMemoryFileInodeContents[offset:(offset+length)]), testCondenseByteSlice(readBuf)) - } - - offset += length - } - - // Issue numZeroLengthReads zero-length reads using both Read() & GetReadPlan() for likely valid offsets - - length = uint64(0) - - for i := uint64(0); i < numZeroLengthReads; i++ { - offset = testChooseExtentStart(t) - - readBuf, err := testVolumeHandle.Read(fileInodeNumber, offset, length, nil) - if nil != err { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 4] failed: %v", err) - } - - if 0 != len(readBuf) { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 4] returned unexpected []byte:\n expected %v\n got %v", testCondenseByteSlice(inMemoryFileInodeContents[offset:(offset+length)]), testCondenseByteSlice(readBuf)) - } - - readPlan, err := testVolumeHandle.GetReadPlan(fileInodeNumber, &offset, &length) - if nil != err { - t.Fatalf("GetReadPlan(fileInodeNumber, offset, length) [case 4] failed: %v", err) - } - - if 0 != len(readPlan) { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 4] returned unexpected readPlan of length %v", len(readPlan)) - } - } - - // Issue numZeroLengthReads zero-length reads using both Read() & GetReadPlan() for definitely invalid offsets - - length = uint64(0) - - for i := uint64(0); i < numZeroLengthReads; i++ { - offset = testChooseExtentStart(t) + sizeOfTestFile - - readBuf, err := testVolumeHandle.Read(fileInodeNumber, offset, length, nil) - if nil != err { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 5] failed: %v", err) - } - - if 0 != len(readBuf) { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 5] returned unexpected []byte:\n expected %v\n got %v", testCondenseByteSlice(inMemoryFileInodeContents[offset:(offset+length)]), testCondenseByteSlice(readBuf)) - } - - readPlan, err := testVolumeHandle.GetReadPlan(fileInodeNumber, &offset, &length) - if nil != err { - t.Fatalf("GetReadPlan(fileInodeNumber, offset, length) [case 5] failed: %v", err) - } - - if 0 != len(readPlan) { - t.Fatalf("Read(fileInodeNumber, offset, length) [case 5] returned unexpected readPlan of length %v", len(readPlan)) - } - } -} - -func TestFileExtents(t *testing.T) { - testSetup(t, false) - - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") should have worked - got error: %v", err) - } - - fileInodeNumber, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - inMemoryFileInodeContents = make([]byte, sizeOfTestFile) // Initially all 0x00... meaning no bytes written - - testPopulateFileInode(t, testVolumeHandle, fileInodeNumber) - - testValidateFileInodeBPlusTree(t, testVolumeHandle, fileInodeNumber) - - testVerifyFileInodeBPlusTree(t, testVolumeHandle, fileInodeNumber) - - // One might expect to be able to call `testVerifyFileInodeContents` here, - // but we haven't flushed yet. - - err = testVolumeHandle.Flush(fileInodeNumber, false) - if nil != err { - t.Fatalf("Flush(fileInodeNumber, false) failed: %v", err) - } - - testVerifyFileInodeContents(t, testVolumeHandle, fileInodeNumber) - - err = testVolumeHandle.Purge(fileInodeNumber) - if nil != err { - t.Fatalf("Purge(fileInodeNumber) [case one] failed: %v", err) - } - - testVerifyFileInodeContents(t, testVolumeHandle, fileInodeNumber) - - err = testVolumeHandle.Purge(fileInodeNumber) - if nil != err { - t.Fatalf("Purge(fileInodeNumber) [case two] failed: %v", err) - } - - err = testVolumeHandle.Destroy(fileInodeNumber) - if nil != err { - t.Fatalf("Destroy(fileInodeNumber) failed: %v", err) - } - - testTeardown(t) -} - -func TestWriteFileExtentAtExtantOffset(t *testing.T) { - testSetup(t, false) - - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") should have worked - got error: %v", err) - } - - fileInodeNumber, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - fileInode, ok, err := (testVolumeHandle.(*volumeStruct)).fetchInode(fileInodeNumber) - if err != nil { - t.Fatalf("testVolumeHandle.fetchInode() failed: %v", err) - } - if !ok { - t.Fatalf("testVolumeHandle.fetchInode() returned a free inode") - } - - extents := fileInode.payload.(sortedmap.BPlusTree) - - err = testVolumeHandle.Write(fileInodeNumber, 0, make([]byte, 20), nil) - if nil != err { - t.Fatalf("Write(fileInodeNumber, 0, make([]byte, 20)) failed: %v", err) - } - - err = testVolumeHandle.Write(fileInodeNumber, 5, []byte("aaaa"), nil) // 4 bytes - if nil != err { - t.Fatalf("Write failed: %v", err) - } - - // At this point, our file B+-tree should have three extents starting at - // file offsets 0, 5, and 5+4=9. - - expectedOffsets := []uint64{0, 5, 9} - expectedLengths := []uint64{5, 4, 11} - for i := 0; i < 3; i++ { - _, value, ok, err := extents.GetByIndex(i) - if nil != err { - t.Fatal(err) - } - extantExtent := value.(*fileExtentStruct) - if !ok { - t.Fatalf("expected to be able to get extent") - } - if expectedOffsets[i] != extantExtent.FileOffset { - t.Fatalf("expected extent to be at offset %v, got %v", expectedOffsets[i], extantExtent.FileOffset) - } - if expectedLengths[i] != extantExtent.Length { - t.Fatalf("expected extent length %v, got %v", expectedLengths[i], extantExtent.Length) - } - } - - err = testVolumeHandle.Write(fileInodeNumber, 9, []byte("bbb"), nil) - if nil != err { - t.Fatalf("Overwrite failed: %v", err) - } - - err = testVolumeHandle.Flush(fileInodeNumber, false) - if nil != err { - t.Fatalf("Flush failed: %v", err) - } - - testTeardown(t) -} - -func TestOverwriteIncludesBeginningOfLastExtent(t *testing.T) { - testSetup(t, false) - - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") should have worked - got error: %v", err) - } - - fileInodeNumber, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - err = testVolumeHandle.Write(fileInodeNumber, 0, make([]byte, 20), nil) - if nil != err { - t.Fatalf("Write(fileInodeNumber, 0, make([]byte, 20)) failed: %v", err) - } - - err = testVolumeHandle.Write(fileInodeNumber, 5, []byte("aaaa"), nil) // 4 bytes - if nil != err { - t.Fatalf("Write failed: %v", err) - } - - err = testVolumeHandle.Write(fileInodeNumber, 3, []byte("bbbbbbbbbb"), nil) - if nil != err { - t.Fatalf("Write failed: %v", err) - } - - err = testVolumeHandle.Flush(fileInodeNumber, false) - if nil != err { - t.Fatalf("Flush failed: %v", err) - } - - testTeardown(t) -} - -func TestReadYourWrite(t *testing.T) { - testSetup(t, false) - - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") should have worked - got error: %v", err) - } - - fileInodeNumber, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - ourBytes := []byte{1, 2, 3, 4, 5, 6, 7, 8} - err = testVolumeHandle.Write(fileInodeNumber, 0, ourBytes, nil) - if nil != err { - t.Fatalf("Write(fileInodeNumber, 0, []byte{1, 2, 3, 4, 5, 6, 7, 8}) failed: %v", err) - } - readBuf, err := testVolumeHandle.Read(fileInodeNumber, 0, 8, nil) - if err != nil { - t.Fatalf("Read(fileInodeNumber, 0, 8) failed: %v", err) - } - - if bytes.Compare(ourBytes, readBuf) != 0 { - t.Fatalf("read after write didn't work: expected %v, got %v", ourBytes, readBuf) - } - - err = testVolumeHandle.Flush(fileInodeNumber, false) - if nil != err { - t.Fatalf("Flush failed: %v", err) - } - - testTeardown(t) -} diff --git a/inode/file_flusher.go b/inode/file_flusher.go deleted file mode 100644 index 779cfd80..00000000 --- a/inode/file_flusher.go +++ /dev/null @@ -1,549 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/utils" -) - -func openLogSegmentLRUInsertWhileLocked(inFlightLogSegment *inFlightLogSegmentStruct) { - // Place inode at the MRU end of openLogSegmentLRU - - if 0 == globals.openLogSegmentLRUItems { - globals.openLogSegmentLRUHead = inFlightLogSegment - globals.openLogSegmentLRUTail = inFlightLogSegment - globals.openLogSegmentLRUItems = 1 - } else { - inFlightLogSegment.openLogSegmentLRUPrev = globals.openLogSegmentLRUTail - inFlightLogSegment.openLogSegmentLRUPrev.openLogSegmentLRUNext = inFlightLogSegment - - globals.openLogSegmentLRUTail = inFlightLogSegment - globals.openLogSegmentLRUItems++ - } -} - -func openLogSegmentLRUInsert(inFlightLogSegment *inFlightLogSegmentStruct) { - globals.Lock() - openLogSegmentLRUInsertWhileLocked(inFlightLogSegment) - globals.Unlock() -} - -func openLogSegmentLRUTouchWhileLocked(inFlightLogSegment *inFlightLogSegmentStruct) { - // Move inode to the MRU end of openLogSegmentLRU - - if inFlightLogSegment != globals.openLogSegmentLRUTail { - if inFlightLogSegment == globals.openLogSegmentLRUHead { - globals.openLogSegmentLRUHead = inFlightLogSegment.openLogSegmentLRUNext - globals.openLogSegmentLRUHead.openLogSegmentLRUPrev = nil - - inFlightLogSegment.openLogSegmentLRUPrev = globals.openLogSegmentLRUTail - inFlightLogSegment.openLogSegmentLRUNext = nil - - globals.openLogSegmentLRUTail.openLogSegmentLRUNext = inFlightLogSegment - globals.openLogSegmentLRUTail = inFlightLogSegment - } else { - inFlightLogSegment.openLogSegmentLRUPrev.openLogSegmentLRUNext = inFlightLogSegment.openLogSegmentLRUNext - inFlightLogSegment.openLogSegmentLRUNext.openLogSegmentLRUPrev = inFlightLogSegment.openLogSegmentLRUPrev - - inFlightLogSegment.openLogSegmentLRUNext = nil - inFlightLogSegment.openLogSegmentLRUPrev = globals.openLogSegmentLRUTail - - globals.openLogSegmentLRUTail.openLogSegmentLRUNext = inFlightLogSegment - globals.openLogSegmentLRUTail = inFlightLogSegment - } - } -} - -func openLogSegmentLRUTouch(inFlightLogSegment *inFlightLogSegmentStruct) { - globals.Lock() - openLogSegmentLRUTouchWhileLocked(inFlightLogSegment) - globals.Unlock() -} - -func openLogSegmentLRURemoveWhileLocked(inFlightLogSegment *inFlightLogSegmentStruct) { - if inFlightLogSegment == globals.openLogSegmentLRUHead { - if inFlightLogSegment == globals.openLogSegmentLRUTail { - globals.openLogSegmentLRUHead = nil - globals.openLogSegmentLRUTail = nil - globals.openLogSegmentLRUItems = 0 - } else { - globals.openLogSegmentLRUHead = inFlightLogSegment.openLogSegmentLRUNext - globals.openLogSegmentLRUHead.openLogSegmentLRUPrev = nil - globals.openLogSegmentLRUItems-- - - inFlightLogSegment.openLogSegmentLRUNext = nil - } - } else { - if inFlightLogSegment == globals.openLogSegmentLRUTail { - globals.openLogSegmentLRUTail = inFlightLogSegment.openLogSegmentLRUPrev - globals.openLogSegmentLRUTail.openLogSegmentLRUNext = nil - globals.openLogSegmentLRUItems-- - - inFlightLogSegment.openLogSegmentLRUPrev = nil - } else { - inFlightLogSegment.openLogSegmentLRUPrev.openLogSegmentLRUNext = inFlightLogSegment.openLogSegmentLRUNext - inFlightLogSegment.openLogSegmentLRUNext.openLogSegmentLRUPrev = inFlightLogSegment.openLogSegmentLRUPrev - globals.openLogSegmentLRUItems-- - - inFlightLogSegment.openLogSegmentLRUNext = nil - inFlightLogSegment.openLogSegmentLRUPrev = nil - } - } -} - -func openLogSegmentLRURemove(inFlightLogSegment *inFlightLogSegmentStruct) { - globals.Lock() - openLogSegmentLRURemoveWhileLocked(inFlightLogSegment) - globals.Unlock() -} - -func (volumeGroup *volumeGroupStruct) capReadCacheWhileLocked() { - for uint64(len(volumeGroup.readCache)) > volumeGroup.readCacheLineCount { - - delete(volumeGroup.readCache, volumeGroup.readCacheLRU.readCacheKey) - volumeGroup.readCacheLRU = volumeGroup.readCacheLRU.prev - volumeGroup.readCacheLRU.next = nil - } -} - -func (volumeGroup *volumeGroupStruct) insertReadCacheElementWhileLocked(readCacheElement *readCacheElementStruct) { - volumeGroup.readCache[readCacheElement.readCacheKey] = readCacheElement - if nil == volumeGroup.readCacheMRU { - volumeGroup.readCacheMRU = readCacheElement - volumeGroup.readCacheLRU = readCacheElement - } else { - readCacheElement.next = volumeGroup.readCacheMRU - readCacheElement.next.prev = readCacheElement - volumeGroup.readCacheMRU = readCacheElement - } - volumeGroup.capReadCacheWhileLocked() -} - -func (volumeGroup *volumeGroupStruct) touchReadCacheElementWhileLocked(readCacheElement *readCacheElementStruct) { - if volumeGroup.readCacheMRU != readCacheElement { - if readCacheElement == volumeGroup.readCacheLRU { - volumeGroup.readCacheLRU = readCacheElement.prev - volumeGroup.readCacheLRU.next = nil - } else { - readCacheElement.prev.next = readCacheElement.next - readCacheElement.next.prev = readCacheElement.prev - } - readCacheElement.next = volumeGroup.readCacheMRU - readCacheElement.prev = nil - volumeGroup.readCacheMRU.prev = readCacheElement - volumeGroup.readCacheMRU = readCacheElement - } -} - -func (vS *volumeStruct) doReadPlan(fileInode *inMemoryInodeStruct, readPlan []ReadPlanStep, readPlanBytes uint64) (buf []byte, err error) { - var ( - cacheLine []byte - cacheLineHitLength uint64 - cacheLineHitOffset uint64 - cacheLineStartOffset uint64 - chunkOffset uint64 - inFlightHit bool - inFlightHitBuf []byte - inFlightLogSegment *inFlightLogSegmentStruct - readCacheElement *readCacheElementStruct - readCacheHit bool - readCacheKey readCacheKeyStruct - readCacheLineSize uint64 - remainingLength uint64 - step ReadPlanStep - stepIndex int - volumeGroup *volumeGroupStruct - ) - - volumeGroup = vS.volumeGroup - readCacheLineSize = volumeGroup.readCacheLineSize - readCacheKey.volumeName = vS.volumeName - - if 1 == len(readPlan) { - // Possibly a trivial case (allowing for a potential zero-copy return)... three exist: - // Case 1: The lone step calls for a zero-filled []byte - // Case 2: The lone step is satisfied by reading from an inFlightLogSegment - // Case 3: The lone step is satisfied by landing completely within a single Read Cache Line - - step = readPlan[0] - - if 0 == step.LogSegmentNumber { - // Case 1: The lone step calls for a zero-filled []byte - buf = make([]byte, step.Length) - stats.IncrementOperationsAndBucketedBytes(stats.FileRead, step.Length) - err = nil - return - } - - fileInode.Lock() - - inFlightLogSegment, inFlightHit = fileInode.inFlightLogSegmentMap[step.LogSegmentNumber] - if inFlightHit { - // Case 2: The lone step is satisfied by reading from an inFlightLogSegment - openLogSegmentLRUTouch(inFlightLogSegment) - buf, err = inFlightLogSegment.Read(step.Offset, step.Length) - if nil != err { - fileInode.Unlock() - logger.ErrorfWithError(err, "Reading back inFlightLogSegment failed - optimal case") - err = blunder.AddError(err, blunder.SegReadError) - return - } - fileInode.Unlock() - stats.IncrementOperations(&stats.FileWritebackHitOps) - stats.IncrementOperationsAndBucketedBytes(stats.FileRead, step.Length) - return - } - - stats.IncrementOperations(&stats.FileWritebackMissOps) - - fileInode.Unlock() - - cacheLineHitOffset = step.Offset % readCacheLineSize - - if (cacheLineHitOffset + step.Length) <= readCacheLineSize { - // Case 3: The lone step is satisfied by landing completely within a single Read Cache Line - readCacheKey.logSegmentNumber = step.LogSegmentNumber - readCacheKey.cacheLineTag = step.Offset / readCacheLineSize - - volumeGroup.Lock() - - readCacheElement, readCacheHit = volumeGroup.readCache[readCacheKey] - - if readCacheHit { - volumeGroup.touchReadCacheElementWhileLocked(readCacheElement) - cacheLine = readCacheElement.cacheLine - volumeGroup.Unlock() - stats.IncrementOperations(&stats.FileReadcacheHitOps) - } else { - volumeGroup.Unlock() - stats.IncrementOperations(&stats.FileReadcacheMissOps) - // Make readCacheHit true (at MRU, likely kicking out LRU) - cacheLineStartOffset = readCacheKey.cacheLineTag * readCacheLineSize - cacheLine, err = swiftclient.ObjectGet(step.AccountName, step.ContainerName, step.ObjectName, cacheLineStartOffset, readCacheLineSize) - if nil != err { - logger.ErrorfWithError(err, "Reading from LogSegment object failed - optimal case") - err = blunder.AddError(err, blunder.SegReadError) - return - } - readCacheElement = &readCacheElementStruct{ - readCacheKey: readCacheKey, - next: nil, - prev: nil, - cacheLine: cacheLine, - } - volumeGroup.Lock() - volumeGroup.insertReadCacheElementWhileLocked(readCacheElement) - volumeGroup.Unlock() - } - - if (cacheLineHitOffset + step.Length) > uint64(len(cacheLine)) { - err = fmt.Errorf("Invalid range for LogSegment object - optimal case") - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.SegReadError) - return - } - - buf = cacheLine[cacheLineHitOffset:(cacheLineHitOffset + step.Length)] - - stats.IncrementOperationsAndBucketedBytes(stats.FileRead, step.Length) - - err = nil - return - } - } - - // If we reach here, normal readPlan processing will be performed... no zero-copy opportunity - - buf = make([]byte, 0, readPlanBytes) - - for stepIndex, step = range readPlan { - if 0 == step.LogSegmentNumber { - // The step calls for a zero-filled []byte - buf = append(buf, make([]byte, step.Length)...) - } else { - fileInode.Lock() - inFlightLogSegment, inFlightHit = fileInode.inFlightLogSegmentMap[step.LogSegmentNumber] - if inFlightHit { - // The step is satisfied by reading from an inFlightLogSegment - openLogSegmentLRUTouch(inFlightLogSegment) - inFlightHitBuf, err = inFlightLogSegment.Read(step.Offset, step.Length) - if nil != err { - fileInode.Unlock() - logger.ErrorfWithError(err, "Reading back inFlightLogSegment failed - general case") - err = blunder.AddError(err, blunder.SegReadError) - return - } - fileInode.Unlock() - buf = append(buf, inFlightHitBuf...) - stats.IncrementOperations(&stats.FileWritebackHitOps) - } else { - fileInode.Unlock() - if (0 == stepIndex) && (1 == len(readPlan)) { - // No need to increment stats.FileWritebackMissOps since it was incremented above - } else { - stats.IncrementOperations(&stats.FileWritebackMissOps) - } - } - if !inFlightHit { - // The step is satisfied by hitting or missing the Read Cache - readCacheKey.logSegmentNumber = step.LogSegmentNumber - chunkOffset = step.Offset - remainingLength = step.Length - for 0 < remainingLength { - readCacheKey.cacheLineTag = chunkOffset / readCacheLineSize - cacheLineHitOffset = chunkOffset % readCacheLineSize - if (cacheLineHitOffset + remainingLength) > readCacheLineSize { - // When we've got a cache hit, the read extends beyond the cache line - cacheLineHitLength = readCacheLineSize - cacheLineHitOffset - } else { - // When we've got a cache hit, all the data is inside the cache line - cacheLineHitLength = remainingLength - } - volumeGroup.Lock() - readCacheElement, readCacheHit = volumeGroup.readCache[readCacheKey] - if readCacheHit { - volumeGroup.touchReadCacheElementWhileLocked(readCacheElement) - cacheLine = readCacheElement.cacheLine - volumeGroup.Unlock() - stats.IncrementOperations(&stats.FileReadcacheHitOps) - } else { - volumeGroup.Unlock() - stats.IncrementOperations(&stats.FileReadcacheMissOps) - // Make readCacheHit true (at MRU, likely kicking out LRU) - cacheLineStartOffset = readCacheKey.cacheLineTag * readCacheLineSize - cacheLine, err = swiftclient.ObjectGet(step.AccountName, step.ContainerName, step.ObjectName, cacheLineStartOffset, readCacheLineSize) - if nil != err { - logger.ErrorfWithError(err, "Reading from LogSegment object failed - general case") - err = blunder.AddError(err, blunder.SegReadError) - return - } - readCacheElement = &readCacheElementStruct{ - readCacheKey: readCacheKey, - next: nil, - prev: nil, - cacheLine: cacheLine, - } - volumeGroup.Lock() - volumeGroup.insertReadCacheElementWhileLocked(readCacheElement) - volumeGroup.Unlock() - } - if (cacheLineHitOffset + cacheLineHitLength) > uint64(len(cacheLine)) { - err = fmt.Errorf("Invalid range for LogSegment object - general case") - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.SegReadError) - return - } - buf = append(buf, cacheLine[cacheLineHitOffset:(cacheLineHitOffset+cacheLineHitLength)]...) - chunkOffset += cacheLineHitLength - remainingLength -= cacheLineHitLength - } - } - } - } - - stats.IncrementOperationsAndBucketedBytes(stats.FileRead, uint64(len(buf))) - - err = nil - return -} - -func (vS *volumeStruct) doSendChunk(fileInode *inMemoryInodeStruct, buf []byte) (logSegmentNumber uint64, logSegmentOffset uint64, err error) { - var ( - inFlightLogSegment *inFlightLogSegmentStruct - openLogSegmentContainerName string - openLogSegmentObjectNumber uint64 - ) - - fileInode.Lock() - - if nil == fileInode.openLogSegment { - // Drop fileInode Lock while preparing an inFlightLogSegment. This is to avoid a deadlock where - // starvation for ChunkedPutContext's might need to grab this fileInode's Lock to check a previous - // openLogSegment associated with this fileInode (and, hence, when we looked was then on the - // openLogSegmentLRU). - - fileInode.Unlock() - - openLogSegmentContainerName, openLogSegmentObjectNumber, err = fileInode.volume.provisionObject() - if nil != err { - logger.ErrorfWithError(err, "Provisioning LogSegment failed") - return - } - - err = fileInode.volume.setLogSegmentContainer(openLogSegmentObjectNumber, openLogSegmentContainerName) - if nil != err { - logger.ErrorfWithError(err, "Recording LogSegment ContainerName failed") - return - } - - inFlightLogSegment = &inFlightLogSegmentStruct{ - logSegmentNumber: openLogSegmentObjectNumber, - fileInode: fileInode, - accountName: fileInode.volume.accountName, - containerName: openLogSegmentContainerName, - objectName: utils.Uint64ToHexStr(openLogSegmentObjectNumber), - } - - inFlightLogSegment.ChunkedPutContext, err = swiftclient.ObjectFetchChunkedPutContext(inFlightLogSegment.accountName, inFlightLogSegment.containerName, inFlightLogSegment.objectName, "") - if nil != err { - logger.ErrorfWithError(err, "Starting Chunked PUT to LogSegment failed") - return - } - - // Now reestablish the fileInode Lock before continuing - - fileInode.Lock() - - fileInode.inFlightLogSegmentMap[inFlightLogSegment.logSegmentNumber] = inFlightLogSegment - - fileInode.openLogSegment = inFlightLogSegment - openLogSegmentLRUInsert(inFlightLogSegment) - } else { - inFlightLogSegment = fileInode.openLogSegment - openLogSegmentLRUTouch(inFlightLogSegment) - } - - logSegmentNumber = inFlightLogSegment.logSegmentNumber - - logSegmentOffset, err = inFlightLogSegment.BytesPut() - if nil != err { - fileInode.Unlock() - logger.ErrorfWithError(err, "Failed to get current LogSegmentOffset") - return - } - - err = inFlightLogSegment.ChunkedPutContext.SendChunk(buf) - if nil != err { - fileInode.Unlock() - logger.ErrorfWithError(err, "Sending Chunked PUT chunk to LogSegment failed") - return - } - - if (logSegmentOffset + uint64(len(buf))) >= fileInode.volume.maxFlushSize { - fileInode.Add(1) - go vS.inFlightLogSegmentFlusher(inFlightLogSegment, true) - // No need to wait for it to complete now... that's only in doFileInodeDataFlush() - } - - fileInode.Unlock() - - err = nil - return -} - -func (vS *volumeStruct) doFileInodeDataFlush(fileInode *inMemoryInodeStruct) (err error) { - var ( - inFlightLogSegment *inFlightLogSegmentStruct - ) - - fileInode.Lock() - if nil != fileInode.openLogSegment { - inFlightLogSegment = fileInode.openLogSegment - fileInode.Add(1) - go vS.inFlightLogSegmentFlusher(inFlightLogSegment, true) - } - fileInode.Unlock() - - // Wait for all invocations of inFlightLogSegmentFlusher() for this fileInode have completed - - fileInode.Wait() - - // REVIEW TODO: Does anybody ever empty the errors map? Should they? Would this mask prior errors? - // File system could go "read only" if that's sufficient... - // Problem with write-back data... must discard it... - - if 0 == len(fileInode.inFlightLogSegmentErrors) { - err = nil - } else { - err = fmt.Errorf("Errors encountered while flushing inFlightLogSegments") - } - - return -} - -func (vS *volumeStruct) inFlightLogSegmentFlusher(inFlightLogSegment *inFlightLogSegmentStruct, doDone bool) { - var ( - err error - fileInode *inMemoryInodeStruct - ) - - // Handle the race between a DLM-serialized Flush triggering this versus the starvatation condition - // doing so... Either one will perform the appropriate steps to enable the Flush() to complete. - - fileInode = inFlightLogSegment.fileInode - - fileInode.Lock() - - if inFlightLogSegment != fileInode.openLogSegment { - // Either a Close() is already in progress or has already completed - - fileInode.Unlock() - if doDone { - fileInode.Done() - } - return - } - - // This, and inFlightLogSegment still being in fileInode.inFlightLogSegmentMap, - // means "a Close() is already in progress" - - fileInode.openLogSegment = nil - - // Terminate Chunked PUT while not holding fileInode.Lock - - fileInode.Unlock() - err = inFlightLogSegment.Close() - fileInode.Lock() - - // Finish up... recording error (if any) in the process - - if nil != err { - err = blunder.AddError(err, blunder.InodeFlushError) - fileInode.inFlightLogSegmentErrors[inFlightLogSegment.logSegmentNumber] = err - } - - delete(inFlightLogSegment.fileInode.inFlightLogSegmentMap, inFlightLogSegment.logSegmentNumber) - - openLogSegmentLRURemove(inFlightLogSegment) - - fileInode.Unlock() - - if doDone { - fileInode.Done() - } -} - -func chunkedPutConnectionPoolStarvationCallback() { - var ( - fileInode *inMemoryInodeStruct - inFlightLogSegment *inFlightLogSegmentStruct - volume *volumeStruct - ) - - globals.Lock() - - if 0 == globals.openLogSegmentLRUItems { - globals.Unlock() - return - } - - inFlightLogSegment = globals.openLogSegmentLRUHead - - fileInode = inFlightLogSegment.fileInode - volume = fileInode.volume - - globals.Unlock() - - // Call inFlightLogSegmentFlusher() synchronously because we only want to return when it completes - // and we don't want to call fileInode.Wait() as this would wait until all invocations of - // inFlightLogSegmentFlusher() for the fileInode have completed. - - volume.inFlightLogSegmentFlusher(inFlightLogSegment, false) -} diff --git a/inode/file_stress_test.go b/inode/file_stress_test.go deleted file mode 100644 index aa6045bf..00000000 --- a/inode/file_stress_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "crypto/rand" - "sync" - "testing" -) - -type testStressFileWritesGlobalsStruct struct { - testStressNumWorkers uint64 - testStressNumFlushesPerFile uint64 - testStressNumBlocksPerFlush uint64 - testStressNumBytesPerBlock uint64 - volumeHandle VolumeHandle - writeBlock []byte - childrenStart sync.WaitGroup - childrenDone sync.WaitGroup -} - -var testStressFileWritesGlobals testStressFileWritesGlobalsStruct - -func TestStressFileWritesWhileStarved(t *testing.T) { - testStressFileWritesGlobals.testStressNumWorkers = 100 - testStressFileWritesGlobals.testStressNumFlushesPerFile = 4 // Currently, each triggers a checkpoint as well - testStressFileWritesGlobals.testStressNumBlocksPerFlush = 20 - testStressFileWritesGlobals.testStressNumBytesPerBlock = 1 - - testStressFileWrites(t, true) -} - -func TestStressFileWritesWhileNotStarved(t *testing.T) { - testStressFileWritesGlobals.testStressNumWorkers = 100 - testStressFileWritesGlobals.testStressNumFlushesPerFile = 10 // Currently, each triggers a checkpoint as well - testStressFileWritesGlobals.testStressNumBlocksPerFlush = 20 - testStressFileWritesGlobals.testStressNumBytesPerBlock = 1 - - testStressFileWrites(t, false) -} - -func testStressFileWrites(t *testing.T, starvationMode bool) { - var ( - err error - workerIndex uint64 - ) - - testSetup(t, starvationMode) - - testStressFileWritesGlobals.volumeHandle, err = FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") failed: %v", err) - } - - testStressFileWritesGlobals.writeBlock = make([]byte, testStressFileWritesGlobals.testStressNumBytesPerBlock) - rand.Read(testStressFileWritesGlobals.writeBlock) - - testStressFileWritesGlobals.childrenStart.Add(1) - testStressFileWritesGlobals.childrenDone.Add(int(testStressFileWritesGlobals.testStressNumWorkers)) - - for workerIndex = 0; workerIndex < testStressFileWritesGlobals.testStressNumWorkers; workerIndex++ { - go testStressFileWritesWorker(t, workerIndex) - } - - testStressFileWritesGlobals.childrenStart.Done() - - testStressFileWritesGlobals.childrenDone.Wait() - - testTeardown(t) -} - -func testStressFileWritesWorker(t *testing.T, workerIndex uint64) { - var ( - err error - fileBlockNumber uint64 - fileFlushNumber uint64 - fileInodeNumber InodeNumber - ) - - fileInodeNumber, err = testStressFileWritesGlobals.volumeHandle.CreateFile(InodeMode(0000), InodeRootUserID, InodeGroupID(0)) - if nil != err { - t.Fatalf("testStressFileWritesGlobals.volumeHandle.CreateFile() failed: %v", err) - } - - testStressFileWritesGlobals.childrenStart.Wait() - - for fileFlushNumber = 0; fileFlushNumber < testStressFileWritesGlobals.testStressNumFlushesPerFile; fileFlushNumber++ { - for fileBlockNumber = 0; fileBlockNumber < testStressFileWritesGlobals.testStressNumBlocksPerFlush; fileBlockNumber++ { - err = testStressFileWritesGlobals.volumeHandle.Write( - fileInodeNumber, - fileBlockNumber*testStressFileWritesGlobals.testStressNumBytesPerBlock, - testStressFileWritesGlobals.writeBlock, - nil) - if nil != err { - t.Fatalf("Write() failed: %v", err) - } - } - - err = testStressFileWritesGlobals.volumeHandle.Flush(fileInodeNumber, false) - if nil != err { - t.Fatalf("Flush() failed: %v", err) - } - } - - err = testStressFileWritesGlobals.volumeHandle.Flush(fileInodeNumber, true) - if nil != err { - t.Fatalf("Flush() failed: %v", err) - } - - err = testStressFileWritesGlobals.volumeHandle.Destroy(fileInodeNumber) - if nil != err { - t.Fatalf("Destroy() failed: %v", err) - } - - testStressFileWritesGlobals.childrenDone.Done() -} diff --git a/inode/gc_test.go b/inode/gc_test.go deleted file mode 100644 index 8cc68afc..00000000 --- a/inode/gc_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "testing" - "time" - - "github.com/NVIDIA/proxyfs/swiftclient" -) - -type testObjectLocationStruct struct { - accountName string - containerName string - objectName string -} - -func TestEmptySegmentDeletion(t *testing.T) { - testSetup(t, false) - - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") failed: %v", err) - } - - volume := testVolumeHandle.(*volumeStruct) - - ino, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if nil != err { - t.Fatalf("CreateFile() failed: %v", err) - } - - // '$$$$$$$$$$$$$$$$' - ourBytes := []byte{0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24, 0x24} - - // repeatedly write and flush to create some log segments - for i := 0; i < 5; i++ { - err := testVolumeHandle.Write(ino, uint64(i*16), ourBytes, nil) - if err != nil { - t.Fatalf("failed to write: %v", err) - } - - err = testVolumeHandle.Flush(ino, false) - if err != nil { - t.Fatalf("failed to flush: %v", err) - } - } - - // inspect log segment map - ourInode, ok, err := volume.fetchInode(ino) - if nil != err { - t.Fatalf("fetchInode(ino==0x%016X) failed: %v", ino, err) - } - if !ok { - t.Fatalf("fetchInode(ino==0x%016X) returned !ok", ino) - } - segmentNumbers := make([]uint64, 0, 5) - segmentObjectLocations := make([]testObjectLocationStruct, 0, 5) - for segmentNumber := range ourInode.LogSegmentMap { - segmentNumbers = append(segmentNumbers, segmentNumber) - containerName, objectName, _, getObjectLocationErr := volume.getObjectLocationFromLogSegmentNumber(segmentNumber) - if nil != getObjectLocationErr { - t.Fatalf("expected to be able to get log segment 0x%016X", segmentNumber) - } - segmentObjectLocations = append(segmentObjectLocations, testObjectLocationStruct{volume.accountName, containerName, objectName}) - } - - // overwrite it - err = testVolumeHandle.Write(ino, 0, make([]byte, 16*5), nil) - if err != nil { - t.Fatalf("expected to be able to write to inode #%v", ino) - } - err = testVolumeHandle.Flush(ino, false) - if err != nil { - t.Fatalf("expected to be able to flush inode #%v", ino) - } - - // wait for headhunter checkpoint which blocks deletes - err = volume.headhunterVolumeHandle.DoCheckpoint() - if nil != err { - t.Errorf("expected volume.headhunterVolumeHandle.DoCheckpoint() to succeed: got %v", err) - } - - // the deletions are async; give them a (literal) second - time.Sleep(1 * time.Second) - - for _, deletedSegmentNumber := range segmentNumbers { - _, getLogSegmentContainerErr := volume.getLogSegmentContainer(deletedSegmentNumber) - if getLogSegmentContainerErr == nil { - t.Errorf("expected volume.getLogSegmentContainer() to fail for allegedly-deleted log segment 0x%016X", deletedSegmentNumber) - } - } - - // the headhunter records got deleted, and ... - for _, segmentObjectLocation := range segmentObjectLocations { - var objectContentLengthErr error - - // wait for up to 20 sec for the object to be (async) deleted - for try := 0; try < 20; try++ { - _, objectContentLengthErr = swiftclient.ObjectContentLength(segmentObjectLocation.accountName, segmentObjectLocation.containerName, segmentObjectLocation.objectName) - //t.Logf("verifying object delete for %v/%v try %d err '%v'\n", segmentObjectLocation.containerName, segmentObjectLocation.objectName, try, objectContentLengthErr) - if objectContentLengthErr != nil { - break - } - time.Sleep(time.Second) - } - if objectContentLengthErr == nil { - t.Errorf("expected ObjectContentLength() to fail for allegedly-deleted log segment object at %s/%s/%s", segmentObjectLocation.accountName, segmentObjectLocation.containerName, segmentObjectLocation.objectName) - } - } - // the underlying log segment objects got deleted!! Yaaay~! ☆✦❤ - - testTeardown(t) -} diff --git a/inode/inode.go b/inode/inode.go deleted file mode 100644 index abfd5fff..00000000 --- a/inode/inode.go +++ /dev/null @@ -1,2439 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "container/list" - "encoding/json" - "fmt" - "runtime/debug" - "strings" - "sync" - "time" - - "github.com/NVIDIA/cstruct" - "github.com/NVIDIA/sortedmap" - "github.com/ansel1/merry" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/dlm" - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/halter" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/utils" -) - -// Shorthand for inode internal API debug log id; global to the package -var int_inode_debug = logger.DbgInodeInternal - -const ( - optimisticInodeFetchBytes = 2048 -) - -type CorruptionDetected bool -type Version uint64 - -const ( - V1 Version = iota + 1 // use type/struct onDiskInodeV1Struct - onDiskInodeV1PayloadObjectOffset uint64 = 0 -) - -type onDiskInodeV1Struct struct { // Preceded "on disk" by CorruptionDetected then Version both in cstruct.LittleEndian form - InodeNumber - InodeType - LinkCount uint64 - Size uint64 - CreationTime time.Time - ModificationTime time.Time - AccessTime time.Time - AttrChangeTime time.Time - NumWrites uint64 - Mode InodeMode - UserID InodeUserID - GroupID InodeGroupID - StreamMap map[string][]byte - PayloadObjectNumber uint64 // DirInode: B+Tree Root with Key == dir_entry_name, Value = InodeNumber - PayloadObjectLength uint64 // FileInode: B+Tree Root with Key == fileOffset, Value = fileExtent - SymlinkTarget string // SymlinkInode: target path of symbolic link - LogSegmentMap map[uint64]uint64 // FileInode: Key == LogSegment#, Value = file user data byte count -} - -type inFlightLogSegmentStruct struct { // Used as (by reference) Value for inMemoryInodeStruct.inFlightLogSegmentMap - logSegmentNumber uint64 // Used as (by value) Key for inMemoryInodeStruct.inFlightLogSegmentMap - openLogSegmentLRUNext *inFlightLogSegmentStruct - openLogSegmentLRUPrev *inFlightLogSegmentStruct - fileInode *inMemoryInodeStruct - accountName string - containerName string - objectName string - openLogSegmentListElement list.Element - swiftclient.ChunkedPutContext -} - -type inMemoryInodeStruct struct { - trackedlock.Mutex // Used to synchronize with background fileInodeFlusherDaemon - sync.WaitGroup // FileInode Flush requests wait on this - inodeCacheLRUNext *inMemoryInodeStruct - inodeCacheLRUPrev *inMemoryInodeStruct - dirty bool - volume *volumeStruct - snapShotID uint64 - payload interface{} // DirInode: B+Tree with Key == dir_entry_name, Value = InodeNumber - // FileInode: B+Tree with Key == fileOffset, Value = *fileExtent - openLogSegment *inFlightLogSegmentStruct // FileInode only... also in inFlightLogSegmentMap - inFlightLogSegmentMap map[uint64]*inFlightLogSegmentStruct // FileInode: key == logSegmentNumber - inFlightLogSegmentErrors map[uint64]error // FileInode: key == logSegmentNumber; value == err (if non nil) - onDiskInodeV1Struct // Real on-disk inode information embedded here -} - -func (vS *volumeStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsInodeNumber, ok := key.(InodeNumber) - if !ok { - err = fmt.Errorf("inode.volumeStruct.DumpKey() could not parse key as a InodeNumber") - return - } - - keyAsString = fmt.Sprintf("0x%016X", keyAsInodeNumber) - - err = nil - return -} - -func (vS *volumeStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - valueAsInMemoryInodeStructPtr, ok := value.(*inMemoryInodeStruct) - if !ok { - err = fmt.Errorf("inode.volumeStruct.DumpValue() could not parse value as a *inMemoryInodeStruct") - return - } - - valueAsString = fmt.Sprintf("%016p", valueAsInMemoryInodeStructPtr) - - err = nil - return -} - -func compareInodeNumber(key1 sortedmap.Key, key2 sortedmap.Key) (result int, err error) { - key1InodeNumber, ok := key1.(InodeNumber) - if !ok { - err = fmt.Errorf("compareInodeNumber(non-InodeNumber,) not supported") - return - } - key2InodeNumber, ok := key2.(InodeNumber) - if !ok { - err = fmt.Errorf("compareInodeNumber(InodeNumber, non-InodeNumber) not supported") - return - } - - if key1InodeNumber < key2InodeNumber { - result = -1 - } else if key1InodeNumber == key2InodeNumber { - result = 0 - } else { // key1InodeNumber > key2InodeNumber - result = 1 - } - - err = nil - - return -} - -func setRWMode(rwMode RWModeType) (err error) { - if rwMode != globals.rwMode { - switch rwMode { - case RWModeNormal: - stats.IncrementOperations(&stats.ReconCheckTriggeredNormalMode) - case RWModeNoWrite: - stats.IncrementOperations(&stats.ReconCheckTriggeredNoWriteMode) - case RWModeReadOnly: - stats.IncrementOperations(&stats.ReconCheckTriggeredReadOnlyMode) - default: - err = fmt.Errorf("SetRWMode(rwMode==%d) not allowed... must be one of RWModeNormal(%d), RWModeNoWrite(%d), or RWModeReadOnly(%d)", rwMode, RWModeNormal, RWModeNoWrite, RWModeReadOnly) - return - } - - globals.rwMode = rwMode - } - - err = nil - return -} - -func enforceRWMode(enforceNoWriteMode bool) (err error) { - var ( - rwModeCopy RWModeType - ) - - rwModeCopy = globals.rwMode - - if rwModeCopy == RWModeReadOnly { - err = blunder.NewError(globals.readOnlyThresholdErrno, globals.readOnlyThresholdErrnoString) - } else if enforceNoWriteMode && (rwModeCopy == RWModeNoWrite) { - err = blunder.NewError(globals.noWriteThresholdErrno, globals.noWriteThresholdErrnoString) - } else { - err = nil - } - - return -} - -func (vS *volumeStruct) FetchOnDiskInode(inodeNumber InodeNumber) (corruptionDetected CorruptionDetected, version Version, onDiskInode []byte, err error) { - var ( - bytesConsumedByCorruptionDetected uint64 - bytesConsumedByVersion uint64 - inodeRec []byte - ok bool - ) - - corruptionDetected = CorruptionDetected(false) - version = Version(0) - onDiskInode = make([]byte, 0) - - inodeRec, ok, err = vS.headhunterVolumeHandle.GetInodeRec(uint64(inodeNumber)) - if nil != err { - err = fmt.Errorf("headhunter.GetInodeRec() failed: %v", err) - return - } - if !ok { - err = fmt.Errorf("headhunter.GetInodeRec() returned !ok") - return - } - - bytesConsumedByCorruptionDetected, err = cstruct.Unpack(inodeRec, &corruptionDetected, cstruct.LittleEndian) - if nil != err { - err = fmt.Errorf("cstruct.Unpack(,&corruptionDetected,) failed: %v", err) - return - } - if corruptionDetected { - return - } - - bytesConsumedByVersion, err = cstruct.Unpack(inodeRec[bytesConsumedByCorruptionDetected:], &version, cstruct.LittleEndian) - if nil != err { - err = fmt.Errorf("cstruct.Unpack(,&version,) failed: %v", err) - return - } - - onDiskInode = inodeRec[bytesConsumedByCorruptionDetected+bytesConsumedByVersion:] - - return -} - -func (vS *volumeStruct) fetchOnDiskInode(inodeNumber InodeNumber) (inMemoryInode *inMemoryInodeStruct, ok bool, err error) { - var ( - bytesConsumedByCorruptionDetected uint64 - bytesConsumedByVersion uint64 - corruptionDetected CorruptionDetected - inodeRec []byte - onDiskInodeV1 *onDiskInodeV1Struct - snapShotID uint64 - snapShotIDType headhunter.SnapShotIDType - version Version - ) - - snapShotIDType, snapShotID, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - logger.Fatalf("fetchOnDiskInode for headhunter.SnapShotIDTypeDotSnapShot not allowed") - } - - inodeRec, ok, err = vS.headhunterVolumeHandle.GetInodeRec(uint64(inodeNumber)) - if nil != err { - stackStr := string(debug.Stack()) - err = fmt.Errorf("%s: unable to get inodeRec for inode %d: %v stack: %s", - utils.GetFnName(), inodeNumber, err, stackStr) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - if !ok { - return - } - - bytesConsumedByCorruptionDetected, err = cstruct.Unpack(inodeRec, &corruptionDetected, cstruct.LittleEndian) - if nil != err { - err = fmt.Errorf("%s: unable to parse inodeRec.CorruptionDetected for inode %d: %v", utils.GetFnName(), inodeNumber, err) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - if corruptionDetected { - err = fmt.Errorf("%s: inode %d has been marked corrupted", utils.GetFnName(), inodeNumber) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - - bytesConsumedByVersion, err = cstruct.Unpack(inodeRec[bytesConsumedByCorruptionDetected:], &version, cstruct.LittleEndian) - if nil != err { - err = fmt.Errorf("%s: unable to get inodeRec.Version for inode %d: %v", utils.GetFnName(), inodeNumber, err) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - if V1 != version { - err = fmt.Errorf("%s: inodeRec.Version for inode %d (%v) not supported", utils.GetFnName(), inodeNumber, version) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - - onDiskInodeV1 = &onDiskInodeV1Struct{StreamMap: make(map[string][]byte)} - - err = json.Unmarshal(inodeRec[bytesConsumedByCorruptionDetected+bytesConsumedByVersion:], onDiskInodeV1) - if nil != err { - err = fmt.Errorf("%s: inodeRec. for inode %d json.Unmarshal() failed: %v", utils.GetFnName(), inodeNumber, err) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - - inMemoryInode = &inMemoryInodeStruct{ - inodeCacheLRUNext: nil, - inodeCacheLRUPrev: nil, - dirty: false, - volume: vS, - snapShotID: snapShotID, - openLogSegment: nil, - inFlightLogSegmentMap: make(map[uint64]*inFlightLogSegmentStruct), - inFlightLogSegmentErrors: make(map[uint64]error), - onDiskInodeV1Struct: *onDiskInodeV1, - } - - inMemoryInode.onDiskInodeV1Struct.InodeNumber = inodeNumber - - switch inMemoryInode.InodeType { - case DirType: - if 0 == inMemoryInode.PayloadObjectNumber { - inMemoryInode.payload = - sortedmap.NewBPlusTree( - vS.maxEntriesPerDirNode, - sortedmap.CompareString, - &dirInodeCallbacks{treeNodeLoadable{inode: inMemoryInode}}, - globals.dirEntryCache) - } else { - inMemoryInode.payload, err = - sortedmap.OldBPlusTree( - inMemoryInode.PayloadObjectNumber, - onDiskInodeV1PayloadObjectOffset, - inMemoryInode.PayloadObjectLength, - sortedmap.CompareString, - &dirInodeCallbacks{treeNodeLoadable{inode: inMemoryInode}}, - globals.dirEntryCache) - if nil != err { - err = fmt.Errorf("%s: sortedmap.OldBPlusTree(inodeRec..PayloadObjectNumber) for DirType inode %d failed: %v", utils.GetFnName(), inodeNumber, err) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - } - case FileType: - if 0 == inMemoryInode.PayloadObjectNumber { - inMemoryInode.payload = - sortedmap.NewBPlusTree( - vS.maxExtentsPerFileNode, - sortedmap.CompareUint64, - &fileInodeCallbacks{treeNodeLoadable{inode: inMemoryInode}}, - globals.fileExtentMapCache) - } else { - inMemoryInode.payload, err = - sortedmap.OldBPlusTree( - inMemoryInode.PayloadObjectNumber, - onDiskInodeV1PayloadObjectOffset, - inMemoryInode.PayloadObjectLength, - sortedmap.CompareUint64, - &fileInodeCallbacks{treeNodeLoadable{inode: inMemoryInode}}, - globals.fileExtentMapCache) - if nil != err { - err = fmt.Errorf("%s: sortedmap.OldBPlusTree(inodeRec..PayloadObjectNumber) for FileType inode %d failed: %v", utils.GetFnName(), inodeNumber, err) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - } - case SymlinkType: - // Nothing special here - default: - err = fmt.Errorf("%s: inodeRec.InodeType for inode %d (%v) not supported", utils.GetFnName(), inodeNumber, inMemoryInode.InodeType) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - - err = nil - return -} - -func (vS *volumeStruct) inodeCacheFetchWhileLocked(inodeNumber InodeNumber) (inode *inMemoryInodeStruct, ok bool, err error) { - var ( - inodeAsValue sortedmap.Value - ) - - inodeAsValue, ok, err = vS.inodeCache.GetByKey(inodeNumber) - if nil != err { - return - } - - if ok { - inode, ok = inodeAsValue.(*inMemoryInodeStruct) - if ok { - vS.inodeCacheTouchWhileLocked(inode) - err = nil - } else { - ok = false - err = fmt.Errorf("inodeCache[inodeNumber==0x%016X] contains a value not mappable to a *inMemoryInodeStruct", inodeNumber) - } - } - - return -} - -func (vS *volumeStruct) inodeCacheFetch(inodeNumber InodeNumber) (inode *inMemoryInodeStruct, ok bool, err error) { - vS.Lock() - inode, ok, err = vS.inodeCacheFetchWhileLocked(inodeNumber) - vS.Unlock() - return -} - -func (vS *volumeStruct) inodeCacheInsertWhileLocked(inode *inMemoryInodeStruct) (ok bool, err error) { - ok, err = vS.inodeCache.Put(inode.InodeNumber, inode) - if (nil != err) || !ok { - return - } - - // Place inode at the MRU end of inodeCacheLRU - - if 0 == vS.inodeCacheLRUItems { - vS.inodeCacheLRUHead = inode - vS.inodeCacheLRUTail = inode - vS.inodeCacheLRUItems = 1 - } else { - inode.inodeCacheLRUPrev = vS.inodeCacheLRUTail - inode.inodeCacheLRUPrev.inodeCacheLRUNext = inode - - vS.inodeCacheLRUTail = inode - vS.inodeCacheLRUItems++ - } - - return -} - -func (vS *volumeStruct) inodeCacheInsert(inode *inMemoryInodeStruct) (ok bool, err error) { - vS.Lock() - ok, err = vS.inodeCacheInsertWhileLocked(inode) - vS.Unlock() - return -} - -func (vS *volumeStruct) inodeCacheTouchWhileLocked(inode *inMemoryInodeStruct) { - // Move inode to the MRU end of inodeCacheLRU - - if inode != vS.inodeCacheLRUTail { - if inode == vS.inodeCacheLRUHead { - vS.inodeCacheLRUHead = inode.inodeCacheLRUNext - vS.inodeCacheLRUHead.inodeCacheLRUPrev = nil - - inode.inodeCacheLRUPrev = vS.inodeCacheLRUTail - inode.inodeCacheLRUNext = nil - - vS.inodeCacheLRUTail.inodeCacheLRUNext = inode - vS.inodeCacheLRUTail = inode - } else { - inode.inodeCacheLRUPrev.inodeCacheLRUNext = inode.inodeCacheLRUNext - inode.inodeCacheLRUNext.inodeCacheLRUPrev = inode.inodeCacheLRUPrev - - inode.inodeCacheLRUNext = nil - inode.inodeCacheLRUPrev = vS.inodeCacheLRUTail - - vS.inodeCacheLRUTail.inodeCacheLRUNext = inode - vS.inodeCacheLRUTail = inode - } - } -} - -func (vS *volumeStruct) inodeCacheTouch(inode *inMemoryInodeStruct) { - vS.Lock() - vS.inodeCacheTouchWhileLocked(inode) - vS.Unlock() -} - -// The inode cache discard thread calls this routine when the ticker goes off. -func (vS *volumeStruct) inodeCacheDiscard() (discarded uint64, dirty uint64, locked uint64, lruItems uint64) { - inodesToDrop := uint64(0) - - vS.Lock() - - if (vS.inodeCacheLRUItems * globals.inodeSize) > vS.inodeCacheLRUMaxBytes { - // Check, at most, 1.25 * (minimum_number_to_drop) - inodesToDrop = (vS.inodeCacheLRUItems * globals.inodeSize) - vS.inodeCacheLRUMaxBytes - inodesToDrop = inodesToDrop / globals.inodeSize - inodesToDrop += inodesToDrop / 4 - for (inodesToDrop > 0) && ((vS.inodeCacheLRUItems * globals.inodeSize) > vS.inodeCacheLRUMaxBytes) { - inodesToDrop-- - - ic := vS.inodeCacheLRUHead - - // Create a DLM lock object - id := dlm.GenerateCallerID() - inodeRWLock, _ := vS.InitInodeLock(ic.InodeNumber, id) - err := inodeRWLock.TryWriteLock() - - // Inode is locked; skip it - if err != nil { - // Move inode to tail of LRU - vS.inodeCacheTouchWhileLocked(ic) - locked++ - continue - } - - if ic.dirty { - // The inode is busy - drop the DLM lock and move to tail - inodeRWLock.Unlock() - dirty++ - vS.inodeCacheTouchWhileLocked(ic) - continue - } - - var ok bool - - discarded++ - ok, err = vS.inodeCacheDropWhileLocked(ic) - if err != nil || !ok { - pStr := fmt.Errorf("The inodes was not found in the inode cache - ok: %v err: %v", ok, err) - panic(pStr) - } - - inodeRWLock.Unlock() - - // NOTE: vS.inodeCacheDropWhileLocked() removed the inode from the LRU list so - // the head is now different - } - } - lruItems = vS.inodeCacheLRUItems - vS.Unlock() - //logger.Infof("discard: %v dirty: %v locked: %v LRUitems: %v", discarded, dirty, locked, lruItems) - return -} - -func (vS *volumeStruct) inodeCacheDropWhileLocked(inode *inMemoryInodeStruct) (ok bool, err error) { - ok, err = vS.inodeCache.DeleteByKey(inode.InodeNumber) - if (nil != err) || !ok { - return - } - - if inode == vS.inodeCacheLRUHead { - if inode == vS.inodeCacheLRUTail { - vS.inodeCacheLRUHead = nil - vS.inodeCacheLRUTail = nil - vS.inodeCacheLRUItems = 0 - } else { - vS.inodeCacheLRUHead = inode.inodeCacheLRUNext - vS.inodeCacheLRUHead.inodeCacheLRUPrev = nil - vS.inodeCacheLRUItems-- - - inode.inodeCacheLRUNext = nil - } - } else { - if inode == vS.inodeCacheLRUTail { - vS.inodeCacheLRUTail = inode.inodeCacheLRUPrev - vS.inodeCacheLRUTail.inodeCacheLRUNext = nil - vS.inodeCacheLRUItems-- - - inode.inodeCacheLRUPrev = nil - } else { - inode.inodeCacheLRUPrev.inodeCacheLRUNext = inode.inodeCacheLRUNext - inode.inodeCacheLRUNext.inodeCacheLRUPrev = inode.inodeCacheLRUPrev - vS.inodeCacheLRUItems-- - - inode.inodeCacheLRUNext = nil - inode.inodeCacheLRUPrev = nil - } - } - - return -} - -func (vS *volumeStruct) inodeCacheDrop(inode *inMemoryInodeStruct) (ok bool, err error) { - vS.Lock() - ok, err = vS.inodeCacheDropWhileLocked(inode) - vS.Unlock() - return -} - -func (vS *volumeStruct) fetchInode(inodeNumber InodeNumber) (inode *inMemoryInodeStruct, ok bool, err error) { - for { - inode, ok, err = vS.inodeCacheFetch(inodeNumber) - if nil != err { - return - } - - if ok { - return - } - - inode, ok, err = vS.fetchOnDiskInode(inodeNumber) - if nil != err { - return - } - if !ok { - err = fmt.Errorf("%s.fetchInode(0x%016X) not found", vS.volumeName, inodeNumber) - return - } - - ok, err = vS.inodeCacheInsert(inode) - if nil != err { - return - } - - if ok { - return - } - - // If we reach here, somebody beat us to it... just restart the fetch... - } -} - -// Fetch inode with inode type checking -func (vS *volumeStruct) fetchInodeType(inodeNumber InodeNumber, expectedType InodeType) (inode *inMemoryInodeStruct, err error) { - inode, ok, err := vS.fetchInode(inodeNumber) - if nil != err { - return - } - if !ok { - err = fmt.Errorf("%s: expected inode %d volume '%s' to be type %v, but it was unallocated", - utils.GetFnName(), inode.InodeNumber, vS.volumeName, expectedType) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - if inode.InodeType == expectedType { - // success - return - } - - err = fmt.Errorf("%s: expected inode %d volume '%s' to be type %v, got %v", - utils.GetFnName(), inode.InodeNumber, vS.volumeName, expectedType, inode.InodeType) - - var errVal blunder.FsError - switch expectedType { - case DirType: - errVal = blunder.NotDirError - case FileType: - errVal = blunder.NotFileError - case SymlinkType: - errVal = blunder.NotSymlinkError - default: - panic(fmt.Sprintf("unknown inode type=%v!", expectedType)) - } - err = blunder.AddError(err, errVal) - - return -} - -func (vS *volumeStruct) makeInMemoryInodeWithThisInodeNumber(inodeType InodeType, fileMode InodeMode, userID InodeUserID, groupID InodeGroupID, inodeNumber InodeNumber, volumeLocked bool) (inMemoryInode *inMemoryInodeStruct) { - var ( - birthTime time.Time - nonce uint64 - snapShotID uint64 - snapShotIDType headhunter.SnapShotIDType - ) - - snapShotIDType, snapShotID, nonce = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - logger.Fatalf("makeInMemoryInodeWithThisInodeNumber for headhunter.SnapShotIDTypeDotSnapShot not allowed") - } - - birthTime = time.Now() - - inMemoryInode = &inMemoryInodeStruct{ - inodeCacheLRUNext: nil, - inodeCacheLRUPrev: nil, - dirty: true, - volume: vS, - snapShotID: snapShotID, - openLogSegment: nil, - inFlightLogSegmentMap: make(map[uint64]*inFlightLogSegmentStruct), - inFlightLogSegmentErrors: make(map[uint64]error), - onDiskInodeV1Struct: onDiskInodeV1Struct{ - InodeNumber: InodeNumber(nonce), - InodeType: inodeType, - CreationTime: birthTime, - ModificationTime: birthTime, - AccessTime: birthTime, - AttrChangeTime: birthTime, - NumWrites: 0, - Mode: fileMode, - UserID: userID, - GroupID: groupID, - StreamMap: make(map[string][]byte), - LogSegmentMap: make(map[uint64]uint64), - }, - } - - return -} - -func (vS *volumeStruct) makeInMemoryInode(inodeType InodeType, fileMode InodeMode, userID InodeUserID, groupID InodeGroupID) (inMemoryInode *inMemoryInodeStruct, err error) { - inodeNumberAsUint64 := vS.headhunterVolumeHandle.FetchNonce() - - inMemoryInode = vS.makeInMemoryInodeWithThisInodeNumber(inodeType, fileMode, userID, groupID, InodeNumber(inodeNumberAsUint64), false) - - return -} - -func (vS *volumeStruct) PatchInode(inodeNumber InodeNumber, inodeType InodeType, linkCount uint64, mode InodeMode, userID InodeUserID, groupID InodeGroupID, parentInodeNumber InodeNumber, symlinkTarget string) (err error) { - var ( - callerID dlm.CallerID - inode *inMemoryInodeStruct - inodeNumberDecodedAsInodeNumber InodeNumber - inodeNumberDecodedAsUint64 uint64 - inodeRWLock *dlm.RWLockStruct - modeAdornedWithInodeType InodeMode - ok bool - parentInodeNumberDecodedAsInodeNumber InodeNumber - parentInodeNumberDecodedAsUint64 uint64 - payload sortedmap.BPlusTree - snapShotIDType headhunter.SnapShotIDType - ) - - snapShotIDType, _, inodeNumberDecodedAsUint64 = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,,,,,,,) must provide a non-SnapShot inodeNumber", inodeNumber) - return - } - inodeNumberDecodedAsInodeNumber = InodeNumber(inodeNumberDecodedAsUint64) - - switch inodeType { - case DirType: - if 2 != linkCount { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,linkCount==%v,,,,,) must set linkCount to 2", inodeNumber, linkCount) - return - } - if InodeNumber(0) == parentInodeNumber { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,,,,,parentInodeNumber==0,) must provide a non-zero parentInodeNumber", inodeNumber) - return - } - if (RootDirInodeNumber == inodeNumber) && (RootDirInodeNumber != parentInodeNumber) { - err = fmt.Errorf("PatchInode(inodeNumber==RootDirInodeNumber,inodeType==DirType,,,,,parentInodeNumber==0x%016X,) must provide RootDirInode's parent as also RootDirInodeNumber", parentInodeNumber) - return - } - snapShotIDType, _, parentInodeNumberDecodedAsUint64 = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,,,,,parentInodeNumber==0x%016X,) must provide a non-SnapShot parentInodeNumber", inodeNumber, parentInodeNumber) - return - } - parentInodeNumberDecodedAsInodeNumber = InodeNumber(parentInodeNumberDecodedAsUint64) - case FileType: - if 0 == linkCount { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==FileType,linkCount==0,,,,,) must provide a non-zero linkCount", inodeNumber) - return - } - case SymlinkType: - if 0 == linkCount { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==SymlinkType,linkCount==0,,,,,) must provide a non-zero linkCount", inodeNumber) - return - } - if "" == symlinkTarget { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==SymlinkType,,,,,,symlinkTarget==\"\") must provide a non-empty symlinkTarget", inodeNumber) - return - } - default: - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==%v,,,,,,) must provide a inodeType of DirType(%v), FileType(%v), or SymlinkType(%v)", inodeNumber, inodeType, DirType, FileType, SymlinkType) - return - } - - modeAdornedWithInodeType, err = determineMode(mode, inodeType) - if nil != err { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==%v,,mode==0o%011o,,,,) failed: %v", inodeNumber, inodeType, mode, err) - return - } - - vS.Lock() - - callerID = dlm.GenerateCallerID() - inodeRWLock, _ = vS.InitInodeLock(inodeNumber, callerID) - err = inodeRWLock.TryWriteLock() - if nil != err { - vS.Unlock() - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,,,,,,,) couldn't create a *dlm.RWLockStruct: %v", inodeNumber, err) - return - } - - inode, ok, err = vS.inodeCacheFetchWhileLocked(inodeNumber) - if nil != err { - _ = inodeRWLock.Unlock() - vS.Unlock() - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,,,,,,,) couldn't search inodeCache for pre-existing inode: %v", inodeNumber, err) - return - } - if ok { - if inode.dirty { - _ = inodeRWLock.Unlock() - vS.Unlock() - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,,,,,,,) of dirty Inode is not allowed", inodeNumber) - return - } - ok, err = vS.inodeCacheDropWhileLocked(inode) - if nil != err { - _ = inodeRWLock.Unlock() - vS.Unlock() - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,,,,,,,) drop of pre-existing inode from inodeCache failed: %v", inodeNumber, err) - return - } - if !ok { - _ = inodeRWLock.Unlock() - vS.Unlock() - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,,,,,,,) drop of pre-existing inode from inodeCache returned !ok", inodeNumber) - return - } - } - - inode = vS.makeInMemoryInodeWithThisInodeNumber(inodeType, modeAdornedWithInodeType, userID, groupID, inodeNumberDecodedAsInodeNumber, true) - - inode.dirty = true - - inode.onDiskInodeV1Struct.LinkCount = linkCount - - switch inodeType { - case DirType: - payload = sortedmap.NewBPlusTree( - vS.maxEntriesPerDirNode, - sortedmap.CompareString, - &dirInodeCallbacks{treeNodeLoadable{inode: inode}}, - globals.dirEntryCache) - - ok, err = payload.Put(".", inodeNumberDecodedAsInodeNumber) - if nil != err { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,,,,,,) failed to insert \".\" dirEntry: %v", inodeNumber, err) - panic(err) - } - if !ok { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,,,,,,) insert \".\" dirEntry got a !ok", inodeNumber) - panic(err) - } - - ok, err = payload.Put("..", parentInodeNumberDecodedAsInodeNumber) - if nil != err { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,,,,,parentInodeNumber==0x%016X,) failed to insert \"..\" dirEntry: %v", inodeNumber, parentInodeNumber, err) - panic(err) - } - if !ok { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,,,,,parentInodeNumber==0x%016X,) insert \"..\" dirEntry got a !ok", inodeNumber, parentInodeNumber) - panic(err) - } - - inode.payload = payload - inode.onDiskInodeV1Struct.SymlinkTarget = "" - case FileType: - payload = sortedmap.NewBPlusTree( - vS.maxExtentsPerFileNode, - sortedmap.CompareUint64, - &fileInodeCallbacks{treeNodeLoadable{inode: inode}}, - globals.fileExtentMapCache) - - inode.payload = payload - inode.onDiskInodeV1Struct.SymlinkTarget = "" - case SymlinkType: - inode.payload = nil - inode.onDiskInodeV1Struct.SymlinkTarget = symlinkTarget - } - - ok, err = vS.inodeCacheInsertWhileLocked(inode) - if nil != err { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,,,,,,) failed to insert inode in inodeCache: %v", inodeNumber, err) - panic(err) - } - if !ok { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,inodeType==DirType,,,,,,) insert of inode in inodeCache got a !ok", inodeNumber) - panic(err) - } - - _ = inodeRWLock.Unlock() - - vS.Unlock() - - err = vS.flushInode(inode) - if nil != err { - err = fmt.Errorf("PatchInode(inodeNumber==0x%016X,,,,,,,) failed to flush: %v", inodeNumber, err) - panic(err) - } - - return -} - -func (inMemoryInode *inMemoryInodeStruct) convertToOnDiskInodeV1() (onDiskInodeV1 *onDiskInodeV1Struct, err error) { - onDiskInode := inMemoryInode.onDiskInodeV1Struct - - if (DirType == inMemoryInode.InodeType) || (FileType == inMemoryInode.InodeType) { - content := inMemoryInode.payload.(sortedmap.BPlusTree) - payloadObjectNumber, payloadObjectOffset, payloadObjectLength, flushErr := content.Flush(false) - if nil != flushErr { - panic(flushErr) - } - pruneErr := content.Prune() - if nil != pruneErr { - panic(pruneErr) - } - if onDiskInodeV1PayloadObjectOffset != payloadObjectOffset { - flushErr = fmt.Errorf("Logic Error: content.Flush() should have returned payloadObjectOffset == %v", onDiskInodeV1PayloadObjectOffset) - panic(flushErr) - } - onDiskInode.PayloadObjectNumber = payloadObjectNumber - onDiskInode.PayloadObjectLength = payloadObjectLength - } - - // maps are refernce types, so this needs to be copied manually - - onDiskInode.StreamMap = make(map[string][]byte) - for key, value := range inMemoryInode.StreamMap { - valueCopy := make([]byte, len(value)) - copy(valueCopy, value) - onDiskInode.StreamMap[key] = valueCopy - } - - onDiskInode.LogSegmentMap = make(map[uint64]uint64) - for logSegmentNumber, logSegmentBytesUsed := range inMemoryInode.LogSegmentMap { - onDiskInode.LogSegmentMap[logSegmentNumber] = logSegmentBytesUsed - } - - return &onDiskInode, nil -} - -func (vS *volumeStruct) flushInode(inode *inMemoryInodeStruct) (err error) { - err = vS.flushInodes([]*inMemoryInodeStruct{inode}) - return -} - -func (vS *volumeStruct) flushInodeNumber(inodeNumber InodeNumber) (err error) { - err = vS.flushInodeNumbers([]InodeNumber{inodeNumber}) - return -} - -// REVIEW: Need to clearly explain what "flush" means (i.e. "to HH", not "to disk") - -func (vS *volumeStruct) flushInodes(inodes []*inMemoryInodeStruct) (err error) { - var ( - dirtyInodeNumbers []uint64 - dirtyInodeRecBytes []byte - dirtyInodeRecs [][]byte - emptyLogSegments []uint64 - emptyLogSegmentsThisInode []uint64 - inode *inMemoryInodeStruct - logSegmentNumber uint64 - logSegmentValidBytes uint64 - onDiskInodeV1 *onDiskInodeV1Struct - onDiskInodeV1Buf []byte - payloadAsBPlusTree sortedmap.BPlusTree - payloadObjectLength uint64 - payloadObjectNumber uint64 - toFlushInodeNumbers []uint64 - ) - - halter.Trigger(halter.InodeFlushInodesEntry) - defer halter.Trigger(halter.InodeFlushInodesExit) - - toFlushInodeNumbers = make([]uint64, 0, len(inodes)) - for _, inode = range inodes { - toFlushInodeNumbers = append(toFlushInodeNumbers, uint64(inode.InodeNumber)) - } - - evtlog.Record(evtlog.FormatFlushInodesEntry, vS.volumeName, toFlushInodeNumbers) - - // Assemble slice of "dirty" inodes while flushing them - dirtyInodeNumbers = make([]uint64, 0, len(inodes)) - dirtyInodeRecs = make([][]byte, 0, len(inodes)) - emptyLogSegments = make([]uint64, 0) - - for _, inode = range inodes { - if FileType == inode.InodeType { - err = vS.doFileInodeDataFlush(inode) - if nil != err { - evtlog.Record(evtlog.FormatFlushInodesErrorOnInode, vS.volumeName, uint64(inode.InodeNumber), err.Error()) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.InodeFlushError) - return - } - emptyLogSegmentsThisInode = make([]uint64, 0) - for logSegmentNumber, logSegmentValidBytes = range inode.LogSegmentMap { - if 0 == logSegmentValidBytes { - emptyLogSegmentsThisInode = append(emptyLogSegmentsThisInode, logSegmentNumber) - } - } - for _, logSegmentNumber = range emptyLogSegmentsThisInode { - delete(inode.LogSegmentMap, logSegmentNumber) - } - emptyLogSegments = append(emptyLogSegments, emptyLogSegmentsThisInode...) - } - if SymlinkType != inode.InodeType { - // (FileType == inode.InodeType || (DirType == inode.InodeType) - payloadAsBPlusTree = inode.payload.(sortedmap.BPlusTree) - payloadObjectNumber, _, payloadObjectLength, err = payloadAsBPlusTree.Flush(false) - if nil != err { - evtlog.Record(evtlog.FormatFlushInodesErrorOnInode, vS.volumeName, uint64(inode.InodeNumber), err.Error()) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.InodeFlushError) - return - } - if payloadObjectNumber > inode.PayloadObjectNumber { - if !inode.dirty { - err = fmt.Errorf("Logic error: inode.dirty should have been true") - evtlog.Record(evtlog.FormatFlushInodesErrorOnInode, vS.volumeName, uint64(inode.InodeNumber), err.Error()) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.InodeFlushError) - return - } - // REVIEW: What if cache pressure flushed before we got here? - // Is it possible that Number doesn't get updated? - - if inode.PayloadObjectNumber != 0 { - logger.Tracef("flushInodes(): volume '%s' %v inode %d: updating Payload"+ - " from Object %016X to %016X bytes %d to %d", - vS.volumeName, inode.InodeType, inode.InodeNumber, - inode.PayloadObjectNumber, payloadObjectNumber, - inode.PayloadObjectLength, payloadObjectLength) - } - inode.PayloadObjectNumber = payloadObjectNumber - inode.PayloadObjectLength = payloadObjectLength - - evtlog.Record(evtlog.FormatFlushInodesDirOrFilePayloadObjectNumberUpdated, vS.volumeName, uint64(inode.InodeNumber), payloadObjectNumber) - } - } - if inode.dirty { - onDiskInodeV1, err = inode.convertToOnDiskInodeV1() - if nil != err { - evtlog.Record(evtlog.FormatFlushInodesErrorOnInode, vS.volumeName, uint64(inode.InodeNumber), err.Error()) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.InodeFlushError) - return - } - onDiskInodeV1Buf, err = json.Marshal(onDiskInodeV1) - if nil != err { - evtlog.Record(evtlog.FormatFlushInodesErrorOnInode, vS.volumeName, uint64(inode.InodeNumber), err.Error()) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.InodeFlushError) - return - } - dirtyInodeRecBytes = make([]byte, 0, len(globals.inodeRecDefaultPreambleBuf)+len(onDiskInodeV1Buf)) - dirtyInodeRecBytes = append(dirtyInodeRecBytes, globals.inodeRecDefaultPreambleBuf...) - dirtyInodeRecBytes = append(dirtyInodeRecBytes, onDiskInodeV1Buf...) - dirtyInodeNumbers = append(dirtyInodeNumbers, uint64(inode.InodeNumber)) - dirtyInodeRecs = append(dirtyInodeRecs, dirtyInodeRecBytes) - } - } - - // Go update HeadHunter (if necessary) - if 0 < len(dirtyInodeNumbers) { - err = vS.headhunterVolumeHandle.PutInodeRecs(dirtyInodeNumbers, dirtyInodeRecs) - if nil != err { - evtlog.Record(evtlog.FormatFlushInodesErrorOnHeadhunterPut, vS.volumeName, err.Error()) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.InodeFlushError) - return - } - for _, inode = range inodes { - inode.dirty = false - } - } - - // Now do phase one of garbage collection - if 0 < len(emptyLogSegments) { - for _, logSegmentNumber = range emptyLogSegments { - err = vS.headhunterVolumeHandle.DeleteLogSegmentRec(logSegmentNumber) - if nil != err { - logger.WarnfWithError(err, "couldn't delete garbage log segment") - } - } - } - - evtlog.Record(evtlog.FormatFlushInodesExit, vS.volumeName, toFlushInodeNumbers) - - err = nil - return -} - -func (vS *volumeStruct) flushInodeNumbers(inodeNumbers []InodeNumber) (err error) { - var ( - inode *inMemoryInodeStruct - inodes []*inMemoryInodeStruct - inodeNumber InodeNumber - ok bool - ) - - // Fetch referenced inodes - inodes = make([]*inMemoryInodeStruct, 0, len(inodeNumbers)) - for _, inodeNumber = range inodeNumbers { - inode, ok, err = vS.fetchInode(inodeNumber) - if nil != err { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode to flush failed", utils.GetFnName()) - err = blunder.AddError(err, blunder.InodeFlushError) - return - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: fetch of inode %d volume '%s' failed because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - inodes = append(inodes, inode) - } - - err = vS.flushInodes(inodes) - - return -} - -func accountNameToVolumeName(accountName string) (volumeName string, ok bool) { - var ( - volume *volumeStruct - ) - - globals.Lock() - - volume, ok = globals.accountMap[accountName] - if ok { - volumeName = volume.volumeName - } - - globals.Unlock() - - return -} - -func volumeNameToAccountName(volumeName string) (accountName string, ok bool) { - var ( - volume *volumeStruct - ) - - globals.Lock() - - volume, ok = globals.volumeMap[volumeName] - if ok { - accountName = volume.accountName - } - - globals.Unlock() - - return -} - -func volumeNameToActivePeerPrivateIPAddr(volumeName string) (activePeerPrivateIPAddr string, ok bool) { - var ( - volume *volumeStruct - ) - - globals.Lock() - - volume, ok = globals.volumeMap[volumeName] - - if ok { - activePeerPrivateIPAddr = volume.volumeGroup.activePeerPrivateIPAddr - } - - globals.Unlock() - - return -} - -func fetchVolumeHandle(volumeName string) (volumeHandle VolumeHandle, err error) { - globals.Lock() - volume, ok := globals.volumeMap[volumeName] - globals.Unlock() - - if !ok { - err = fmt.Errorf("%s: volumeName \"%v\" not found", utils.GetFnName(), volumeName) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - volumeHandle = volume - - volume.Lock() // REVIEW: Once Tracker https://www.pivotaltracker.com/story/show/133377567 - defer volume.Unlock() // is resolved, these two lines should be removed - - if !volume.served { - err = fmt.Errorf("%s: volumeName \"%v\" not served", utils.GetFnName(), volumeName) - err = blunder.AddError(err, blunder.NotActiveError) - return - } - - _, ok, err = volume.headhunterVolumeHandle.GetInodeRec(uint64(RootDirInodeNumber)) - if nil != err { - // disk corruption of the inode btree (or software error) - err = fmt.Errorf("%s: unable to lookup root inode for volume '%s': %v", - utils.GetFnName(), volume.volumeName, err) - err = blunder.AddError(err, blunder.NotFoundError) - } - if !ok { - // First access didn't find root dir... so create it - _, err = volume.createRootOrSubDir(PosixModePerm, 0, 0, true) - if nil != err { - err = fmt.Errorf("%s: unable to create root inode for volume '%s': %v", - utils.GetFnName(), volume.volumeName, err) - err = blunder.AddError(err, blunder.NotFoundError) - } - } - - // If we get this far, return values are already set as desired - - err = nil - - return -} - -func (vS *volumeStruct) provisionPhysicalContainer(physicalContainerLayout *physicalContainerLayoutStruct) (err error) { - if 0 == (physicalContainerLayout.containerNameSliceLoopCount % physicalContainerLayout.maxObjectsPerContainer) { - // We need to provision a new PhysicalContainer in this PhysicalContainerLayout - - physicalContainerNameSuffix := vS.headhunterVolumeHandle.FetchNonce() - - newContainerName := fmt.Sprintf("%s%s", physicalContainerLayout.containerNamePrefix, utils.Uint64ToHexStr(physicalContainerNameSuffix)) - - storagePolicyHeaderValues := []string{vS.defaultPhysicalContainerLayout.containerStoragePolicy} - newContainerHeaders := make(map[string][]string) - newContainerHeaders["X-Storage-Policy"] = storagePolicyHeaderValues - - err = swiftclient.ContainerPut(vS.accountName, newContainerName, newContainerHeaders) - if nil != err { - return - } - - physicalContainerLayout.containerNameSlice[physicalContainerLayout.containerNameSliceNextIndex] = newContainerName - } - - err = nil - return -} - -func (vS *volumeStruct) provisionObject() (containerName string, objectNumber uint64, err error) { - objectNumber = vS.headhunterVolumeHandle.FetchNonce() - - vS.Lock() - - err = vS.provisionPhysicalContainer(vS.defaultPhysicalContainerLayout) - if nil != err { - vS.Unlock() - return - } - - containerName = vS.defaultPhysicalContainerLayout.containerNameSlice[vS.defaultPhysicalContainerLayout.containerNameSliceNextIndex] - - vS.defaultPhysicalContainerLayout.containerNameSliceNextIndex++ - - if vS.defaultPhysicalContainerLayout.containerNameSliceNextIndex == vS.defaultPhysicalContainerLayout.containersPerPeer { - vS.defaultPhysicalContainerLayout.containerNameSliceNextIndex = 0 - vS.defaultPhysicalContainerLayout.containerNameSliceLoopCount++ - } - - vS.Unlock() - - err = nil - return -} - -func (vS *volumeStruct) Access(inodeNumber InodeNumber, userID InodeUserID, groupID InodeGroupID, otherGroupIDs []InodeGroupID, accessMode InodeMode, override AccessOverride) (accessReturn bool) { - var ( - adjustedInodeNumber InodeNumber - err error - groupIDCheck bool - ok bool - otherGroupID InodeGroupID - ourInode *inMemoryInodeStruct - ourInodeGroupID InodeGroupID - ourInodeMode InodeMode - ourInodeUserID InodeUserID - snapShotIDType headhunter.SnapShotIDType - ) - - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - - switch snapShotIDType { - case headhunter.SnapShotIDTypeLive: - adjustedInodeNumber = inodeNumber - case headhunter.SnapShotIDTypeSnapShot: - adjustedInodeNumber = inodeNumber - case headhunter.SnapShotIDTypeDotSnapShot: - adjustedInodeNumber = RootDirInodeNumber - default: - logger.Fatalf("headhunter.SnapShotU64Decode(inodeNumber == 0x%016X) returned unknown snapShotIDType: %v", inodeNumber, snapShotIDType) - } - if (headhunter.SnapShotIDTypeLive != snapShotIDType) && (0 != (W_OK & accessMode)) { - err = blunder.NewError(blunder.InvalidArgError, "Access() where accessMode includes W_OK of non-LiveView inodeNumber not allowed") - return - } - - ourInode, ok, err = vS.fetchInode(adjustedInodeNumber) - if nil != err { - // this indicates disk corruption or software bug - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - - // if we can't fetch the inode we can't access it - accessReturn = false - return - } - if !ok { - // disk corruption or client requested a free inode - logger.Infof("%s: fetch of inode %d volume '%s' failed because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - - // if the inode is free then we can't access it - accessReturn = false - return - } - - ourInodeUserID = ourInode.UserID - ourInodeGroupID = ourInode.GroupID - - if headhunter.SnapShotIDTypeLive == snapShotIDType { - ourInodeMode = ourInode.Mode - } else { - ourInodeMode = ourInode.Mode // TODO: Make it read-only... - } - - if F_OK == accessMode { - // the inode exists so its F_OK - accessReturn = true - return - } - - if P_OK == accessMode { - accessReturn = (InodeRootUserID == userID) || (userID == ourInodeUserID) - return - } - - if accessMode != (accessMode & (R_OK | W_OK | X_OK)) { - // Default to false if P_OK bit set along with any others) - accessReturn = false - return - } - - // Only the LiveView is ever writeable... even by the root user - if (accessMode&W_OK != 0) && (headhunter.SnapShotIDTypeLive != snapShotIDType) { - accessReturn = false - return - } - - // The root user (if not squashed) can do anything except exec files - // that are not executable by any user - if userID == InodeRootUserID { - if (accessMode&X_OK != 0) && (ourInodeMode&(X_OK<<6|X_OK<<3|X_OK) == 0) { - accessReturn = false - } else { - accessReturn = true - } - return - } - - // We check against permissions for the user, group, and other. The - // first match wins (not the first permission granted). If the user is - // the owner of the file then those permission bits determine what - // happens. In other words, if the permission bits deny read permission - // to the owner of a file but allow read permission for group and other, - // then everyone except the owner of the file can read it. - // - // On a local file system, the owner of a file is *not* allowed to write - // to the file unless it was opened for writing and the permission bits - // allowed it *or* the process created the file and opened it for - // writing at the same time. However, NFS does not have an open state - // (there's no file descriptor that tracks permissions when the the file - // was opened) so we check for write permission on every write. This - // breaks things like tar when it tries to unpack a file which has - // permission 0444 (read only). On a local file system that works, but - // it doesn't work for NFS unless we bend the rules a bit for the owner - // of the file and allow the owner to write to the file even if - // appropriate permissions are lacking. (This is only done for the user - // that owns the file, not the group that owns the file. Note that the - // owner can always change the permissions to allow writing so its not a - // security risk, but the owning group cannot). - // - // Note that the NFS client will typically call Access() when an app - // wants to open the file and fail an open request for writing that if - // the permission bits do not allow it. - // - // Similar rules apply to Read() and Truncate() (for ftruncate(2)), but - // not for execute permission. Also, this only applies to regular files - // but we'll rely on the caller for that. - if userID == ourInodeUserID { - if override == OwnerOverride && (accessMode&X_OK == 0) { - accessReturn = true - } else { - accessReturn = (((ourInodeMode >> 6) & accessMode) == accessMode) - } - return - } - - groupIDCheck = (groupID == ourInodeGroupID) - if !groupIDCheck { - for _, otherGroupID = range otherGroupIDs { - if otherGroupID == ourInodeGroupID { - groupIDCheck = true - break - } - } - } - if groupIDCheck { - accessReturn = ((((ourInodeMode >> 3) & 07) & accessMode) == accessMode) - return - } - - accessReturn = ((((ourInodeMode >> 0) & 07) & accessMode) == accessMode) - return -} - -func (vS *volumeStruct) ProvisionObject() (objectPath string, err error) { - err = enforceRWMode(true) - if nil != err { - return - } - - containerName, objectNumber, err := vS.provisionObject() - if nil != err { - return - } - - objectPath = fmt.Sprintf("/v1/%s/%s/%016X", vS.accountName, containerName, objectNumber) - - err = nil - return -} - -func (vS *volumeStruct) Purge(inodeNumber InodeNumber) (err error) { - var ( - inode *inMemoryInodeStruct - ok bool - ) - - err = enforceRWMode(false) - if nil != err { - return - } - - inode, ok, err = vS.inodeCacheFetch(inodeNumber) - if (nil != err) || !ok { - return - } - - if inode.dirty { - err = fmt.Errorf("Inode dirty... cannot be purged") - return - } - - ok, err = vS.inodeCacheDrop(inode) - if nil != err { - return - } - if !ok { - err = fmt.Errorf("inodeCacheDrop(inode) failed") - } - - return -} - -func (vS *volumeStruct) Destroy(inodeNumber InodeNumber) (err error) { - logger.Tracef("inode.Destroy(): volume '%s' inode %d", vS.volumeName, inodeNumber) - - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("Destroy() on non-LiveView inodeNumber not allowed") - return - } - - ourInode, ok, err := vS.fetchInode(inodeNumber) - if nil != err { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: cannot destroy inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - err = blunder.AddError(err, blunder.NotFoundError) - logger.ErrorWithError(err) - return - } - - ok, err = vS.inodeCacheDrop(ourInode) - if nil != err { - logger.ErrorfWithError(err, "%s: inodeCacheDrop() of inode failed: %v", utils.GetFnName(), err) - return - } - if !ok { - logger.ErrorfWithError(err, "%s: inodeCacheDrop() of inode returned !ok", utils.GetFnName()) - return - } - - if ourInode.InodeType == FileType { - _ = vS.doFileInodeDataFlush(ourInode) - } - - err = vS.headhunterVolumeHandle.DeleteInodeRec(uint64(inodeNumber)) - if nil != err { - logger.ErrorWithError(err) - return - } - - if DirType == ourInode.InodeType { - logger.Tracef("inode.Destroy(): volume '%s' inode %d: discarding dirmap payload Object %016X len %d", - vS.volumeName, inodeNumber, ourInode.PayloadObjectNumber, ourInode.PayloadObjectLength) - - dirMapping := ourInode.payload.(sortedmap.BPlusTree) - - err = dirMapping.Discard() - if nil != err { - logger.ErrorWithError(err) - return - } - - stats.IncrementOperations(&stats.DirDestroyOps) - - } else if FileType == ourInode.InodeType { - logger.Tracef("inode.Destroy(): volume '%s' inode %d: discarding extmap payload Object %016X len %d", - vS.volumeName, inodeNumber, ourInode.PayloadObjectNumber, ourInode.PayloadObjectLength) - - extents := ourInode.payload.(sortedmap.BPlusTree) - - err = extents.Discard() - if nil != err { - logger.ErrorWithError(err) - return - } - - for logSegmentNumber := range ourInode.LogSegmentMap { - deleteSegmentErr := vS.headhunterVolumeHandle.DeleteLogSegmentRec(logSegmentNumber) - if nil != deleteSegmentErr { - logger.WarnfWithError(deleteSegmentErr, "couldn't delete destroy'd log segment") - return - } - stats.IncrementOperations(&stats.GcLogSegDeleteOps) - } - stats.IncrementOperations(&stats.GcLogSegOps) - - stats.IncrementOperations(&stats.FileDestroyOps) - } else { // SymlinkType == ourInode.InodeType - stats.IncrementOperations(&stats.SymlinkDestroyOps) - } - - return -} - -func (vS *volumeStruct) GetMetadata(inodeNumber InodeNumber) (metadata *MetadataStruct, err error) { - var ( - inode *inMemoryInodeStruct - ok bool - pos int - snapShotIDType headhunter.SnapShotIDType - ) - - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - // For /, start with metadata from / - inode, ok, err = vS.fetchInode(RootDirInodeNumber) - } else { - inode, ok, err = vS.fetchInode(inodeNumber) - } - - if nil != err { - // this indicates disk corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - err = blunder.AddError(err, blunder.NotFoundError) - logger.InfoWithError(err) - return - } - - metadata = &MetadataStruct{ - InodeType: inode.InodeType, - LinkCount: inode.LinkCount, - Size: inode.Size, - CreationTime: inode.CreationTime, - ModificationTime: inode.ModificationTime, - AccessTime: inode.AccessTime, - AttrChangeTime: inode.AttrChangeTime, - NumWrites: inode.NumWrites, - InodeStreamNameSlice: make([]string, len(inode.StreamMap)), - Mode: inode.Mode, - UserID: inode.UserID, - GroupID: inode.GroupID, - } - - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - // For /, simply remove Write Access... and skip InodeStreamNameSlice - metadata.Mode &= metadata.Mode & ^(W_OK<<6 | W_OK<<3 | W_OK<<0) - } else { - if headhunter.SnapShotIDTypeSnapShot == snapShotIDType { - // For inodes in a SnapShot, simply remove Write Access - metadata.Mode &= metadata.Mode & ^(W_OK<<6 | W_OK<<3 | W_OK<<0) - } - pos = 0 - for inodeStreamName := range inode.StreamMap { - metadata.InodeStreamNameSlice[pos] = inodeStreamName - pos++ - } - } - - stats.IncrementOperations(&stats.InodeGetMetadataOps) - return -} - -func (vS *volumeStruct) GetType(inodeNumber InodeNumber) (inodeType InodeType, err error) { - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - inodeType = DirType - err = nil - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if nil != err { - // this indicates disk corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.InfoWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - inodeType = inode.InodeType - - stats.IncrementOperations(&stats.InodeGetTypeOps) - return -} - -func (vS *volumeStruct) GetLinkCount(inodeNumber InodeNumber) (linkCount uint64, err error) { - var ( - adjustLinkCountForSnapShotSubDirInRootDirInode bool - inode *inMemoryInodeStruct - ok bool - snapShotCount uint64 - snapShotIDType headhunter.SnapShotIDType - ) - - if RootDirInodeNumber == inodeNumber { - // Account for .. in / if any SnapShot's exist - snapShotCount = vS.headhunterVolumeHandle.SnapShotCount() - adjustLinkCountForSnapShotSubDirInRootDirInode = (0 != snapShotCount) - } else { - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - // linkCount == 1 (/'s '.') + 1 (/'s reference to ) + # SnapShot's (/..' in each SnapShot's /) - snapShotCount = vS.headhunterVolumeHandle.SnapShotCount() - linkCount = 1 + 1 + snapShotCount - err = nil - return - } - adjustLinkCountForSnapShotSubDirInRootDirInode = false - } - - inode, ok, err = vS.fetchInode(inodeNumber) - if nil != err { - // this indicates disk corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.InfoWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - if adjustLinkCountForSnapShotSubDirInRootDirInode { - linkCount = inode.LinkCount + 1 - } else { - linkCount = inode.LinkCount - } - - return -} - -// SetLinkCount is used to adjust the LinkCount property to match current reference count during FSCK TreeWalk. -func (vS *volumeStruct) SetLinkCount(inodeNumber InodeNumber, linkCount uint64) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetLinkCount() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // this indicates disk corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.InfoWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - inode.dirty = true - inode.LinkCount = linkCount - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func (vS *volumeStruct) SetCreationTime(inodeNumber InodeNumber, CreationTime time.Time) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetCreationTime() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - logger.ErrorfWithError(err, "%s: fetch of target inode failed", utils.GetFnName()) - return err - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return err - } - - inode.dirty = true - inode.AttrChangeTime = time.Now() - inode.CreationTime = CreationTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - return -} - -func (vS *volumeStruct) SetModificationTime(inodeNumber InodeNumber, ModificationTime time.Time) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetModificationTime() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - logger.ErrorfWithError(err, "%s: fetch of target inode failed", utils.GetFnName()) - return err - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return err - } - - inode.dirty = true - inode.AttrChangeTime = time.Now() - inode.ModificationTime = ModificationTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func (vS *volumeStruct) SetAccessTime(inodeNumber InodeNumber, accessTime time.Time) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetAccessTime() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - logger.ErrorfWithError(err, "%s: fetch of target inode failed", utils.GetFnName()) - return err - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return err - } - - inode.dirty = true - inode.AttrChangeTime = time.Now() - inode.AccessTime = accessTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func determineMode(filePerm InodeMode, inodeType InodeType) (fileMode InodeMode, err error) { - // Caller should only be setting the file perm bits, but samba seems to send file type - // bits as well. Since we need to work with whatever samba does, let's just silently - // mask off the other bits. - if filePerm&^PosixModePerm != 0 { - logger.Tracef("inode.determineMode(): invalid file mode 0x%x (max 0x%x); removing file type bits.", uint32(filePerm), uint32(PosixModePerm)) - } - - // Build fileMode starting with the file permission bits - fileMode = filePerm & PosixModePerm - - // Add the file type to the mode. - switch inodeType { - case DirType: - fileMode |= PosixModeDir - case FileType: - fileMode |= PosixModeFile - case SymlinkType: - fileMode |= PosixModeSymlink - default: - err = fmt.Errorf("%s: unrecognized inode type %v", utils.GetFnName(), inodeType) - err = blunder.AddError(err, blunder.InvalidInodeTypeError) - return - } - - err = nil - return -} - -func (vS *volumeStruct) SetPermMode(inodeNumber InodeNumber, filePerm InodeMode) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetPermMode() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - logger.ErrorfWithError(err, "%s: fetch of target inode failed", utils.GetFnName()) - return err - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return err - } - - // Create file mode out of file permissions plus inode type - fileMode, err := determineMode(filePerm, inode.InodeType) - if err != nil { - return err - } - - inode.dirty = true - inode.Mode = fileMode - - updateTime := time.Now() - inode.AttrChangeTime = updateTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func (vS *volumeStruct) SetOwnerUserID(inodeNumber InodeNumber, userID InodeUserID) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetOwnerUserID() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - logger.ErrorfWithError(err, "%s: fetch of target inode failed", utils.GetFnName()) - return err - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return err - } - - inode.dirty = true - inode.UserID = userID - - updateTime := time.Now() - inode.AttrChangeTime = updateTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func (vS *volumeStruct) SetOwnerUserIDGroupID(inodeNumber InodeNumber, userID InodeUserID, groupID InodeGroupID) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetOwnerUserIDGroupID() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - logger.ErrorfWithError(err, "%s: fetch of target inode failed", utils.GetFnName()) - return err - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return err - } - - inode.dirty = true - inode.UserID = userID - inode.GroupID = groupID - - updateTime := time.Now() - inode.AttrChangeTime = updateTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func (vS *volumeStruct) SetOwnerGroupID(inodeNumber InodeNumber, groupID InodeGroupID) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("SetOwnerGroupID() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // the inode is locked so this should never happen (unless the inode - // was evicted from the cache and it was corrupt when read from disk) - logger.ErrorfWithError(err, "%s: fetch of target inode failed", utils.GetFnName()) - return err - } - if !ok { - // this should never happen (see above) - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return err - } - - inode.dirty = true - inode.GroupID = groupID - - updateTime := time.Now() - inode.AttrChangeTime = updateTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func (vS *volumeStruct) GetStream(inodeNumber InodeNumber, inodeStreamName string) (buf []byte, err error) { - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - err = fmt.Errorf("No stream '%v'", inodeStreamName) - return buf, blunder.AddError(err, blunder.StreamNotFound) - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // this indicates disk corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return nil, err - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.InfoWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return nil, err - } - - inodeStreamBuf, ok := inode.StreamMap[inodeStreamName] - - if !ok { - err = fmt.Errorf("No stream '%v'", inodeStreamName) - return buf, blunder.AddError(err, blunder.StreamNotFound) - } - - buf = make([]byte, len(inodeStreamBuf)) - - copy(buf, inodeStreamBuf) - - err = nil - - return -} - -func (vS *volumeStruct) PutStream(inodeNumber InodeNumber, inodeStreamName string, buf []byte) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("PutStream() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // this indicates disk corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return err - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.InfoWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return err - } - - inodeStreamBuf := make([]byte, len(buf)) - - copy(inodeStreamBuf, buf) - - inode.dirty = true - inode.StreamMap[inodeStreamName] = inodeStreamBuf - - updateTime := time.Now() - inode.AttrChangeTime = updateTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func (vS *volumeStruct) DeleteStream(inodeNumber InodeNumber, inodeStreamName string) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = fmt.Errorf("DeleteStream() on non-LiveView inodeNumber not allowed") - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // this indicates disk corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.InfoWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - inode.dirty = true - delete(inode.StreamMap, inodeStreamName) - - updateTime := time.Now() - inode.AttrChangeTime = updateTime - - err = vS.flushInode(inode) - if err != nil { - logger.ErrorWithError(err) - return err - } - - return -} - -func (vS *volumeStruct) FetchLayoutReport(inodeNumber InodeNumber) (layoutReport sortedmap.LayoutReport, err error) { - snapShotIDType, _, _ := vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - layoutReport = make(sortedmap.LayoutReport) - err = nil - return - } - - inode, ok, err := vS.fetchInode(inodeNumber) - if err != nil { - // this indicates disk corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - return nil, err - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.InfoWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return nil, err - } - - if SymlinkType == inode.InodeType { - layoutReport = make(sortedmap.LayoutReport) - err = nil - } else { - layoutReport, err = inode.payload.(sortedmap.BPlusTree).FetchLayoutReport() - } - - return -} - -func (vS *volumeStruct) FetchFragmentationReport(inodeNumber InodeNumber) (fragmentationReport FragmentationReport, err error) { - err = fmt.Errorf("FetchFragmentationReport not yet implemented") - return -} - -func (vS *volumeStruct) Optimize(inodeNumber InodeNumber, maxDuration time.Duration) (err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - err = fmt.Errorf("Optimize not yet implemented") - return -} - -func validateFileExtents(snapShotID uint64, ourInode *inMemoryInodeStruct) (err error) { - var ( - zero = uint64(0) - ) - - readPlan, readPlanBytes, err := ourInode.volume.getReadPlanHelper(snapShotID, ourInode, &zero, nil) - if err != nil { - return err - } - - // We read the whole file, so these should match - if readPlanBytes != ourInode.Size { - return blunder.NewError(blunder.CorruptInodeError, "inode %v had recorded size %v bytes, but full read plan was only %v bytes", ourInode.InodeNumber, ourInode.Size, readPlanBytes) - } - - // Let's check that the read plan is consistent with what the inode's - // internal log-segment map says about which segments should have how much data. - // - // Make a copy of the inode's LogSegmentMap map so we can decrement the - // byte count for each segment as we walk the readPlan entries. - remainingExpectedBytes := make(map[uint64]uint64) - for segmentNumber, segmentBytesUsed := range ourInode.LogSegmentMap { - remainingExpectedBytes[segmentNumber] += segmentBytesUsed - } - // Then we can compare with the actual read plan we got ... - for _, readPlanStep := range readPlan { - - // holes in a sparse file aren't counted - if readPlanStep.LogSegmentNumber == 0 { - continue - } - pathSegments := strings.Split(readPlanStep.ObjectPath, "/") - logSegmentRepresentation := pathSegments[len(pathSegments)-1] - logSegmentNumber, hexConvErr := utils.HexStrToUint64(logSegmentRepresentation) - if hexConvErr != nil { - return blunder.NewError(blunder.CorruptInodeError, - "conversion of read plan object name to log segment number failed; "+ - "readPlanStep: %v logSegmentString: '%v' err: %v", - readPlanStep, logSegmentRepresentation, hexConvErr) - } - remainingExpectedBytes[logSegmentNumber] -= readPlanStep.Length - } - // ... and fail validation if any log segment didn't match. We'll put the - // mismatches in a separate map that we'll attach to the error in case a - // consumer or logger wants it. - logSegmentByteCountMismatches := make(map[uint64]uint64) - for logSegmentNumber, remainingExpectedByteCount := range remainingExpectedBytes { - if remainingExpectedByteCount != 0 { - logSegmentByteCountMismatches[logSegmentNumber] = remainingExpectedByteCount - } - } - if len(logSegmentByteCountMismatches) != 0 { - rootErr := fmt.Errorf("inconsistency detected between log segment map and read plan for inode %v", ourInode.InodeNumber) - return merry.WithValue(blunder.AddError(rootErr, blunder.CorruptInodeError), "logSegmentByteCountMismatches", logSegmentByteCountMismatches) - } - - // Having verified that our read plan is consistent with our internal log - // segment map, we also want to check that it's consistent with the actual log - // segment objects in Swift. First, we'll construct a map of object paths to - // the largest offset we would need read up to in that object. - objectPathToEndOffset := make(map[string]uint64) - - for _, planStep := range readPlan { - - // holes in a sparse file don't have objects - if planStep.LogSegmentNumber == 0 { - continue - } - stepEndOffset := planStep.Offset + planStep.Length - endOffset, ok := objectPathToEndOffset[planStep.ObjectPath] - if !ok || stepEndOffset > endOffset { - objectPathToEndOffset[planStep.ObjectPath] = stepEndOffset - } - } - - // then, HEAD each object to make sure that it has enough bytes. - for objectPath, endOffset := range objectPathToEndOffset { - accountName, containerName, objectName, err := utils.PathToAcctContObj(objectPath) - if err != nil { - logger.ErrorWithError(err) - return err - } - - contentLength, err := swiftclient.ObjectContentLength(accountName, containerName, objectName) - if err != nil { - logger.ErrorWithError(err) - return err - } - - if contentLength < endOffset { - // REVIEW: it might be helpful to continue and make a combined report of all - // insufficiently long log segments, rather than erroring out immediately - err = fmt.Errorf("expected %q to have at least %v bytes, content length was %v", objectPath, endOffset, contentLength) - logger.ErrorWithError(err) - return err - } - - } - - return nil -} - -func (vS *volumeStruct) markCorrupted(inodeNumber InodeNumber) (err error) { - var ( - inodeRec []byte - ok bool - snapShotIDType headhunter.SnapShotIDType - ) - - snapShotIDType, _, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeLive != snapShotIDType { - err = blunder.NewError(blunder.InvalidArgError, "markCorrupted() of non-LiveView inodeNumber not allowed") - return - } - - inodeRec, ok, err = vS.headhunterVolumeHandle.GetInodeRec(uint64(inodeNumber)) - if nil == err && ok && (len(globals.corruptionDetectedTrueBuf) <= len(inodeRec)) { - // Just overwrite CorruptionDetected field with true - _ = copy(inodeRec, globals.corruptionDetectedTrueBuf) - } else { - // Use a simple CorruptionDetected == true inodeRec - inodeRec = globals.corruptionDetectedTrueBuf - } - - err = vS.headhunterVolumeHandle.PutInodeRec(uint64(inodeNumber), inodeRec) - - return -} - -func (vS *volumeStruct) Validate(inodeNumber InodeNumber, deeply bool) (err error) { - var ( - ok bool - ourInode *inMemoryInodeStruct - snapShotID uint64 - snapShotIDType headhunter.SnapShotIDType - tree sortedmap.BPlusTree - ) - - snapShotIDType, snapShotID, _ = vS.headhunterVolumeHandle.SnapShotU64Decode(uint64(inodeNumber)) - if headhunter.SnapShotIDTypeDotSnapShot == snapShotIDType { - err = nil // Since / is emulated, always return success - return - } - - // we don't want to use the in-memory cache for this; we'll need to fetch - // the current real-world bits from disk. - - // If this is a file inode, we flush to ensure that the inode is not dirty - // (and that DLM locking therefore ensures we have exclusive access to the - // inode and don't need to serialize this operation, as there can be no pending - // time-based flush to race with). - - err = vS.flushInodeNumber(inodeNumber) - if nil != err { - logger.ErrorfWithError(err, "couldn't flush inode %v", inodeNumber) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - - err = vS.Purge(inodeNumber) - if nil != err { - logger.ErrorfWithError(err, "couldn't purge inode %v", inodeNumber) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - - ourInode, ok, err = vS.fetchInode(inodeNumber) - if nil != err { - // this indicates diskj corruption or software error - // (err includes volume name and inode number) - logger.ErrorfWithError(err, "%s: fetch of inode failed", utils.GetFnName()) - err = blunder.AddError(err, blunder.CorruptInodeError) - return - } - if !ok { - // disk corruption or client request for unallocated inode - err = fmt.Errorf("%s: failing request for inode %d volume '%s' because it is unallocated", - utils.GetFnName(), inodeNumber, vS.volumeName) - logger.InfoWithError(err) - err = blunder.AddError(err, blunder.NotFoundError) - return - } - - switch ourInode.InodeType { - case DirType, FileType: - tree, ok = ourInode.payload.(sortedmap.BPlusTree) - if !ok { - err = fmt.Errorf("type conversion of inode %v payload to sortedmap.BPlusTree failed", ourInode.InodeNumber) - err = blunder.AddError(err, blunder.CorruptInodeError) - _ = vS.markCorrupted(inodeNumber) - return - } - err = tree.Validate() - if nil != err { - err = blunder.AddError(err, blunder.CorruptInodeError) - _ = vS.markCorrupted(inodeNumber) - return - } - if FileType == ourInode.InodeType { - if deeply { - err = validateFileExtents(snapShotID, ourInode) - if nil != err { - err = blunder.AddError(err, blunder.CorruptInodeError) - _ = vS.markCorrupted(inodeNumber) - return - } - } - } - case SymlinkType: - // Nothing to be done here - default: - err = fmt.Errorf("unrecognized inode type") - err = blunder.AddError(err, blunder.CorruptInodeError) - _ = vS.markCorrupted(inodeNumber) - return - } - - err = nil - return -} diff --git a/inode/locker.go b/inode/locker.go deleted file mode 100644 index 6f32d157..00000000 --- a/inode/locker.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -// Lock-related wrappers for inodes -// -// These APIs wrap calls to package DLM, which is a generic locking package. -// -// These wrappers enforce a naming convention for lock IDs. -// - -import ( - "fmt" - - "github.com/NVIDIA/proxyfs/dlm" -) - -// MakeLockID creates the ID of an inode used in volume -func (vS *volumeStruct) MakeLockID(inodeNumber InodeNumber) (lockID string, err error) { - myLockID := fmt.Sprintf("vol.%s:ino.%d", vS.volumeName, inodeNumber) - - return myLockID, nil -} - -// InitInodeLock creates an inode lock. If callerID is non-nil, it is used. -// Otherwise a new callerID is allocated. -func (vS *volumeStruct) InitInodeLock(inodeNumber InodeNumber, callerID dlm.CallerID) (lock *dlm.RWLockStruct, err error) { - lockID, err := vS.MakeLockID(inodeNumber) - if err != nil { - return nil, err - } - - if callerID == nil { - callerID = dlm.GenerateCallerID() - } - - return &dlm.RWLockStruct{LockID: lockID, - Notify: nil, - LockCallerID: callerID, - }, nil -} - -// GetReadLock is a convenience function to create and acquire an inode lock -func (vS *volumeStruct) GetReadLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) { - lock, err := vS.InitInodeLock(inodeNumber, callerID) - if err != nil { - return nil, err - } - - err = lock.ReadLock() - return lock, err -} - -// GetWriteLock is a convenience function to create and acquire an inode lock -func (vS *volumeStruct) GetWriteLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) { - lock, err := vS.InitInodeLock(inodeNumber, callerID) - if err != nil { - return nil, err - } - - err = lock.WriteLock() - return lock, err -} - -// AttemptReadLock is a convenience function to create and try to acquire an inode lock -func (vS *volumeStruct) AttemptReadLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) { - lock, err := vS.InitInodeLock(inodeNumber, callerID) - if err != nil { - return nil, err - } - - err = lock.TryReadLock() - return lock, err -} - -// AttemptWriteLock is a convenience function to create and try to acquire an inode lock -func (vS *volumeStruct) AttemptWriteLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) { - lock, err := vS.InitInodeLock(inodeNumber, callerID) - if err != nil { - return nil, err - } - - err = lock.TryWriteLock() - return lock, err -} - -// EnsureReadLock ensures that a lock of the right type is held by the given callerID. If the lock is not held, it -// acquires it. If the lock is held, it returns nil (so you don't unlock twice; even if that's not crashworthy, you'd -// still release a lock that other code thinks it still holds). -func (vS *volumeStruct) EnsureReadLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) { - lock, err := vS.InitInodeLock(inodeNumber, callerID) - if err != nil { - return nil, err - } - - if lock.IsReadHeld() { - return nil, nil - } - - err = lock.ReadLock() - return lock, err -} - -// EnsureWriteLock ensures that a lock of the right type is held by the given callerID. If the lock is not held, it -// acquires it. If the lock is held, it returns nil (so you don't unlock twice; even if that's not crashworthy, you'd -// still release a lock that other code thinks it still holds). -func (vS *volumeStruct) EnsureWriteLock(inodeNumber InodeNumber, callerID dlm.CallerID) (*dlm.RWLockStruct, error) { - lock, err := vS.InitInodeLock(inodeNumber, callerID) - if err != nil { - return nil, err - } - - if lock.IsWriteHeld() { - return nil, nil - } - - err = lock.WriteLock() - return lock, err -} diff --git a/inode/locker_test.go b/inode/locker_test.go deleted file mode 100644 index 17adc468..00000000 --- a/inode/locker_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "testing" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/dlm" -) - -func TestLockerStuff(t *testing.T) { - var ( - inodeNumber InodeNumber - ) - - testSetup(t, false) - - inodeNumber = InodeNumber(2) - - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if nil != err { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") failed: %v", err) - } - - volume := testVolumeHandle.(*volumeStruct) - - // Test: get a lock for vol A inode 2 - srcDirLock, err := volume.InitInodeLock(inodeNumber, nil) - if err != nil { - t.Fatalf("Failed to initInodeLock: %v", err) - } - srcCallerID := srcDirLock.GetCallerID() - - // Test: get a lock for vol A inode 3 - inodeNumber++ - dstDirLock, err := volume.InitInodeLock(inodeNumber, nil) - if err != nil { - t.Fatalf("Failed to initInodeLock: %v", err) - } - dstCallerID := dstDirLock.GetCallerID() - - // Check that the caller IDs are different - if *dstCallerID == *srcCallerID { - t.Fatalf("Caller IDs should be different!") - } - - // Now try to lock something - srcDirLock.ReadLock() - srcDirLock.Unlock() - - // Simulate multi-lock move sequence - var srcDirInodeNumber InodeNumber = 9001 - var dstDirInodeNumber InodeNumber = 9002 - callerID := dlm.GenerateCallerID() - srcDirLock, err = volume.InitInodeLock(srcDirInodeNumber, callerID) - if err != nil { - return - } - srcCallerID = srcDirLock.GetCallerID() - - dstDirLock, err = volume.InitInodeLock(dstDirInodeNumber, callerID) - if err != nil { - return - } - dstCallerID = dstDirLock.GetCallerID() - - // Check that the caller IDs are the same - if *dstCallerID != *srcCallerID { - t.Fatalf("Caller IDs should be the same!") - } - -retryLock: - err = srcDirLock.WriteLock() - if err != nil { - return - } - - err = dstDirLock.TryWriteLock() - if blunder.Is(err, blunder.TryAgainError) { - srcDirLock.Unlock() - goto retryLock - } - - // Here is where we would do our move - - dstDirLock.Unlock() - srcDirLock.Unlock() - - testTeardown(t) -} diff --git a/inode/payload.go b/inode/payload.go deleted file mode 100644 index 48331c73..00000000 --- a/inode/payload.go +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "bytes" - "fmt" - - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - - "github.com/NVIDIA/cstruct" - "github.com/NVIDIA/sortedmap" -) - -// For DirType inodes, our payload tree-map is from basenames to inode numbers. -// For FileType inodes, our payload tree-map is from file offsets to file extents. - -// The signature of the pack/unpack methods says that failure is not an option -// (which is OK because it's reasonable to expect a well-defined data structure to -// have a well-defined serialization/deserialization), which is why our unpack -// methods have so many panic codepaths (that we expect to never run). - -type treeNodeLoadable struct { - inode *inMemoryInodeStruct -} - -func (tnl *treeNodeLoadable) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { - var ( - inodeNumberAdordnedWithSnapShotID uint64 - objectNumberAdordedWithSnapShotID uint64 - ) - - inodeNumberAdordnedWithSnapShotID = tnl.inode.volume.headhunterVolumeHandle.SnapShotIDAndNonceEncode(tnl.inode.snapShotID, uint64(tnl.inode.InodeNumber)) - objectNumberAdordedWithSnapShotID = tnl.inode.volume.headhunterVolumeHandle.SnapShotIDAndNonceEncode(tnl.inode.snapShotID, objectNumber) - - if 0 != objectOffset { - err = fmt.Errorf("*treeNodeLoadable.GetNode(): Unexpected (non-zero) objectOffset (%v)", objectOffset) - return - } - - stats.IncrementOperations(&stats.DirFileBPlusTreeNodeFaults) - evtlog.Record(evtlog.FormatDirFileBPlusTreeNodeFault, tnl.inode.volume.volumeName, inodeNumberAdordnedWithSnapShotID, objectNumberAdordedWithSnapShotID) - - nodeByteSlice, err = tnl.inode.volume.headhunterVolumeHandle.GetBPlusTreeObject(objectNumberAdordedWithSnapShotID) - - if (nil == err) && (uint64(len(nodeByteSlice)) != objectLength) { - err = fmt.Errorf("*treeNodeLoadable.GetNode(): Requested objectLength (%v) != Actual objectLength (%v)", objectLength, len(nodeByteSlice)) - return - } - - return -} - -func (tnl *treeNodeLoadable) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { - objectNumber = tnl.inode.volume.headhunterVolumeHandle.FetchNonce() - - objectOffset = 0 - - err = tnl.inode.volume.headhunterVolumeHandle.PutBPlusTreeObject(objectNumber, nodeByteSlice) - - return -} - -func (tnl *treeNodeLoadable) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { - logger.Tracef("inode.Discardnode(): volume '%s' inode %d: "+ - "root Object %016X discarding Object %016X len %d", - tnl.inode.volume.volumeName, tnl.inode.onDiskInodeV1Struct.InodeNumber, - tnl.inode.onDiskInodeV1Struct.PayloadObjectNumber, - objectNumber, objectLength) - - if 0 != objectOffset { - err = fmt.Errorf("*treeNodeLoadable.DiscardNode(): Unexpected (non-zero) objectOffset (%v)", objectOffset) - return - } - - // Tell Headhunter we are done with this persisted node - - err = tnl.inode.volume.headhunterVolumeHandle.DeleteBPlusTreeObject(objectNumber) - - return -} - -type fileInodeCallbacks struct { - treeNodeLoadable -} - -func (c *fileInodeCallbacks) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsUint64, ok := key.(uint64) - if !ok { - err = fmt.Errorf("fileInodeCallbacks.DumpKey() could not parse key as a uint64") - return - } - - keyAsString = fmt.Sprintf("0x%016X", keyAsUint64) - - err = nil - return -} - -func (c *fileInodeCallbacks) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - valueAsFileExtentPtr, ok := value.(*fileExtentStruct) - if !ok { - err = fmt.Errorf("fileInodeCallbacks.DumpValue() could not parse key as a *fileExtent") - return - } - - valueAsString = fmt.Sprintf("@%p: %#v", valueAsFileExtentPtr, valueAsFileExtentPtr) - - err = nil - return -} - -func (c *fileInodeCallbacks) PackKey(key sortedmap.Key) (packedKey []byte, err error) { - packedKey, err = cstruct.Pack(key, sortedmap.OnDiskByteOrder) - return -} - -func (c *fileInodeCallbacks) PackValue(value sortedmap.Value) (packedValue []byte, err error) { - fileExtent, ok := value.(*fileExtentStruct) - if !ok { - err = fmt.Errorf("PackValue() arg is not a *fileExtentStruct") - return - } - packedValue, err = cstruct.Pack(fileExtent, sortedmap.OnDiskByteOrder) - if nil != err { - return - } - if uint64(len(packedValue)) != globals.fileExtentStructSize { - err = fmt.Errorf("PackValue() should have produced len(packedValue) == %v", globals.fileExtentStructSize) - } - return -} - -func (c *fileInodeCallbacks) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { - var fileOffset uint64 - bytesConsumed, err = cstruct.Unpack(payloadData, &fileOffset, sortedmap.OnDiskByteOrder) - if nil != err { - return - } - key = fileOffset - return -} - -func (c *fileInodeCallbacks) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { - if uint64(len(payloadData)) < globals.fileExtentStructSize { - err = fmt.Errorf("UnpackValue() arg not big enough to encode fileExtentStruct") - return - } - valueAsFileExtentPtr := &fileExtentStruct{} - _, err = cstruct.Unpack(payloadData, valueAsFileExtentPtr, sortedmap.OnDiskByteOrder) - if nil != err { - return - } - value = valueAsFileExtentPtr - bytesConsumed = globals.fileExtentStructSize - err = nil - return -} - -type dirInodeCallbacks struct { - treeNodeLoadable -} - -func (c *dirInodeCallbacks) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsString, ok := key.(string) - - if ok { - err = nil - } else { - err = fmt.Errorf("dirInodeCallbacks.DumpKey() could not parse key as a string") - } - - return -} - -func (c *dirInodeCallbacks) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - valueAsUint64, ok := value.(InodeNumber) - if !ok { - err = fmt.Errorf("dirInodeCallbacks.DumpValue() could not parse value as a uint64") - return - } - - valueAsString = fmt.Sprintf("0x%016X", valueAsUint64) - - err = nil - return -} - -func (c *dirInodeCallbacks) PackKey(key sortedmap.Key) (packedKey []byte, err error) { - basename, ok := key.(string) - if !ok { - err = fmt.Errorf("PackKey() arg not a string") - return - } - packedKey = []byte(basename) - packedKey = append(packedKey, 0) // null terminator - err = nil - return -} - -func (c *dirInodeCallbacks) PackValue(value sortedmap.Value) (packedValue []byte, err error) { - valueAsInodeNumber, ok := value.(InodeNumber) - if !ok { - err = fmt.Errorf("PackValue() arg is not an InodeNumber") - return - } - valueAsUint64 := uint64(valueAsInodeNumber) - packedValue, err = cstruct.Pack(valueAsUint64, sortedmap.OnDiskByteOrder) - return -} - -func (c *dirInodeCallbacks) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { - basenameAndRemainderBytes := bytes.SplitN(payloadData, []byte{0}, 2) - basenameBytes := basenameAndRemainderBytes[0] - basename := string(basenameBytes) - key = basename - bytesConsumed = uint64(len(basenameBytes) + 1) - err = nil - return -} - -func (c *dirInodeCallbacks) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { - var valueAsUint64 uint64 - bytesConsumed, err = cstruct.Unpack(payloadData, &valueAsUint64, sortedmap.OnDiskByteOrder) - if nil != err { - return - } - value = InodeNumber(valueAsUint64) - bytesConsumed = 8 - return -} diff --git a/inode/setup_teardown_test.go b/inode/setup_teardown_test.go deleted file mode 100644 index d2c307f4..00000000 --- a/inode/setup_teardown_test.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "io/ioutil" - "os" - "runtime" - "sync" - "syscall" - "testing" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/ramswift" - "github.com/NVIDIA/proxyfs/transitions" -) - -var ( - ramswiftDoneChan chan bool - testConfMap conf.ConfMap -) - -func testSetup(t *testing.T, starvationMode bool) { - var ( - err error - rLimit syscall.Rlimit - rLimitMinimum uint64 - signalHandlerIsArmedWG sync.WaitGroup - testChunkedConnectionPoolSize uint64 - testConfStrings []string - testConfUpdateStrings []string - testDir string - ) - - testDir, err = ioutil.TempDir(os.TempDir(), "ProxyFS_test_inode_") - if nil != err { - t.Fatalf("ioutil.TempDir() failed: %v", err) - } - - err = os.Chdir(testDir) - if nil != err { - t.Fatalf("os.Chdir() failed: %v", err) - } - - testConfStrings = []string{ - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=35262", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=3", - "SwiftClient.RetryLimitObject=3", - "SwiftClient.RetryDelay=10ms", - "SwiftClient.RetryDelayObject=10ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=1.0", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000", - "Peer:Peer0.PublicIPAddr=127.0.0.1", - "Peer:Peer0.PrivateIPAddr=127.0.0.1", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - "Cluster.Peers=Peer0", - "Cluster.WhoAmI=Peer0", - "Volume:TestVolume.FSID=1", - "Volume:TestVolume.AccountName=AUTH_test", - "Volume:TestVolume.AutoFormat=true", - "Volume:TestVolume.CheckpointContainerName=.__checkpoint__", - "Volume:TestVolume.CheckpointContainerStoragePolicy=gold", - "Volume:TestVolume.CheckpointInterval=10s", - "Volume:TestVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way", - "Volume:TestVolume.MaxFlushSize=10485760", - "Volume:TestVolume.MaxFlushTime=10s", - "Volume:TestVolume.FileDefragmentChunkSize=10485760", - "Volume:TestVolume.FileDefragmentChunkDelay=10ms", - "Volume:TestVolume.NonceValuesToReserve=100", - "Volume:TestVolume.MaxEntriesPerDirNode=32", - "Volume:TestVolume.MaxExtentsPerFileNode=32", - "Volume:TestVolume.MaxInodesPerMetadataNode=32", - "Volume:TestVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:TestVolume.MaxDirFileNodesPerMetadataNode=16", - "Volume:TestVolume.MaxBytesInodeCache=100000", - "Volume:TestVolume.InodeCacheEvictInterval=1s", - "Volume:TestVolume.ActiveLeaseEvictLowLimit=5000", - "Volume:TestVolume.ActiveLeaseEvictHighLimit=5010", - "VolumeGroup:TestVolumeGroup.VolumeList=TestVolume", - "VolumeGroup:TestVolumeGroup.VirtualIPAddr=", - "VolumeGroup:TestVolumeGroup.PrimaryPeer=Peer0", - "VolumeGroup:TestVolumeGroup.ReadCacheLineSize=1000000", - "VolumeGroup:TestVolumeGroup.ReadCacheWeight=100", - "FSGlobals.VolumeGroupList=TestVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.DirEntryCacheEvictLowLimit=10000", - "FSGlobals.DirEntryCacheEvictHighLimit=10010", - "FSGlobals.FileExtentMapEvictLowLimit=10000", - "FSGlobals.FileExtentMapEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - if starvationMode { - testConfUpdateStrings = []string{ - "SwiftClient.ChunkedConnectionPoolSize=1", - "SwiftClient.NonChunkedConnectionPoolSize=1", - } - } else { - testConfUpdateStrings = []string{ - "SwiftClient.ChunkedConnectionPoolSize=256", - "SwiftClient.NonChunkedConnectionPoolSize=64", - } - } - - err = testConfMap.UpdateFromStrings(testConfUpdateStrings) - if nil != err { - t.Fatalf("testConfMap.UpdateFromStrings(testConfUpdateStrings) failed: %v", err) - } - - testChunkedConnectionPoolSize, err = testConfMap.FetchOptionValueUint64("SwiftClient", "ChunkedConnectionPoolSize") - if nil != err { - t.Fatalf("testConfMap.FetchOptionValueUint64(\"SwiftClient\", \"ChunkedConnectionPoolSize\") failed: %v", err) - } - - err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if nil != err { - t.Fatalf("syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)() failed: %v", err) - } - rLimitMinimum = 3 * testChunkedConnectionPoolSize - if rLimitMinimum > rLimit.Cur { - t.Errorf("RLIMIT too low... must be at least %v (was %v)", rLimitMinimum, rLimit.Cur) - t.Errorf("On Mac:") - t.Errorf(" ") - t.Errorf(" sysctl -a | grep ^kern.maxfiles") - t.Errorf(" ") - t.Errorf(" sudo /usr/libexec/PlistBuddy /Library/LaunchAgents/com.kern.maxfiles.plist -c \"add Label string com.kern.maxfiles\" -c \"add ProgramArguments array\" -c \"add ProgramArguments: string sysctl\" -c \"add ProgramArguments: string -w\" -c \"add ProgramArguments: string kern.maxfiles=20480\" -c \"add RunAtLoad bool true\"") - t.Errorf(" sudo /usr/libexec/PlistBuddy /Library/LaunchAgents/com.kern.maxfilesperproc.plist -c \"add Label string com.kern.maxfilesperproc\" -c \"add ProgramArguments array\" -c \"add ProgramArguments: string sysctl\" -c \"add ProgramArguments: string -w\" -c \"add ProgramArguments: string kern.perprocmaxfiles=10240\" -c \"add RunAtLoad bool true\"") - t.Errorf(" ") - t.Errorf(" sudo /usr/libexec/PlistBuddy /Library/LaunchAgents/com.launchd.maxfiles.plist -c \"add Label string com.launchd.maxfiles\" -c \"add ProgramArguments array\" -c \"add ProgramArguments: string launchctl\" -c \"add ProgramArguments: string limit\" -c \"add ProgramArguments: string maxfiles\" -c \"add ProgramArguments: string 5120\" -c \"add ProgramArguments: string unlimited\" -c \"add RunAtLoad bool true\"") - t.Errorf(" ") - t.Errorf(" ") - t.Errorf(" ulimit -n 2560") - t.Errorf("On Linux:") - t.Fatalf(" ulimit -n 2560") - } - - signalHandlerIsArmedWG.Add(1) - ramswiftDoneChan = make(chan bool, 1) - go ramswift.Daemon("/dev/null", testConfStrings, &signalHandlerIsArmedWG, ramswiftDoneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - err = transitions.Up(testConfMap) - if nil != err { - t.Fatalf("transitions.Up() failed: %v", err) - } -} - -func testTeardown(t *testing.T) { - var ( - err error - testDir string - ) - - err = transitions.Down(testConfMap) - if nil != err { - t.Fatalf("transitions.Down() failed: %v", err) - } - - _ = syscall.Kill(syscall.Getpid(), unix.SIGTERM) - _ = <-ramswiftDoneChan - - // Run GC to reclaim memory before we proceed to next test - runtime.GC() - - testDir, err = os.Getwd() - if nil != err { - t.Fatalf("os.Getwd() failed: %v", err) - } - - err = os.Chdir("..") - if nil != err { - t.Fatalf("os.Chdir() failed: %v", err) - } - - err = os.RemoveAll(testDir) - if nil != err { - t.Fatalf("os.RemoveAll() failed: %v", err) - } -} diff --git a/inode/symlink.go b/inode/symlink.go deleted file mode 100644 index d5abb19f..00000000 --- a/inode/symlink.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" -) - -func (vS *volumeStruct) CreateSymlink(target string, filePerm InodeMode, userID InodeUserID, groupID InodeGroupID) (symlinkInodeNumber InodeNumber, err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - // Create file mode out of file permissions plus inode type - fileMode, err := determineMode(filePerm, SymlinkType) - if err != nil { - return - } - - symlinkInode, err := vS.makeInMemoryInode(SymlinkType, fileMode, userID, groupID) - if err != nil { - return - } - - symlinkInode.dirty = true - - symlinkInode.SymlinkTarget = target - symlinkInodeNumber = symlinkInode.InodeNumber - - ok, err := vS.inodeCacheInsert(symlinkInode) - if nil != err { - return - } - if !ok { - err = fmt.Errorf("inodeCacheInsert(symlinkInode) failed") - return - } - - err = vS.flushInode(symlinkInode) - if err != nil { - logger.ErrorWithError(err) - return - } - - stats.IncrementOperations(&stats.SymlinkCreateOps) - - return -} - -func (vS *volumeStruct) GetSymlink(symlinkInodeNumber InodeNumber) (target string, err error) { - symlinkInode, err := vS.fetchInodeType(symlinkInodeNumber, SymlinkType) - if err != nil { - logger.ErrorWithError(err) - return - } - - target = symlinkInode.SymlinkTarget - - stats.IncrementOperations(&stats.SymlinkReadOps) - - return -} diff --git a/inode/validate_test.go b/inode/validate_test.go deleted file mode 100644 index 1aba9f32..00000000 --- a/inode/validate_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "strings" - "testing" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/swiftclient" -) - -// Helper to fetch a volume handle and create a file for a test. Return the -// volume handle and the file inode number. -func volumeAndFileInoForTest(t *testing.T) (VolumeHandle, InodeNumber) { - testVolumeHandle, err := FetchVolumeHandle("TestVolume") - if err != nil { - t.Fatalf("FetchVolumeHandle(\"TestVolume\") should have worked - got error: %v", err) - } - - fileInodeNumber, err := testVolumeHandle.CreateFile(PosixModePerm, 0, 0) - if err != nil { - t.Fatalf("CreateFile() failed: %v", err) - } - - err = testVolumeHandle.Write(fileInodeNumber, 0, []byte{0x00, 0x01, 0x02, 0x03, 0x04}, nil) - if err != nil { - t.Fatalf("Write(fileInodeNumber, 0, []byte{0x00, 0x01, 0x02, 0x03, 0x04}) failed: %v", err) - } - - err = testVolumeHandle.Flush(fileInodeNumber, true) - if err != nil { - t.Fatalf("Flush of inode %v failed", fileInodeNumber) - } - - return testVolumeHandle, fileInodeNumber -} - -func TestValidate(t *testing.T) { - testSetup(t, false) - - testVolumeHandle, fileInodeNumber := volumeAndFileInoForTest(t) - - // Validation succeeds if nothing has gone wrong - err := testVolumeHandle.Validate(fileInodeNumber, false) - if err != nil { - t.Fatalf("Validate() failed on presumably-good inode") - } - - // Now let's artificially make something go wrong and confirm that - // Validate() catches it! - - testVolume := testVolumeHandle.(*volumeStruct) - - // Grab the InodeRec - inodeRec, ok, err := testVolume.headhunterVolumeHandle.GetInodeRec(uint64(fileInodeNumber)) - if err != nil || !ok { - t.Fatalf("failed to fetch InodeRec for inode 0x%016X", fileInodeNumber) - } - - // Corrupt the non-preamble part - for i := len(globals.inodeRecDefaultPreambleBuf); i < len(inodeRec); i++ { - inodeRec[i] = 0xff - } - - // And put it back - err = testVolume.headhunterVolumeHandle.PutInodeRec(uint64(fileInodeNumber), inodeRec) - if err != nil { - t.Fatalf("failed to save corrupted InodeRec for inode 0x%016X", fileInodeNumber) - } - - // Now remove fileInodeNumber from inodeCache - fileInode, ok, err := testVolume.inodeCacheFetch(fileInodeNumber) - if err != nil { - t.Fatalf("inodeCacheFetch(fileInodeNumber) failed: %v", err) - } - if ok { - ok, err = testVolume.inodeCacheDrop(fileInode) - if err != nil { - t.Fatalf("inodeCacheDrop(fileInode) failed: %v", err) - } - if !ok { - t.Fatalf("inodeCacheDrop(fileInode) returned !ok") - } - } - - // Try to Validate, observe that it fails - validationErr := testVolumeHandle.Validate(fileInodeNumber, false) - if validationErr == nil { - t.Fatalf("expected validation to fail") - } - if blunder.IsNot(validationErr, blunder.CorruptInodeError) { - t.Fatalf("expected validation error %q to have error value %v, actual value was %v", validationErr, blunder.CorruptInodeError, blunder.FsError(blunder.Errno(validationErr)).String()) - } - - // Try to fetch from disk, observe that corruption was marked in the headhunter database - _, ok, corruptionErr := testVolume.fetchOnDiskInode(fileInodeNumber) - if corruptionErr == nil && ok { - t.Fatalf("expected not to get inode pointer when fetching presumptively corrupt inode %v", fileInodeNumber) - } - - testTeardown(t) -} - -func TestValidateFileExtents(t *testing.T) { - testSetup(t, false) - - testVolumeHandle, fileInodeNumber := volumeAndFileInoForTest(t) - - // then let's write some more data into the file so that there will be more segments to verify - testVolumeHandle.Write(fileInodeNumber, 2, []byte{0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06}, nil) - testVolumeHandle.Flush(fileInodeNumber, true) - testVolumeHandle.Write(fileInodeNumber, 12, []byte{0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07}, nil) - testVolumeHandle.Write(fileInodeNumber, 22, []byte{0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08}, nil) - testVolumeHandle.Flush(fileInodeNumber, true) - testVolumeHandle.Write(fileInodeNumber, 32, []byte{0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09}, nil) - testVolumeHandle.Write(fileInodeNumber, 39, []byte{0x0A, 0x0A, 0x0A, 0x0A, 0x0A, 0x0A, 0x0A, 0x0A, 0x0A}, nil) - testVolumeHandle.Write(fileInodeNumber, 48, []byte{0x0B, 0x0B, 0x0B, 0x0B, 0x0B, 0x0B}, nil) - testVolumeHandle.Flush(fileInodeNumber, false) - - err := testVolumeHandle.Validate(fileInodeNumber, true) - if err != nil { - t.Fatalf("Validate() failed on presumably-good inode") - } - - // let's get a read plan and then sneakily sabotage the log segments - var zero uint64 - readPlan, err := testVolumeHandle.GetReadPlan(fileInodeNumber, &zero, nil) - if err != nil { - t.Fatalf("failed to get read plan for inode %v", fileInodeNumber) - } - - // go delete the LogSegment referenced in the 1st readPlan step (that must exist) - - if 0 == len(readPlan) { - t.Fatalf("readPlan should have contained at least one entry") - } - - readPlanStep := readPlan[0] - - deleteErr := swiftclient.ObjectDelete(readPlanStep.AccountName, readPlanStep.ContainerName, readPlanStep.ObjectName, 0) - if nil != deleteErr { - t.Fatalf("HTTP DELETE %v should have worked... failed: %v", readPlanStep.ObjectPath, deleteErr) - } - - err = testVolumeHandle.Validate(fileInodeNumber, true) - if err == nil { - t.Fatalf("expected validation to fail!") - } - // TODO: validate error type more rigorously with blunder &c. - if !strings.Contains(err.Error(), "returned HTTP StatusCode 404") { - t.Fatalf("expected error to contain 'returned HTTP StatusCode 404'") - } - - testTeardown(t) -} diff --git a/inode/volume.go b/inode/volume.go deleted file mode 100644 index 3e8b3563..00000000 --- a/inode/volume.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package inode - -import ( - "fmt" - - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/sortedmap" -) - -func (vS *volumeStruct) GetFSID() (fsid uint64) { - fsid = vS.fsid - return -} - -func (vS *volumeStruct) SnapShotCreate(name string) (id uint64, err error) { - err = enforceRWMode(false) - if nil != err { - return - } - - if ("." == name) || (".." == name) { - err = fmt.Errorf("SnapShot cannot be named either '.' or '..'") - return - } - - vS.Lock() - id, err = vS.headhunterVolumeHandle.SnapShotCreateByInodeLayer(name) - vS.Unlock() - return -} - -func (vS *volumeStruct) SnapShotDelete(id uint64) (err error) { - var ( - found bool - keyAsInodeNumber InodeNumber - keyAsKey sortedmap.Key - indexWherePurgesAreToHappen int - maxInodeNumberToPurgeFromInodeCache InodeNumber - minInodeNumberToPurgeFromInodeCache InodeNumber - ok bool - valueAsValue sortedmap.Value - valueAsInodeStructPtr *inMemoryInodeStruct - ) - - err = enforceRWMode(false) - if nil != err { - return - } - - vS.Lock() - err = vS.headhunterVolumeHandle.SnapShotDeleteByInodeLayer(id) - if nil == err { - // Purge elements in inodeCache to avoid aliasing if/when SnapShotID is reused - - minInodeNumberToPurgeFromInodeCache = InodeNumber(vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(id, uint64(0))) - maxInodeNumberToPurgeFromInodeCache = InodeNumber(vS.headhunterVolumeHandle.SnapShotIDAndNonceEncode(id+1, uint64(0))) - - indexWherePurgesAreToHappen, found, err = vS.inodeCache.BisectRight(minInodeNumberToPurgeFromInodeCache) - if nil != err { - vS.Unlock() - err = fmt.Errorf("Volume %v InodeCache BisectRight() failed: %v", vS.volumeName, err) - logger.Error(err) - return - } - - // If there is some InodeNumber at or beyond minInodeNumberToPurgeFromInodeCache, found == TRUE - - for found { - keyAsKey, valueAsValue, ok, err = vS.inodeCache.GetByIndex(indexWherePurgesAreToHappen) - if nil != err { - vS.Unlock() - err = fmt.Errorf("Volume %v InodeCache GetByIndex() failed: %v", vS.volumeName, err) - logger.Error(err) - return - } - - if !ok { - vS.Unlock() - return - } - - keyAsInodeNumber, ok = keyAsKey.(InodeNumber) - if !ok { - vS.Unlock() - err = fmt.Errorf("Volume %v InodeCache GetByIndex() returned non-InodeNumber", vS.volumeName) - return - } - - // Redefine found to indicate we've found an InodeCache entry to evict (used next iteration) - - found = (keyAsInodeNumber < maxInodeNumberToPurgeFromInodeCache) - - // func (vS *volumeStruct) inodeCacheDropWhileLocked(inode *inMemoryInodeStruct) (ok bool, err error) - if found { - valueAsInodeStructPtr, ok = valueAsValue.(*inMemoryInodeStruct) - if !ok { - vS.Unlock() - err = fmt.Errorf("Volume %v InodeCache GetByIndex() returned non-inMemoryInodeStructPtr", vS.volumeName) - return - } - ok, err = vS.inodeCacheDropWhileLocked(valueAsInodeStructPtr) - if nil != err { - vS.Unlock() - err = fmt.Errorf("Volume %v inodeCacheDropWhileLocked() failed: %v", vS.volumeName, err) - return - } - if !ok { - vS.Unlock() - err = fmt.Errorf("Volume %v inodeCacheDropWhileLocked() returned !ok", vS.volumeName) - return - } - } - } - } - vS.Unlock() - return -} - -func (vS *volumeStruct) CheckpointCompleted() { - var ( - dirEntryCacheHitsDelta uint64 - dirEntryCacheMissesDelta uint64 - dirEntryCacheStats *sortedmap.BPlusTreeCacheStats - fileExtentMapCacheHitsDelta uint64 - fileExtentMapCacheMissesDelta uint64 - fileExtentMapCacheStats *sortedmap.BPlusTreeCacheStats - ) - - dirEntryCacheStats = globals.dirEntryCache.Stats() - fileExtentMapCacheStats = globals.fileExtentMapCache.Stats() - - globals.Lock() - - dirEntryCacheHitsDelta = dirEntryCacheStats.CacheHits - globals.dirEntryCachePriorCacheHits - dirEntryCacheMissesDelta = dirEntryCacheStats.CacheMisses - globals.dirEntryCachePriorCacheMisses - - fileExtentMapCacheHitsDelta = fileExtentMapCacheStats.CacheHits - globals.fileExtentMapCachePriorCacheHits - fileExtentMapCacheMissesDelta = fileExtentMapCacheStats.CacheMisses - globals.fileExtentMapCachePriorCacheMisses - - if 0 != dirEntryCacheHitsDelta { - stats.IncrementOperationsBy(&stats.DirEntryCacheHits, dirEntryCacheHitsDelta) - globals.dirEntryCachePriorCacheHits = dirEntryCacheStats.CacheHits - } - if 0 != dirEntryCacheMissesDelta { - stats.IncrementOperationsBy(&stats.DirEntryCacheMisses, dirEntryCacheMissesDelta) - globals.dirEntryCachePriorCacheMisses = dirEntryCacheStats.CacheMisses - } - - if 0 != fileExtentMapCacheHitsDelta { - stats.IncrementOperationsBy(&stats.FileExtentMapCacheHits, fileExtentMapCacheHitsDelta) - globals.fileExtentMapCachePriorCacheHits = fileExtentMapCacheStats.CacheHits - } - if 0 != fileExtentMapCacheMissesDelta { - stats.IncrementOperationsBy(&stats.FileExtentMapCacheMisses, fileExtentMapCacheMissesDelta) - globals.fileExtentMapCachePriorCacheMisses = fileExtentMapCacheStats.CacheMisses - } - - globals.Unlock() -} diff --git a/inodeworkout/Makefile b/inodeworkout/Makefile deleted file mode 100644 index 38103373..00000000 --- a/inodeworkout/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/inodeworkout - -include ../GoMakefile diff --git a/inodeworkout/dummy_test.go b/inodeworkout/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/inodeworkout/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/inodeworkout/main.go b/inodeworkout/main.go deleted file mode 100644 index 1a1e9d22..00000000 --- a/inodeworkout/main.go +++ /dev/null @@ -1,427 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "os" - "runtime" - "strconv" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -const ( - dirInodeNamePrefix = "__inodeworkout_dir_" - fileInodeNamePrefix = "__inodeworkout_file_" -) - -var ( - doNextStepChan chan bool - inodesPerThread uint64 - measureCreate bool - measureDestroy bool - measureStat bool - perThreadDir bool - rootDirMutex trackedlock.Mutex - stepErrChan chan error - threads uint64 - volumeHandle inode.VolumeHandle - volumeName string -) - -func usage(file *os.File) { - fmt.Fprintf(file, "Usage:\n") - fmt.Fprintf(file, " %v [cCsSdD] threads inodes-per-thread conf-file [section.option=value]*\n", os.Args[0]) - fmt.Fprintf(file, " where:\n") - fmt.Fprintf(file, " c run create test in common root dir\n") - fmt.Fprintf(file, " C run create test in per thread dir\n") - fmt.Fprintf(file, " s run stat test in common root dir\n") - fmt.Fprintf(file, " S run stat test in per thread dir\n") - fmt.Fprintf(file, " d run destroy test in common root dir\n") - fmt.Fprintf(file, " D run destroy test in per thread dir\n") - fmt.Fprintf(file, " threads number of threads\n") - fmt.Fprintf(file, " inodes-per-thread number of inodes each thread will reference\n") - fmt.Fprintf(file, " conf-file input to conf.MakeConfMapFromFile()\n") - fmt.Fprintf(file, " [section.option=value]* optional input to conf.UpdateFromStrings()\n") - fmt.Fprintf(file, "\n") - fmt.Fprintf(file, "Note: Precisely one test selector must be specified\n") - fmt.Fprintf(file, " It is expected that c, s, then d are run in sequence\n") - fmt.Fprintf(file, " or that C, S, then D are run in sequence\n") - fmt.Fprintf(file, " It is expected that cleanproxyfs is run before & after the sequence\n") -} - -func main() { - var ( - confMap conf.ConfMap - durationOfMeasuredOperations time.Duration - err error - latencyPerOpInMilliSeconds float64 - opsPerSecond float64 - primaryPeer string - timeAfterMeasuredOperations time.Time - timeBeforeMeasuredOperations time.Time - volumeGroupToCheck string - volumeGroupToUse string - volumeGroupList []string - volumeList []string - whoAmI string - ) - - // Parse arguments - - if 5 > len(os.Args) { - usage(os.Stderr) - os.Exit(1) - } - - switch os.Args[1] { - case "c": - measureCreate = true - case "C": - measureCreate = true - perThreadDir = true - case "s": - measureStat = true - case "S": - measureStat = true - perThreadDir = true - case "d": - measureDestroy = true - case "D": - measureDestroy = true - perThreadDir = true - default: - fmt.Fprintf(os.Stderr, "os.Args[1] ('%v') must be one of 'c', 'C', 'r', 'R', 'd', or 'D'\n", os.Args[1]) - os.Exit(1) - } - - threads, err = strconv.ParseUint(os.Args[2], 10, 64) - if nil != err { - fmt.Fprintf(os.Stderr, "strconv.ParseUint(\"%v\", 10, 64) of threads failed: %v\n", os.Args[2], err) - os.Exit(1) - } - if 0 == threads { - fmt.Fprintf(os.Stderr, "threads must be a positive number\n") - os.Exit(1) - } - - inodesPerThread, err = strconv.ParseUint(os.Args[3], 10, 64) - if nil != err { - fmt.Fprintf(os.Stderr, "strconv.ParseUint(\"%v\", 10, 64) of inodes-per-thread failed: %v\n", os.Args[3], err) - os.Exit(1) - } - if 0 == inodesPerThread { - fmt.Fprintf(os.Stderr, "inodes-per-thread must be a positive number\n") - os.Exit(1) - } - - confMap, err = conf.MakeConfMapFromFile(os.Args[4]) - if nil != err { - fmt.Fprintf(os.Stderr, "conf.MakeConfMapFromFile(\"%v\") failed: %v\n", os.Args[4], err) - os.Exit(1) - } - - if 5 < len(os.Args) { - err = confMap.UpdateFromStrings(os.Args[5:]) - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.UpdateFromStrings(%#v) failed: %v\n", os.Args[5:], err) - os.Exit(1) - } - } - - // Upgrade confMap if necessary - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "Failed to upgrade config: %v", err) - os.Exit(1) - } - - // Start up needed ProxyFS components - - err = transitions.Up(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "transitions.Up() failed: %v\n", err) - os.Exit(1) - } - - // Select first Volume of the first "active" VolumeGroup in [FSGlobals]VolumeGroupList - - whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueString(\"Cluster\", \"WhoAmI\") failed: %v\n", err) - os.Exit(1) - } - - volumeGroupList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"FSGlobals\", \"VolumeGroupList\") failed: %v\n", err) - os.Exit(1) - } - - volumeGroupToUse = "" - - for _, volumeGroupToCheck = range volumeGroupList { - primaryPeer, err = confMap.FetchOptionValueString("VolumeGroup:"+volumeGroupToCheck, "PrimaryPeer") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueString(\"VolumeGroup:%s\", \"PrimaryPeer\") failed: %v\n", volumeGroupToCheck, err) - os.Exit(1) - } - if whoAmI == primaryPeer { - volumeGroupToUse = volumeGroupToCheck - break - } - } - - if "" == volumeGroupToUse { - fmt.Fprintf(os.Stderr, "confMap didn't contain an \"active\" VolumeGroup") - os.Exit(1) - } - - volumeList, err = confMap.FetchOptionValueStringSlice("VolumeGroup:"+volumeGroupToUse, "VolumeList") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"VolumeGroup:%s\", \"PrimaryPeer\") failed: %v\n", volumeGroupToUse, err) - os.Exit(1) - } - if 1 > len(volumeList) { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"VolumeGroup:%s\", \"VolumeList\") returned empty volumeList", volumeGroupToUse) - os.Exit(1) - } - - volumeName = volumeList[0] - - volumeHandle, err = inode.FetchVolumeHandle(volumeName) - if nil != err { - fmt.Fprintf(os.Stderr, "inode.FetchVolumeHandle(\"%value\") failed: %v\n", volumeName, err) - os.Exit(1) - } - - // Perform tests - - stepErrChan = make(chan error, 0) //threads) - doNextStepChan = make(chan bool, 0) //threads) - - // Do initialization step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - go inodeWorkout(threadIndex) - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "inodeWorkout() initialization step returned: %v\n", err) - os.Exit(1) - } - } - - // Do measured operations step - timeBeforeMeasuredOperations = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "inodeWorkout() measured operations step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterMeasuredOperations = time.Now() - - // Do shutdown step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "inodeWorkout() shutdown step returned: %v\n", err) - os.Exit(1) - } - } - - // Stop ProxyFS components launched above - - err = transitions.Down(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "transitions.Down() failed: %v\n", err) - os.Exit(1) - } - - // Report results - - durationOfMeasuredOperations = timeAfterMeasuredOperations.Sub(timeBeforeMeasuredOperations) - - opsPerSecond = float64(threads*inodesPerThread*1000*1000*1000) / float64(durationOfMeasuredOperations.Nanoseconds()) - latencyPerOpInMilliSeconds = float64(durationOfMeasuredOperations.Nanoseconds()) / float64(inodesPerThread*1000*1000) - - fmt.Printf("opsPerSecond = %10.2f\n", opsPerSecond) - fmt.Printf("latencyPerOp = %10.2f ms\n", latencyPerOpInMilliSeconds) -} - -func inodeWorkout(threadIndex uint64) { - var ( - dirInodeName string - dirInodeNumber inode.InodeNumber - err error - fileInodeName []string - fileInodeNumber []inode.InodeNumber - i uint64 - toDestroyInodeNumber inode.InodeNumber - ) - - // Do initialization step - if perThreadDir { - dirInodeName = fmt.Sprintf("%s%016X", dirInodeNamePrefix, threadIndex) - rootDirMutex.Lock() - if measureCreate { - dirInodeNumber, err = volumeHandle.CreateDir(inode.PosixModePerm, inode.InodeUserID(0), inode.InodeGroupID(0)) - if nil != err { - rootDirMutex.Unlock() - stepErrChan <- err - runtime.Goexit() - } - err = volumeHandle.Link(inode.RootDirInodeNumber, dirInodeName, dirInodeNumber, false) - if nil != err { - rootDirMutex.Unlock() - stepErrChan <- err - runtime.Goexit() - } - } else { // measureStat || measureDestroy - dirInodeNumber, err = volumeHandle.Lookup(inode.RootDirInodeNumber, dirInodeName) - if nil != err { - rootDirMutex.Unlock() - stepErrChan <- err - runtime.Goexit() - } - } - rootDirMutex.Unlock() - } else { // !perThreadDir - dirInodeNumber = inode.RootDirInodeNumber - } - fileInodeName = make([]string, inodesPerThread) - fileInodeNumber = make([]inode.InodeNumber, inodesPerThread) - for i = 0; i < inodesPerThread; i++ { - fileInodeName[i] = fmt.Sprintf("%s%016X_%016X", fileInodeNamePrefix, threadIndex, i) - } - - // Indicate initialization step is done - stepErrChan <- nil - - // Await signal to proceed with measured operations step - _ = <-doNextStepChan - - // Do measured operations - for i = 0; i < inodesPerThread; i++ { - if measureCreate { - fileInodeNumber[i], err = volumeHandle.CreateFile(inode.PosixModePerm, inode.InodeUserID(0), inode.InodeGroupID(0)) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - if !perThreadDir { - rootDirMutex.Lock() - } - err = volumeHandle.Link(dirInodeNumber, fileInodeName[i], fileInodeNumber[i], false) - if nil != err { - if !perThreadDir { - rootDirMutex.Unlock() - } - stepErrChan <- err - runtime.Goexit() - } - if !perThreadDir { - rootDirMutex.Unlock() - } - } else if measureStat { - if !perThreadDir { - rootDirMutex.Lock() - } - fileInodeNumber[i], err = volumeHandle.Lookup(dirInodeNumber, fileInodeName[i]) - if nil != err { - if !perThreadDir { - rootDirMutex.Unlock() - } - stepErrChan <- err - runtime.Goexit() - } - if !perThreadDir { - rootDirMutex.Unlock() - } - _, err = volumeHandle.GetMetadata(fileInodeNumber[i]) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - } else { // measureDestroy - if !perThreadDir { - rootDirMutex.Lock() - } - fileInodeNumber[i], err = volumeHandle.Lookup(dirInodeNumber, fileInodeName[i]) - if nil != err { - if !perThreadDir { - rootDirMutex.Unlock() - } - stepErrChan <- err - runtime.Goexit() - } - toDestroyInodeNumber, err = volumeHandle.Unlink(dirInodeNumber, fileInodeName[i], false) - if nil != err { - if !perThreadDir { - rootDirMutex.Unlock() - } - stepErrChan <- err - runtime.Goexit() - } - if !perThreadDir { - rootDirMutex.Unlock() - } - if toDestroyInodeNumber != fileInodeNumber[i] { - err = fmt.Errorf("volumeHandle.Unlink(dirInodeNumber, fileInodeName[i], false) should have returned toDestroyInodeNumber == fileInodeNumber[i]") - stepErrChan <- err - runtime.Goexit() - } - err = volumeHandle.Destroy(fileInodeNumber[i]) - if nil != err { - stepErrChan <- err - runtime.Goexit() - } - } - } - - // Indicate measured operations step is done - stepErrChan <- nil - - // Await signal to proceed with shutdown step - _ = <-doNextStepChan - - // Do shutdown step - if perThreadDir && measureDestroy { - rootDirMutex.Lock() - toDestroyInodeNumber, err = volumeHandle.Unlink(inode.RootDirInodeNumber, dirInodeName, false) - if nil != err { - rootDirMutex.Unlock() - stepErrChan <- err - runtime.Goexit() - } - if toDestroyInodeNumber != dirInodeNumber { - err = fmt.Errorf("volumeHandle.Unlink(dirInodeNumber, fileInodeName[i], false) should have returned toDestroyInodeNumber == dirInodeNumber") - stepErrChan <- err - runtime.Goexit() - } - err = volumeHandle.Destroy(dirInodeNumber) - if nil != err { - rootDirMutex.Unlock() - stepErrChan <- err - runtime.Goexit() - } - rootDirMutex.Unlock() - } - - // Indicate shutdown step is done - stepErrChan <- nil -} diff --git a/jrpcfs/Makefile b/jrpcfs/Makefile deleted file mode 100644 index 37cc1843..00000000 --- a/jrpcfs/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/jrpcfs - -include ../GoMakefile diff --git a/jrpcfs/api.go b/jrpcfs/api.go deleted file mode 100644 index 244bd5f4..00000000 --- a/jrpcfs/api.go +++ /dev/null @@ -1,1010 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package jrpcfs implements a JSON RPC interface to package fs. -// -// The structs in this file are used as parameters to the Server methods -// found in filesystem.go and middleware.go. -// -// NOTE: Please try to keep the definitions in this file in alphabetical order. -// -// -// Using the JSON RPC APIs from outside of proxyfsd: -// -// - The types defined here and in filesystem.go are not available outside of proxyfs, -// so one must use the JSON RPC, which is defined here: -// https://en.wikipedia.org/wiki/JSON-RPC -// -// - JSON RPC expects a request with the following format: -// "method": -// "params": [] -// "jsonrpc": "2.0" -// "id": -// -// NOTE: the "id" here is just used to match request/response. -// -// - A JSON RPC response will look like this: -// "id": -// "error": -// "result": -// -// -// EXAMPLE: -// -// As an example, let's look at doing a mount. -// -// The Go-side definitions of interest are: -// -// From this file: -// type MountRequestByVolumeName struct { -// VolumeName string -// MountOptions uint64 -// } -// -// type MountReply struct { -// MountID MountIDAsString -// RootDirInodeNumber uint64 -// } -// -// From filesystem.go: -// func (s *Server) RpcMount(in *MountRequest, reply *MountReply) error -// -// -// It may be easiest to represent what needs to be sent to the proxyfs RPC -// server in Python: -// -// # The args for RpcMount are defined in MountRequest. The arg names used -// # must be exactly the same as the names in the Go-side struct. -// # -// # For the expected type for each argument, see the definition of the -// # appropriate request/response struct in this file. -// # -// args = {'VolumeName' : "CommonVolume", 'MountOptions': 0} -// -// # Start our client-side request numbering at 0 -// # -// id = 0 -// -// # This will become the JSON request once we encode it -// # -// payload = { -// "method": "Server.RpcMountByVolumeName", # This will always be "Server." -// "params": [args], # Args must be encoded in an array here! -// "jsonrpc": "2.0", # JSON RPC version -// "id": id, # Client request id -// } -// -// # Encode payload into JSON -// # -// data = json.dumps((payload)) -// -// # Send request over socket. Ignore socket specifics here. -// # -// s = socket.create_connection(("localhost", 12345)) -// s.sendall(data) -// -// Now we receive and decode the response from the proxyfsd RPC server. -// -// -// # Read response from socket. Ignore socket specifics here. -// # -// # (Note that the size of 1024 here will not work for all calls; this -// # is just a simple example). -// # -// rdata = s.recv(1024) -// -// # Decode response out of JSON -// # -// resp = json.loads(rdata) -// -// # Check that id from response is the same as our request -// # -// if resp["id"] != id: -// raise Exception("expected id=%s, received id=%s: %s" %(id, resp["id"], resp["error"])) -// -// # Check whether there was an error in handling the request -// # -// if resp["error"] is not None: -// raise Exception(resp["error"]) -// -// # If we got this far, we can check out the response. The response -// # contents will consist of the content of the response struct for -// # this particular request. In this case, that is the contents of -// # the MountReply structure. -// # -// # Note that it's generally good practice to check for the presence -// # of the key before using the value... -// # -// print "Returned MountID: ", resp["result"]["MountID"] -// print "Returned rootInode:", resp["result"]["RootDirInodeNumber"] -// -// On the C-side, jrpcclient leverages the popular json-c library. One glaring ommission of json-c is -// support for unsigned integers... specifically uint64_t's. The parser actually substitutes for any -// "number" that is bigger than math.MaxInt64 with math.MaxInt64 - rather than converting to the -// bit-compatible uint64_t interpretation. It's a mystery why... but this choice has spurred several -// to request json-c expand to directly support (particularly) uint64_t. Efforts have been started but, -// alas, never completed. -// -// The ProxyFS work-around will be to pass vulnerable uint64's that have practical cases where the -// upper-most bit (the "sign" bit if it was an int64) as int64's. The int64 value will be the equivalent -// such that casting between int64_t's and uint64_t's will result in the desired value. It just so -// happens that jrpcclient is already doing the casting back and forth, so making that possible on the -// (here) Go side resolves the issue. -// -// The impact on this change for the other JSON RPC client, pfs_middleware, should not be noticable so -// long as the cases where a uint64 comes across as a negative int64 are opaque to pfs_middleware. -// -// It turns out that all uint64's previously in the jrpcfs-specified RPC (i.e. those in api.go) fall -// into three categories: -// -// Practically never > math.MaxInt64 - e.g. Stat.Size -// Possibly > math.MaxInt64 - specifically SnapShotIDs adorning InodeNumbers -// -// In the "Possibly" category, the InodeNumbers are the worry here. Fortunately, InodeNumbers are -// considered "opaque" handles to ProxyFS resources and, as such, only need to preserve this identity -// property (whether signed or unsigned). For this reason, all InodeNumbers in the following API are -// passed as int64's rather than uint64's. In the case where the InodeNumber > math.MaxInt64, care -// is taken such that the negative value passed via the int64 is cast to the proper (large) uint64 -// on each side of the RPC consistently. -// -package jrpcfs - -import ( - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/inode" -) - -// The first section of this file defines the structs used by protocols like Samba. -// The API for this section is implemented in filesystem.go. -// -// Structs used by Swift middleware are defined below. - -// MountID is embedded in a number of request objects (as well as InodeHandle & PathHandle) -// -// For Reads/Writes, the binary form of MountID will be used -// For Non-Reads/Writes, a base64.StdEncoding.Encode() of the binary form of MountID will be used -// -type MountIDAsByteArray [16]byte -type MountIDAsString string - -// InodeHandle is embedded in a number of the request objects. -type InodeHandle struct { - MountID MountIDAsString - InodeNumber int64 -} - -// PathHandle is embedded in a number of the request objects. -type PathHandle struct { - MountID MountIDAsString - Fullpath string -} - -// ChmodRequest is the request object for RpcChmod. -type ChmodRequest struct { - InodeHandle - FileMode uint32 -} - -// ChmodPathRequest is the request object for RpcChmodPath. -type ChmodPathRequest struct { - PathHandle - FileMode uint32 -} - -// ChownRequest is the request object for RpcChown. -type ChownRequest struct { - InodeHandle - UserID int32 - GroupID int32 -} - -// ChownPathRequest is the request object for RpcChownPath. -type ChownPathRequest struct { - PathHandle - UserID int32 - GroupID int32 -} - -// CreateRequest is the request object for RpcCreate. -type CreateRequest struct { - InodeHandle - Basename string - UserID int32 - GroupID int32 - FileMode uint32 -} - -// CreatePathRequest is the request object for RpcCreatePath. -type CreatePathRequest struct { - PathHandle - UserID int32 - GroupID int32 - FileMode uint32 -} - -// DirEntry is used as part of ReaddirReply and ReaddirPlusReply. -// -// FileType here will be a uint16 containing DT_DIR|DT_REG|DT_LNK. -// -type DirEntry struct { - InodeNumber int64 - FileType uint16 - Basename string - NextDirLocation int64 -} - -// FetchExtentMapChunkRequest is the request object for RpcFetchExtentMapChunk. -type FetchExtentMapChunkRequest struct { - InodeHandle - FileOffset uint64 - MaxEntriesFromFileOffset int64 - MaxEntriesBeforeFileOffset int64 -} - -// FetchExtentMapChunkReply is the response object for RpcFetchExtentMapChunk. -type FetchExtentMapChunkReply struct { - FileOffsetRangeStart uint64 // Holes in [FileOffsetRangeStart:FileOffsetRangeEnd) - FileOffsetRangeEnd uint64 // not covered in ExtentMapEntry slice should "read-as-zero" - FileSize uint64 // up to the end-of-file as indicated by FileSize - ExtentMapEntry []inode.ExtentMapEntryStruct // All will be in [FileOffsetRangeStart:FileOffsetRangeEnd) -} - -// FlushRequest is the request object for RpcFlush. -type FlushRequest struct { - InodeHandle - SendTimeSec int64 - SendTimeNsec int64 -} - -// GetStatRequest is the request object for RpcGetStat. -type GetStatRequest struct { - InodeHandle -} - -// GetStatPathRequest is the request object for RpcGetStatPath. -type GetStatPathRequest struct { - PathHandle -} - -type GetXAttrRequest struct { - InodeHandle - AttrName string -} - -type GetXAttrPathRequest struct { - PathHandle - AttrName string -} - -type GetXAttrReply struct { - AttrValueSize uint64 - AttrValue []byte -} - -type FlockRequest struct { - InodeHandle - FlockCmd int32 - FlockType int32 - FlockWhence int32 - FlockStart uint64 - FlockLen uint64 - FlockPid uint64 -} - -type FlockReply struct { - FlockType int32 - FlockWhence int32 - FlockStart uint64 - FlockLen uint64 - FlockPid uint64 -} - -// InodeReply is the reply object for requests that return an inode number. -// This response object is used by a number of the methods. -type InodeReply struct { - InodeNumber int64 -} - -// LogRequest is the request object for RpcLog. -type LogRequest struct { - Message string -} - -// LookupPathRequest is the request object for RpcLookupPath. -type LookupPathRequest struct { - MountID MountIDAsString - Fullpath string -} - -// LinkRequest is the request object for RpcLink. -type LinkRequest struct { - InodeHandle - Basename string - TargetInodeNumber int64 -} - -// LinkPathRequest is the request object for RpcLinkPath. -type LinkPathRequest struct { - PathHandle - TargetFullpath string -} - -type ListXAttrRequest struct { - InodeHandle -} - -type ListXAttrPathRequest struct { - PathHandle -} - -type ListXAttrReply struct { - AttrNames []string -} - -// LookupRequest is the request object for RpcLookup. -type LookupRequest struct { - InodeHandle - Basename string -} - -// LookupPlusRequest is the request object for RpcLookupPlus. -type LookupPlusRequest struct { - InodeHandle - Basename string -} - -// LookupPlusReply is the reply object for RpcLookupPlus. -type LookupPlusReply struct { - InodeNumber int64 - StatStruct -} - -// AccessRequest is the request object for RpcAccess. -type AccessRequest struct { - InodeHandle - UserID int32 - GroupID int32 - AccessMode uint32 -} - -// MkdirRequest is the request object for RpcMkdir. -type MkdirRequest struct { - InodeHandle - Basename string - UserID int32 - GroupID int32 - FileMode uint32 -} - -// MkdirPathRequest is the request object for RpcMkdirPath. -type MkdirPathRequest struct { - PathHandle - UserID int32 - GroupID int32 - FileMode uint32 -} - -// MountByAccountNameRequest is the request object for RpcMountByAccountName. -type MountByAccountNameRequest struct { - AccountName string - AuthToken string -} - -// MountByAccountNameReply is the reply object for RpcMountByAccountName. -type MountByAccountNameReply struct { - MountID MountIDAsString -} - -// MountByVolumeNameRequest is the request object for RpcMountByVolumeName. -type MountByVolumeNameRequest struct { - VolumeName string - AuthToken string -} - -// MountByVolumeNameReply is the reply object for RpcMountByVolumeName. -type MountByVolumeNameReply struct { - MountID MountIDAsString -} - -// UnmountRequest is the request object for RpcUnmount. -// -// Note that all leases are implicitly released as part of servicing this request. -type UnmountRequest struct { - MountID MountIDAsString -} - -// ReaddirRequest is the request object for RpcReaddir. -type ReaddirRequest struct { - InodeHandle - MaxEntries uint64 - PrevDirEntName string -} - -// ReaddirByLocRequest is the request object for RpcReaddirByLoc. -type ReaddirByLocRequest struct { - InodeHandle - MaxEntries uint64 - PrevDirEntLocation int64 -} - -// ReaddirReply is the reply object for RpcReaddir and RpcReaddirByLoc. -type ReaddirReply struct { - DirEnts []DirEntry -} - -// ReaddirPlusRequest is the request object for RpcReaddirPlus. -type ReaddirPlusRequest struct { - InodeHandle - MaxEntries uint64 - PrevDirEntName string -} - -// ReaddirPlusByLocRequest is the request object for RpcReaddirPlusByLoc. -type ReaddirPlusByLocRequest struct { - InodeHandle - MaxEntries uint64 - PrevDirEntLocation int64 -} - -// ReaddirPlusReply is the reply object for RpcReaddirPlus and RpcReaddirPlusByLoc. -type ReaddirPlusReply struct { - DirEnts []DirEntry - StatEnts []StatStruct -} - -// ReadSymlinkRequest is the request object for RpcReadSymlink. -type ReadSymlinkRequest struct { - InodeHandle -} - -// ReadSymlinkPathRequest is the request object for RpcReadSymlinkPath. -type ReadSymlinkPathRequest struct { - PathHandle -} - -// ReadSymlinkReply is the reply object for RpcReadSymlink and RpcReadSymlinkPath. -type ReadSymlinkReply struct { - Target string -} - -type RemoveXAttrRequest struct { - InodeHandle - AttrName string -} - -type RemoveXAttrPathRequest struct { - PathHandle - AttrName string -} - -// RenameRequest is the request object for RpcRename. -type RenameRequest struct { - MountID MountIDAsString - SrcDirInodeNumber int64 - SrcBasename string - DstDirInodeNumber int64 - DstBasename string -} - -// RenamePathRequest is the request object for RpcRenamePath. -type RenamePathRequest struct { - PathHandle - DstFullpath string -} - -// MoveRequest is the request object for RpcMove. -// -// Note the similarity with RenameRequest except that implicit Destroy of an Inode -// when a replacement DirEntry reduces the prior DirEntry's Inode LinkCount to zero -// is not performed... instead leaving it up to the client to do so. -// -type MoveRequest struct { - MountID MountIDAsString - SrcDirInodeNumber int64 - SrcBasename string - DstDirInodeNumber int64 - DstBasename string -} - -// MoveReply is the reply object for RpcMove. -type MoveReply struct { - ToDestroyInodeNumber int64 -} - -// DestroyRequest is teh request object for RpcDestroy. -type DestroyRequest struct { - InodeHandle -} - -// Reply is a generic response object used when no values need to be returned. -// This response object is used by a number of the methods. -type Reply struct { - RequestTimeSec int64 - RequestTimeNsec int64 - SendTimeSec int64 - SendTimeNsec int64 -} - -// ResizeRequest is the request object for RpcResize. -type ResizeRequest struct { - InodeHandle - NewSize uint64 -} - -// SetstatRequest is the request object for RpcSetstat. -type SetstatRequest struct { - InodeHandle - StatStruct -} - -// SetTimeRequest is the request object for RpcSetTime. -type SetTimeRequest struct { - InodeHandle - StatStruct -} - -// SetTimePathRequest is the request object for RpcSetTimePath. -type SetTimePathRequest struct { - PathHandle - StatStruct -} - -type SetXAttrRequest struct { - InodeHandle - AttrName string - AttrValue []byte - AttrFlags int -} - -type SetXAttrPathRequest struct { - PathHandle - AttrName string - AttrValue []byte - AttrFlags int -} - -// StatVFSRequest is the request object for RpcStatVFS. -type StatVFSRequest struct { - MountID MountIDAsString -} - -// StatVFS is used when filesystem stats need to be conveyed. It is used by RpcStatVFS. -type StatVFS struct { - BlockSize uint64 - FragmentSize uint64 - TotalBlocks uint64 - FreeBlocks uint64 - AvailBlocks uint64 - TotalInodes uint64 - FreeInodes uint64 - AvailInodes uint64 - FileSystemID uint64 - MountFlags uint64 - MaxFilenameLen uint64 -} - -// StatStruct is used when stats need to be conveyed. It is used as the response to RpcGetStat and RpcGetStatPath, -// as well as in RpcSetStat and RpcReaddirPlus. -// -// Note that times are conveyed as nanoseconds since epoch. -// -type StatStruct struct { - CTimeNs uint64 - CRTimeNs uint64 - MTimeNs uint64 - ATimeNs uint64 - Size uint64 - NumLinks uint64 - StatInodeNumber int64 - FileMode uint32 - UserID uint32 - GroupID uint32 -} - -// SymlinkRequest is the request object for RpcSymlink. -type SymlinkRequest struct { - InodeHandle - Basename string - Target string - UserID int32 - GroupID int32 -} - -// SymlinkPathRequest is the request object for RpcSymlinkPath. -type SymlinkPathRequest struct { - PathHandle - TargetFullpath string - UserID int32 - GroupID int32 -} - -// TypeRequest is the request object for RpcType. -type TypeRequest struct { - InodeHandle -} - -// TypeReply is the reply object for RpcType. -// -// FileType here will be a uint16 containing DT_DIR|DT_REG|DT_LNK. -// -type TypeReply struct { - FileType uint16 -} - -// UnlinkRequest is the request object for RpcUnlink & RpcRmdir. -type UnlinkRequest struct { - InodeHandle - Basename string -} - -// UnlinkPathRequest is the request object for RpcUnlinkPath & RpcRmdirPath. -type UnlinkPathRequest struct { - PathHandle -} - -// This section of the file contains RPC data structures for Swift middleware bimodal support. -// -// The API for this section is implemented in middleware.go. -// -// TODO - Prefix all of the structs with "Middleware" to distinguish from other RPCs. - -// CreateContainerRequest is the request object for RpcCreateContainer. -type CreateContainerRequest struct { - VirtPath string -} - -// CreateContainerReply is the reply object for RpcCreateContainer. -type CreateContainerReply struct { -} - -// DeleteReply is the response object for RpcDelete -type DeleteReply struct { -} - -// DeleteReq is the request object for RpcDelete -type DeleteReq struct { - VirtPath string -} - -type HeadReply struct { - FileSize uint64 - IsDir bool - ModificationTime uint64 // nanoseconds since epoch - AttrChangeTime uint64 // nanoseconds since epoch - InodeNumber int64 - NumWrites uint64 - Metadata []byte // entity metadata, serialized -} - -type HeadReq struct { - VirtPath string // virtual entity path, e.g. /v1/AUTH_acc/some-dir[/some-file] -} - -// GetContainerReply is the response object for RpcGetContainer -type GetContainerReply struct { - ContainerEntries []fs.ContainerEntry - ModificationTime uint64 - AttrChangeTime uint64 - Metadata []byte // container metadata, serialized -} - -// GetContainerReq is the request object for RpcGetContainer -type GetContainerReq struct { - VirtPath string // virtual container path, e.g. /v1/AUTH_acc/some-dir - Marker string // marker from query string, used in pagination - EndMarker string // endmarker from query string, used in pagination - Prefix string // only look at entries starting with this - MaxEntries uint64 // maximum number of entries to return - Delimiter string // only match up to the first occurrence of delimiter (excluding prefix) -} - -// Response object for RpcGetAccount -type GetAccountReply struct { - AccountEntries []fs.AccountEntry - ModificationTime uint64 - AttrChangeTime uint64 -} - -// Request object for RpcGetAccount -type GetAccountReq struct { - VirtPath string // account path, e.g. /v1/AUTH_acc - Marker string // marker from query string, used in pagination - EndMarker string // endmarker from query string, used in pagination - MaxEntries uint64 // maximum number of entries to return -} - -// GetObjectReply is the response object for RpcGetObject -type GetObjectReply struct { - FileSize uint64 // size of the file, in bytes - IsDir bool // true if directory - ReadEntsOut []inode.ReadPlanStep // object/length/offset triples where the data is found - InodeNumber uint64 - NumWrites uint64 - Metadata []byte // serialized object metadata (previously set by middleware empty if absent) - ModificationTime uint64 // file's mtime in nanoseconds since the epoch - AttrChangeTime uint64 - LeaseId string -} - -// GetObjectReq is the request object for RpcGetObject -type GetObjectReq struct { - // Virtual path to be read. Refers to an object, e.g. - // /v1/AUTH_acc/a-container/an-object - VirtPath string - - // Ranges to be read from virtual path. Note: these are - // offset/length pairs, not HTTP byte ranges; please remember - // to convert the values. To obtain a read plan for the entire - // object, leave ReadEntsIn empty. - ReadEntsIn []fs.ReadRangeIn -} - -// MiddlewarePostReply is the reply object for RpcPost -type MiddlewarePostReply struct { -} - -// MiddlewarePostReq is the request object for RpcPost -type MiddlewarePostReq struct { - - // Virtual path to be read. This could be account, account/container or account/container/object - VirtPath string - - // New or updated HTTP metadata to be stored - NewMetaData []byte - - // Last MetaData known by caller - used to resolve races between clients by doing read/modify/write - OldMetaData []byte -} - -type MiddlewareMkdirReply struct { - ModificationTime uint64 - AttrChangeTime uint64 - InodeNumber int64 - NumWrites uint64 -} - -type MiddlewareMkdirReq struct { - - // Virtual path of the directory to be created - VirtPath string - - // HTTP metadata to be stored - Metadata []byte -} - -// PutCompleteReq is the request object for RpcPutComplete -type PutCompleteReq struct { - VirtPath string - PhysPaths []string - PhysLengths []uint64 - Metadata []byte -} - -// PutCompleteReply is the response object for RpcPutComplete -type PutCompleteReply struct { - ModificationTime uint64 - AttrChangeTime uint64 - InodeNumber int64 - NumWrites uint64 -} - -// PutLocationReq is the request object for RpcPutLocation -type PutLocationReq struct { - VirtPath string -} - -// PutLocationReply is the response object for RpcPutLocation -type PutLocationReply struct { - PhysPath string -} - -// PingReq is the request object for RpcPing -type PingReq struct { - Message string -} - -// PingReply is the response object for RpcPutLocation -type PingReply struct { - Message string -} - -// IsAccountBimodalReq is the request object for RpcIsAccountBimodal -type IsAccountBimodalReq struct { - AccountName string -} - -// IsAccountBimodalReply is the response object for RpcPutLocation -type IsAccountBimodalReply struct { - IsBimodal bool - ActivePeerPrivateIPAddr string -} - -// Types for RpcPutContainer -type PutContainerReq struct { - VirtPath string - NewMetadata []byte - OldMetadata []byte -} - -type PutContainerReply struct { -} - -type CoalesceReq struct { - VirtPath string - ElementAccountRelativePaths []string - // New or updated HTTP metadata to be stored - NewMetaData []byte -} - -type CoalesceReply struct { - ModificationTime uint64 - AttrChangeTime uint64 - InodeNumber int64 - NumWrites uint64 -} - -type ProvisionObjectRequest struct { - MountID MountIDAsString -} - -type ProvisionObjectReply struct { - PhysPath string -} - -type WroteRequest struct { - InodeHandle - ContainerName string - ObjectName string - FileOffset []uint64 - ObjectOffset []uint64 - Length []uint64 - WroteTimeNs uint64 // New value for File's Stat.CTimeNs & Stat.MTimeNs -} - -type WroteReply struct { -} - -type RenewLeaseReq struct { - LeaseId string -} -type RenewLeaseReply struct{} - -type ReleaseLeaseReq struct { - LeaseId string -} -type ReleaseLeaseReply struct{} - -// SnapShotCreateRequest is the request object for RpcSnapShotCreate -type SnapShotCreateRequest struct { - MountID MountIDAsString - Name string -} - -// SnapShotCreateReply is the reply object for RpcSnapShotCreate -type SnapShotCreateReply struct { - SnapShotID uint64 -} - -// SnapShotDeleteRequest is the request object for RpcSnapShotDelete -type SnapShotDeleteRequest struct { - MountID MountIDAsString - SnapShotID uint64 -} - -// SnapShotDeleteReply is the reply object for RpcSnapShotDelete -type SnapShotDeleteReply struct{} - -// SnapShotListRequest is the request object for RpcSnapShotListBy{ID|Name|Time} -type SnapShotListRequest struct { - MountID MountIDAsString - Reversed bool -} - -// SnapShotListReply is the reply object for RpcSnapShotListBy{ID|Name|Time} -type SnapShotListReply struct { - List []headhunter.SnapShotStruct -} - -// SnapShotLookupByNameRequest is the request object for RpcSnapShotLookupByName -type SnapShotLookupByNameRequest struct { - MountID MountIDAsString - Name string -} - -// SnapShotLookupByNameReply is the reply object for RpcSnapShotLookupByName -type SnapShotLookupByNameReply struct { - SnapShot headhunter.SnapShotStruct -} - -// LeaseRequestType specifies the requested lease operation -// -type LeaseRequestType uint32 - -const ( - LeaseRequestTypeShared LeaseRequestType = iota // Currently unleased, requesting SharedLease - LeaseRequestTypePromote // Currently SharedLease held, requesting promoting to ExclusiveLease - LeaseRequestTypeExclusive // Currently unleased, requesting ExclusiveLease - LeaseRequestTypeDemote // Currently ExclusiveLease held, requesting demotion to SharedLease - LeaseRequestTypeRelease // Currently SharedLease or ExclusiveLease held, releasing it -) - -// LeaseRequest is the request object for RpcLease -// -type LeaseRequest struct { - InodeHandle - LeaseRequestType // One of LeaseRequestType* -} - -// LeaseReplyType specifies the acknowledgement that the requested lease operation -// has been completed or denied (e.g. when a Promotion request cannot be satisfied -// and the client will soon be receiving a LeaseInterruptTypeRelease) -// -type LeaseReplyType uint32 - -const ( - LeaseReplyTypeDenied LeaseReplyType = iota // Request denied (e.g. Promotion deadlock avoidance) - LeaseReplyTypeShared // SharedLease granted - LeaseReplyTypePromoted // SharedLease promoted to ExclusiveLease - LeaseReplyTypeExclusive // ExclusiveLease granted - LeaseReplyTypeDemoted // ExclusiveLease demoted to SharedLease - LeaseReplyTypeReleased // SharedLease or ExclusiveLease released -) - -// LeaseReply is the reply object for RpcLease -// -type LeaseReply struct { - LeaseReplyType // One of LeaseReplyType* -} - -// RPCInterruptType specifies the action (unmount, demotion, or release) requested by ProxyFS -// of the client in an RpcInterrupt "upcall" to indicate that a lease or leases must be demoted -// or released -// -type RPCInterruptType uint32 - -const ( - // RPCInterruptTypeUnmount indicates all Leases should be released (after performing necessary - // state saving RPCs) and the client should unmount - // - RPCInterruptTypeUnmount RPCInterruptType = iota - - // RPCInterruptTypeDemote indicates the specified LeaseHandle should (at least) be demoted - // from Exclusive to Shared (after performing necessary state saving RPCs) - // - RPCInterruptTypeDemote - - // RPCInterruptTypeRelease indicates the specified LeaseHandle should be released (after - // performing state saving RPCs and invalidating such cached state) - // - RPCInterruptTypeRelease -) - -// RPCInterrupt is the "upcall" mechanism used by ProxyFS to interrupt the client -type RPCInterrupt struct { - RPCInterruptType // One of RPCInterruptType* - InodeNumber int64 // if RPCInterruptType == RPCInterruptTypeUnmount, InodeNumber == 0 (ignored) -} - -// LeaseReportElementStruct describes the state of a particular Lease. Any SharedLease holders or -// a singular ExclusiveLease holder will be reported (if any) as well as a list of any clients -// awaiting Shared or Exclusive Leases. -// -type LeaseReportElementStruct struct { - InodeNumber string - SharedHoldersList []MountIDAsString - PromotingHolder MountIDAsString - ExclusiveHolder MountIDAsString - DemotingHolder MountIDAsString - ReleasingHoldersList []MountIDAsString - RequestedList []string // Time-ordered (i.e. 1st element is next to receive Lease) - // SharedLease waiters will be of the form "S[]" - // ExclusiveLease waiters will be of the form "E[]" -} - -// FetchLeaseReport returns a JSON-marshalable dump of the state of Leases for the specified Volume. -// -func FetchLeaseReport(volumeName string) (leaseReport []*LeaseReportElementStruct, err error) { - leaseReport, err = fetchLeaseReport(volumeName) - return -} diff --git a/jrpcfs/config.go b/jrpcfs/config.go deleted file mode 100644 index f405e817..00000000 --- a/jrpcfs/config.go +++ /dev/null @@ -1,500 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "container/list" - "crypto/tls" - "fmt" - "io/ioutil" - "net" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/retryrpc" - "github.com/NVIDIA/proxyfs/transitions" -) - -type leaseRequestOperationStruct struct { - mount *mountStruct - inodeLease *inodeLeaseStruct - LeaseRequestType - replyChan chan *LeaseReply -} - -type leaseRequestStateType uint32 - -const ( - leaseRequestStateNone leaseRequestStateType = iota - leaseRequestStateSharedRequested - leaseRequestStateSharedGranted - leaseRequestStateSharedPromoting - leaseRequestStateSharedReleasing - leaseRequestStateExclusiveRequested - leaseRequestStateExclusiveGranted - leaseRequestStateExclusiveDemoting - leaseRequestStateExclusiveReleasing -) - -type leaseRequestStruct struct { - mount *mountStruct - inodeLease *inodeLeaseStruct - requestState leaseRequestStateType - replyChan chan *LeaseReply // copied from leaseRequestOperationStruct.replyChan for LeaseRequestType == LeaseRequestType{Shared|Promote|Exclusive} - listElement *list.Element // used when on one of inodeList.*List's -} - -type inodeLeaseStateType uint32 - -const ( - inodeLeaseStateNone inodeLeaseStateType = iota - inodeLeaseStateSharedGrantedRecently - inodeLeaseStateSharedGrantedLongAgo - inodeLeaseStateSharedPromoting - inodeLeaseStateSharedReleasing - inodeLeaseStateSharedExpired - inodeLeaseStateExclusiveGrantedRecently - inodeLeaseStateExclusiveGrantedLongAgo - inodeLeaseStateExclusiveDemoting - inodeLeaseStateExclusiveReleasing - inodeLeaseStateExclusiveExpired -) - -type inodeLeaseStruct struct { - volume *volumeStruct - inode.InodeNumber - lruElement *list.Element // link into volumeStruct.inodeLeaseLRU - beingEvicted bool - leaseState inodeLeaseStateType - - requestChan chan *leaseRequestOperationStruct - stopChan chan struct{} // closing this chan will trigger *inodeLeaseStruct.handler() to: - // revoke/reject all leaseRequestStruct's in *Holder* & requestedList - // issue volume.leaseHandlerWG.Done() - // and exit - - sharedHoldersList *list.List // each list.Element.Value.(*leaseRequestStruct).requestState == leaseRequestStateSharedGranted - promotingHolder *leaseRequestStruct // leaseRequest.requestState == leaseRequestStateSharedPromoting - exclusiveHolder *leaseRequestStruct // leaseRequest.requestState == leaseRequestStateExclusiveGranted - demotingHolder *leaseRequestStruct // leaseRequest.requestState == leaseRequestStateExclusiveDemoting - releasingHoldersList *list.List // each list.Element.Value.(*leaseRequestStruct).requestState == leaseRequestState{Shared|Exclusive}Releasing - requestedList *list.List // each list.Element.Value.(*leaseRequestStruct).requestState == leaseRequestState{Shared|Exclusive}Requested - - lastGrantTime time.Time // records the time at which the last exclusive or shared holder was set/added-to exclusiveHolder/sharedHoldersList - lastInterruptTime time.Time // records the time at which the last Interrupt was sent - interruptsSent uint32 - - longAgoTimer *time.Timer // if .C != nil, timing when to state transition from {Shared|Exclusive}LeaseGrantedRecently to {Shared|Exclusive}LeaseGrantedLogAgo - interruptTimer *time.Timer // if .C != nil, timing when to issue next Interrupt... or expire a Lease -} - -type mountStruct struct { - volume *volumeStruct - mountIDAsByteArray MountIDAsByteArray - mountIDAsString MountIDAsString - authToken string - retryRpcUniqueID uint64 - acceptingLeaseRequests bool // also an indicator (when false) that mount is being unmounted - leaseRequestMap map[inode.InodeNumber]*leaseRequestStruct // if present, there is an ongoing Lease Request for this inode.InodeNumber - // if not present, there is no ongoing Lease Request for this inode.InodeNumber -} - -type volumeStruct struct { - volumeName string - volumeHandle fs.VolumeHandle - acceptingMountsAndLeaseRequests bool - delayedUnmountList *list.List - mountMapByMountIDAsByteArray map[MountIDAsByteArray]*mountStruct // key == mountStruct.mountIDAsByteArray - mountMapByMountIDAsString map[MountIDAsString]*mountStruct // key == mountStruct.mountIDAsString - inodeLeaseMap map[inode.InodeNumber]*inodeLeaseStruct // key == inodeLeaseStruct.InodeNumber - inodeLeaseLRU *list.List // .Front() is the LRU inodeLeaseStruct.listElement - ongoingLeaseEvictions uint64 // tracks the number of inodeLeaseStruct evictions currently ongoing - activeLeaseEvictLowLimit uint64 // number if inodeLeaseStruct's desired after evictions - activeLeaseEvictHighLimit uint64 // trigger on inodeLease{Map|LRU}.Len() for evicting inodeLeaseStructs - leaseHandlerWG sync.WaitGroup // .Add(1) each inodeLease insertion into inodeLeaseMap - // .Done() each inodeLease after it is removed from inodeLeaseMap -} - -type globalsStruct struct { - gate sync.RWMutex // API Requests RLock()/RUnlock() - // confMap changes Lock()/Unlock() - - volumesLock sync.Mutex // protects mountMapByMountIDAsByteArray & mountMapByMountIDAsString - // as well as each volumeStruct/mountStruct map - - whoAmI string - publicIPAddr string - privateIPAddr string - portString string - fastPortString string - retryRPCPort uint16 - retryRPCTTLCompleted time.Duration - retryRPCAckTrim time.Duration - retryRPCDeadlineIO time.Duration - retryRPCKeepAlivePeriod time.Duration - retryRPCCertFilePath string - retryRPCKeyFilePath string - minLeaseDuration time.Duration - leaseInterruptInterval time.Duration - leaseInterruptLimit uint32 - dataPathLogging bool - - retryRPCCertPEM []byte - retryRPCKeyPEM []byte - - retryRPCCertificate tls.Certificate - - volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName - mountMapByMountIDAsByteArray map[MountIDAsByteArray]*mountStruct // key == mountStruct.mountIDAsByteArray - mountMapByMountIDAsString map[MountIDAsString]*mountStruct // key == mountStruct.mountIDAsString - - // RetryRPC server - retryrpcSvr *retryrpc.Server - - // Connection list and listener list to close during shutdown: - halting bool - connLock sync.Mutex - connections *list.List - connWG sync.WaitGroup - listeners []net.Listener - listenersWG sync.WaitGroup -} - -var globals globalsStruct - -func init() { - transitions.Register("jrpcfs", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - globals.volumeMap = make(map[string]*volumeStruct) - globals.mountMapByMountIDAsByteArray = make(map[MountIDAsByteArray]*mountStruct) - globals.mountMapByMountIDAsString = make(map[MountIDAsString]*mountStruct) - - // Fetch IPAddrs from config file - globals.whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - logger.ErrorfWithError(err, "failed to get Cluster.WhoAmI from config file") - return - } - globals.publicIPAddr, err = confMap.FetchOptionValueString("Peer:"+globals.whoAmI, "PublicIPAddr") - if nil != err { - logger.ErrorfWithError(err, "failed to get %s.PublicIPAddr from config file", globals.whoAmI) - return - } - globals.privateIPAddr, err = confMap.FetchOptionValueString("Peer:"+globals.whoAmI, "PrivateIPAddr") - if nil != err { - logger.ErrorfWithError(err, "failed to get %s.PrivateIPAddr from config file", globals.whoAmI) - return - } - - // Fetch port number from config file - globals.portString, err = confMap.FetchOptionValueString("JSONRPCServer", "TCPPort") - if nil != err { - logger.ErrorfWithError(err, "failed to get JSONRPCServer.TCPPort from config file") - return - } - - // Fetch fastPort number from config file - globals.fastPortString, err = confMap.FetchOptionValueString("JSONRPCServer", "FastTCPPort") - if nil != err { - logger.ErrorfWithError(err, "failed to get JSONRPCServer.FastTCPPort from config file") - return - } - - globals.retryRPCPort, err = confMap.FetchOptionValueUint16("JSONRPCServer", "RetryRPCPort") - if nil == err { - globals.retryRPCTTLCompleted, err = confMap.FetchOptionValueDuration("JSONRPCServer", "RetryRPCTTLCompleted") - if nil != err { - logger.Infof("failed to get JSONRPCServer.RetryRPCTTLCompleted from config file - defaulting to 10m") - globals.retryRPCTTLCompleted = 10 * time.Minute - } - globals.retryRPCAckTrim, err = confMap.FetchOptionValueDuration("JSONRPCServer", "RetryRPCAckTrim") - if nil != err { - logger.Infof("failed to get JSONRPCServer.RetryRPCAckTrim from config file - defaulting to 100ms") - globals.retryRPCAckTrim = 100 * time.Millisecond - } - globals.retryRPCDeadlineIO, err = confMap.FetchOptionValueDuration("JSONRPCServer", "RetryRPCDeadlineIO") - if nil != err { - logger.Infof("failed to get JSONRPCServer.RetryRPCDeadlineIO from config file - defaulting to 60s") - globals.retryRPCDeadlineIO = 60 * time.Second - } - globals.retryRPCKeepAlivePeriod, err = confMap.FetchOptionValueDuration("JSONRPCServer", "RetryRPCKeepAlivePeriod") - if nil != err { - logger.Infof("failed to get JSONRPCServer.RetryRPCKeepAlivePeriod from config file - defaulting to 60s") - globals.retryRPCKeepAlivePeriod = 60 * time.Second - } - globals.retryRPCCertFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCCertFilePath") - if (nil == err) && ("" != globals.retryRPCCertFilePath) { - globals.retryRPCKeyFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCKeyFilePath") - if (nil != err) || ("" == globals.retryRPCKeyFilePath) { - logger.Error("if [JSOPNRPCServer]RetryRPCCertFilePath is specified, [JSOPNRPCServer]RetryRPCKeyFilePath must be specified as well") - return - } - globals.retryRPCCertPEM, err = ioutil.ReadFile(globals.retryRPCCertFilePath) - if nil != err { - logger.ErrorfWithError(err, "failed to load PEM-formatted [JSONRPCServer]RetryRPCCertFilePath [\"%s\"]", globals.retryRPCCertFilePath) - return - } - globals.retryRPCKeyPEM, err = ioutil.ReadFile(globals.retryRPCKeyFilePath) - if nil != err { - logger.ErrorfWithError(err, "failed to load PEM-formatted [JSONRPCServer]retryRPCKeyFilePath [\"%s\"]", globals.retryRPCKeyFilePath) - return - } - globals.retryRPCCertificate, err = tls.X509KeyPair(globals.retryRPCCertPEM, globals.retryRPCKeyPEM) - if nil != err { - logger.ErrorfWithError(err, "tls.LoadX509KeyPair(\"%s\", \"%s\") failed", globals.retryRPCCertFilePath, globals.retryRPCKeyFilePath) - return - } - } else { - globals.retryRPCKeyFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCKeyFilePath") - if (nil == err) && ("" != globals.retryRPCKeyFilePath) { - logger.Error("if [JSOPNRPCServer]RetryRPCKeyFilePath is specified, [JSOPNRPCServer]RetryRPCCertFilePath must be specified as well") - return - } - logger.Infof("failed to get JSONRPCServer.RetryRPC{Cert|Key}FilePath from config file - defaulting to \"\"") - globals.retryRPCCertFilePath = "" - globals.retryRPCKeyFilePath = "" - globals.retryRPCCertPEM = nil - globals.retryRPCKeyPEM = nil - globals.retryRPCCertificate = tls.Certificate{} - } - } else { - logger.Infof("failed to get JSONRPCServer.RetryRPCPort from config file - skipping......") - globals.retryRPCPort = 0 - globals.retryRPCTTLCompleted = time.Duration(0) - globals.retryRPCAckTrim = time.Duration(0) - globals.retryRPCDeadlineIO = time.Duration(0) - globals.retryRPCKeepAlivePeriod = time.Duration(0) - globals.retryRPCCertFilePath = "" - globals.retryRPCKeyFilePath = "" - globals.retryRPCCertPEM = nil - globals.retryRPCKeyPEM = nil - globals.retryRPCCertificate = tls.Certificate{} - } - - // Set data path logging level to true, so that all trace logging is controlled by settings - // in the logger package. To enable jrpcfs trace logging, set Logging.TraceLevelLogging to jrpcfs. - // This will enable all jrpcfs trace logs, including those formerly controled by globals.dataPathLogging. - // To disable read/write/flush trace logs separately, change this setting here to false. - globals.dataPathLogging, err = confMap.FetchOptionValueBool("JSONRPCServer", "DataPathLogging") - if nil != err { - logger.ErrorfWithError(err, "failed to get JSONRPCServer.DataPathLogging from config file") - return - } - - globals.minLeaseDuration, err = confMap.FetchOptionValueDuration("JSONRPCServer", "MinLeaseDuration") - if nil != err { - logger.Infof("failed to get JSONRPCServer.MinLeaseDuration from config file - defaulting to 250ms") - globals.minLeaseDuration = 250 * time.Millisecond - } - globals.leaseInterruptInterval, err = confMap.FetchOptionValueDuration("JSONRPCServer", "LeaseInterruptInterval") - if nil != err { - logger.Infof("failed to get JSONRPCServer.LeaseInterruptInterval from config file - defaulting to 250ms") - globals.leaseInterruptInterval = 250 * time.Millisecond - } - globals.leaseInterruptLimit, err = confMap.FetchOptionValueUint32("JSONRPCServer", "LeaseInterruptLimit") - if nil != err { - logger.Infof("failed to get JSONRPCServer.LeaseInterruptLimit from config file - defaulting to 20") - globals.leaseInterruptLimit = 20 - } - - // Ensure gate starts out in the Exclusively Locked state - closeGate() - - // Init listeners - globals.listeners = make([]net.Listener, 0, 2) - globals.connections = list.New() - globals.halting = false - - // Init JSON RPC server stuff - jsonRpcServerUp(globals.privateIPAddr, globals.portString) - - // Now kick off our other, faster RPC server - ioServerUp(globals.privateIPAddr, globals.fastPortString) - - // Init Retry RPC server - retryRPCServerUp(jserver) - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} - -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - currentlyInVolumeMap bool - volume *volumeStruct - volumeHandle fs.VolumeHandle - ) - - globals.volumesLock.Lock() - - _, currentlyInVolumeMap = globals.volumeMap[volumeName] - if currentlyInVolumeMap { - globals.volumesLock.Unlock() - err = fmt.Errorf("Cannot be told to ServeVolume(,\"%s\") twice", volumeName) - return - } - - volumeHandle, err = fs.FetchVolumeHandleByVolumeName(volumeName) - if nil != err { - globals.volumesLock.Unlock() - return - } - - volume = &volumeStruct{ - volumeName: volumeName, - volumeHandle: volumeHandle, - acceptingMountsAndLeaseRequests: true, - delayedUnmountList: list.New(), - mountMapByMountIDAsByteArray: make(map[MountIDAsByteArray]*mountStruct), - mountMapByMountIDAsString: make(map[MountIDAsString]*mountStruct), - inodeLeaseMap: make(map[inode.InodeNumber]*inodeLeaseStruct), - inodeLeaseLRU: list.New(), - ongoingLeaseEvictions: 0, - } - - volume.activeLeaseEvictLowLimit, err = confMap.FetchOptionValueUint64("Volume:"+volumeName, "ActiveLeaseEvictLowLimit") - if nil != err { - logger.Infof("failed to get Volume:" + volumeName + ".ActiveLeaseEvictLowLimit from config file - defaulting to 5000") - volume.activeLeaseEvictLowLimit = 500000 - } - volume.activeLeaseEvictHighLimit, err = confMap.FetchOptionValueUint64("Volume:"+volumeName, "ActiveLeaseEvictHighLimit") - if nil != err { - logger.Infof("failed to get Volume:" + volumeName + ".ActiveLeaseEvictHighLimit from config file - defaulting to 5010") - volume.activeLeaseEvictHighLimit = 500010 - } - - globals.volumeMap[volumeName] = volume - - globals.volumesLock.Unlock() - - err = nil - return -} - -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - var ( - currentlyInVolumeMap bool - mountIDAsByteArray MountIDAsByteArray - mountIDAsString MountIDAsString - volume *volumeStruct - ) - - globals.volumesLock.Lock() - - volume, currentlyInVolumeMap = globals.volumeMap[volumeName] - if !currentlyInVolumeMap { - globals.volumesLock.Unlock() - err = fmt.Errorf("Cannot be told to UnserveVolume(,\"%s\") a non-served volume", volumeName) - return - } - - volume.acceptingMountsAndLeaseRequests = false - - // TODO: Lease Management changes - somehow while *not* holding volumesLock.Lock(): - // Prevent new lease requests - // Fail outstanding lease requests - // Revoke granted leases - - delete(globals.volumeMap, volumeName) - - for mountIDAsByteArray = range volume.mountMapByMountIDAsByteArray { - delete(globals.mountMapByMountIDAsByteArray, mountIDAsByteArray) - } - - for mountIDAsString = range volume.mountMapByMountIDAsString { - delete(globals.mountMapByMountIDAsString, mountIDAsString) - } - - globals.volumesLock.Unlock() - - err = nil - return -} - -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil // TODO: this is where we get a chance to tell our clients to unmount !!! -} - -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - closeGate() - - err = nil - return -} - -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - openGate() - - err = nil - return -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - if 0 != len(globals.volumeMap) { - err = fmt.Errorf("jrpcfs.Down() called with 0 != len(globals.volumeMap)") - return - } - if 0 != len(globals.mountMapByMountIDAsByteArray) { - err = fmt.Errorf("jrpcfs.Down() called with 0 != len(globals.mountMapByMountIDAsByteArray)") - return - } - if 0 != len(globals.mountMapByMountIDAsString) { - err = fmt.Errorf("jrpcfs.Down() called with 0 != len(globals.mountMapByMountIDAsString)") - return - } - - globals.halting = true - - jsonRpcServerDown() - ioServerDown() - retryRPCServerDown() - - globals.listenersWG.Wait() - - openGate() // In case we are restarted... Up() expects Gate to initially be open - - err = nil - return -} - -func openGate() { - globals.gate.Unlock() -} - -func closeGate() { - globals.gate.Lock() -} - -func enterGate() { - globals.gate.RLock() -} - -func leaveGate() { - globals.gate.RUnlock() -} diff --git a/jrpcfs/encoding_test.go b/jrpcfs/encoding_test.go deleted file mode 100644 index 5c68fcbe..00000000 --- a/jrpcfs/encoding_test.go +++ /dev/null @@ -1,1140 +0,0 @@ -//Allocate response/ Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "bytes" - "encoding/binary" - "encoding/gob" - "encoding/json" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -// The encoding tests want to experiment with various ways to encode the same -// RPC requests sent over the wire. The retryrpc code wraps an RPC request with -// jsonRequest and then prepends ioHeader before sending it over the wire. -// Replies get the same treatment. -// -// Copy jsonRequest, ioRequest, jsonReply, ioReply, and some routines that fill -// them in from the retryrpc code so we can experiment with different ways of -// encoding & decoding them. -type requestID uint64 -type MsgType uint16 - -type jsonRequest struct { - MyUniqueID uint64 `json:"myuniqueid"` // ID of client - RequestID requestID `json:"requestid"` // ID of this request - HighestReplySeen requestID `json:"highestReplySeen"` // Used to trim completedRequests on server - Method string `json:"method"` - Params [1]interface{} `json:"params"` -} - -// jsonReply is used to marshal an RPC response in/out of JSON -type jsonReply struct { - MyUniqueID uint64 `json:"myuniqueid"` // ID of client - RequestID requestID `json:"requestid"` // ID of this request - ErrStr string `json:"errstr"` - Result interface{} `json:"result"` -} - -// ioRequest tracks fields written on wire -type ioRequestRetryRpc struct { - Hdr ioHeader - JReq []byte // JSON containing request -} - -// ioReply is the structure returned over the wire -type ioReplyRetryRpc struct { - Hdr ioHeader - JResult []byte // JSON containing response -} - -type ioHeader struct { - Len uint32 // Number of bytes following header - Protocol uint16 - Version uint16 - Type MsgType - Magic uint32 // Magic number - if invalid means have not read complete header -} - -// Encode a LeaseRequest to binary, json, or some other form. -type encodeLeaseRequestFunc func(leaseReq *LeaseRequest, jreq *jsonRequest) (hdrBytes []byte, jsonBytes []byte) - -// Decode a LeaseRequest from binary, json, or some other form. -type decodeLeaseRequestFunc func(hdrBytes []byte, jsonBytes []byte) (leaseReq *LeaseRequest, jreq *jsonRequest) - -// Encode a LeaseRequest to binary, json, or some other form. -type encodeLeaseReplyFunc func(leaseReply *LeaseReply, jreply *jsonReply) (hdrBytes []byte, jsonBytes []byte) - -// Decode a LeaseRequest from binary, json, or some other form. -type decodeLeaseReplyFunc func(hdrBytes []byte, jsonBytes []byte) (leaseReply *LeaseReply, jreply *jsonReply) - -// Test that the request and reply decode functions match the encode functions -func TestEncodeDecodeBinary(t *testing.T) { - testEncodeDecodeFunctions(t, - encodeLeaseRequestBinary, encodeLeaseReplyBinary, - decodeLeaseRequestBinary, decodeLeaseReplyBinary) -} - -// Test that the request and reply decode functions match the encode functions -func TestEncodeDecodeJSON(t *testing.T) { - testEncodeDecodeFunctions(t, - encodeLeaseRequestJson, encodeLeaseReplyJson, - decodeLeaseRequestJson, decodeLeaseReplyJson) -} - -// Test that the request and reply decode functions match the encode functions -func TestEncodeDecodeGob(t *testing.T) { - testEncodeDecodeFunctions(t, - encodeLeaseRequestGob, encodeLeaseReplyGob, - decodeLeaseRequestGob, decodeLeaseReplyGob) -} - -// BenchmarkRpcLeaseEncodeBinary emulates the Lease request/reply RPC encoding -// by building the structures and simply calling the routines that do a binary -// encoding. -func BenchmarkRpcLeaseEncodeBinary(b *testing.B) { - benchmarkRpcLeaseEncode(b, - encodeLeaseRequestBinary, encodeLeaseReplyBinary) -} - -// BenchmarkRpcLeaseEncodeJSON emulates the Lease request/reply RPC encoding -// by building the structures and simply calling the routines that do a json -// encoding. -func BenchmarkRpcLeaseEncodeJSON(b *testing.B) { - benchmarkRpcLeaseEncode(b, - encodeLeaseRequestJson, encodeLeaseReplyJson) -} - -// BenchmarkRpcLeaseEncodeGob emulates the Lease request/reply RPC encoding -// by building the structures and simply calling the routines that do a gob -// encoding. -func BenchmarkRpcLeaseEncodeGob(b *testing.B) { - benchmarkRpcLeaseEncode(b, - encodeLeaseRequestGob, encodeLeaseReplyGob) -} - -// BenchmarkRpcLeaseDecodeBinary emulates the Lease request/reply RPC encoding -// by building the structures and simply calling the routines that do a binary -// encoding. -func BenchmarkRpcLeaseDecodeBinary(b *testing.B) { - benchmarkRpcLeaseDecode(b, - encodeLeaseRequestJson, encodeLeaseReplyJson, - decodeLeaseRequestJson, decodeLeaseReplyJson) -} - -// BenchmarkRpcLeaseDecodeJSON emulates the Lease request/reply RPC encoding -// by building the structures and simply calling the routines that do a binary -// encoding. -func BenchmarkRpcLeaseDecodeJSON(b *testing.B) { - benchmarkRpcLeaseDecode(b, - encodeLeaseRequestJson, encodeLeaseReplyJson, - decodeLeaseRequestJson, decodeLeaseReplyJson) -} - -// BenchmarkRpcLeaseDecodeGob emulates the Lease request/reply RPC encoding -// by building the structures and simply calling the routines that do a binary -// encoding. -func BenchmarkRpcLeaseDecodeGob(b *testing.B) { - benchmarkRpcLeaseDecode(b, - encodeLeaseRequestGob, encodeLeaseReplyGob, - decodeLeaseRequestGob, decodeLeaseReplyGob) -} - -// testEncodeDecodeFunctions tests "generic" encoding and decoding functions for -// LeaseRequests to see if the decoded requests match what was encoded. -func testEncodeDecodeFunctions(t *testing.T, - encodeLeaseRequest encodeLeaseRequestFunc, - encodeLeaseReply encodeLeaseReplyFunc, - decodeLeaseRequest decodeLeaseRequestFunc, - decodeLeaseReply decodeLeaseReplyFunc) { - - assert := assert.New(t) - - // encode a lease request and then decode it - mountByAccountNameReply := &MountByAccountNameReply{ - MountID: "66", - } - leaseRequest := &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeExclusive, - } - jsonRequest := newJsonRequest("RpcLease") - - // this marshals both the RPCrequest and the header for the request - leaseRequestHdrBuf, leaseRequestPayloadBuf := - encodeLeaseRequest(leaseRequest, jsonRequest) - - leaseRequest2, jsonRequest2 := - decodeLeaseRequest(leaseRequestHdrBuf, leaseRequestPayloadBuf) - - assert.Equal(leaseRequest, leaseRequest2, "Decoded struct should match the original") - assert.Equal(jsonRequest, jsonRequest2, "Decoded struct should match the original") - - // try again with release lease request - leaseRequest.LeaseRequestType = LeaseRequestTypeRelease - - leaseRequestHdrBuf, leaseRequestPayloadBuf = - encodeLeaseRequest(leaseRequest, jsonRequest) - - leaseRequest2, jsonRequest2 = - decodeLeaseRequest(leaseRequestHdrBuf, leaseRequestPayloadBuf) - - assert.Equal(leaseRequest, leaseRequest2, "Decoded struct should match the original") - assert.Equal(jsonRequest, jsonRequest2, "Decoded struct should match the original") - - // encode reply and decode it - leaseReply := &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - jsonReply := newJsonReply() - leaseReplyHdrBuf, leaseReplyPayloadBuf := - encodeLeaseReply(leaseReply, jsonReply) - - leaseReply2, jsonReply2 := - decodeLeaseReply(leaseReplyHdrBuf, leaseReplyPayloadBuf) - - assert.Equal(leaseReply, leaseReply2, "Decoded struct should match the original") - assert.Equal(jsonReply, jsonReply2, "Decoded struct should match the original") - - // try again with release lease reply - leaseReply.LeaseReplyType = LeaseReplyTypeReleased - - leaseReplyHdrBuf, leaseReplyPayloadBuf = - encodeLeaseReply(leaseReply, jsonReply) - - leaseReply2, jsonReply2 = - decodeLeaseReply(leaseReplyHdrBuf, leaseReplyPayloadBuf) - - assert.Equal(leaseReply, leaseReply2, "Decoded struct should match the original") - assert.Equal(jsonReply, jsonReply2, "Decoded struct should match the original") -} - -// benchmarkRpcLeaseEncode is the guts to benchmark the cost of encoding the LeaseRequest -// and LeaseReply parts of a LeaseRequest RPC. -func benchmarkRpcLeaseEncode(b *testing.B, - encodeLeaseRequest encodeLeaseRequestFunc, - encodeLeaseReply encodeLeaseReplyFunc) { - - var ( - benchmarkIteration int - leaseReply *LeaseReply - mountByAccountNameReply *MountByAccountNameReply - ) - - mountByAccountNameReply = &MountByAccountNameReply{ - MountID: "66", - } - - b.ResetTimer() - - for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { - leaseRequest := &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeExclusive, - } - - // this marshals both the RPCrequest and the header for the request - hdrBuf, requestBuf := encodeLeaseRequest(leaseRequest, newJsonRequest("RpcLease")) - _ = hdrBuf - _ = requestBuf - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - - // marshal the RPC reply - hdrBuf, replyBuf := encodeLeaseReply(leaseReply, newJsonReply()) - _ = hdrBuf - _ = replyBuf - - // "validate" the returned value - if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReply.LeaseReplyType) - } - - // release the lock - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeRelease, - } - - hdrBuf, requestBuf = encodeLeaseRequest(leaseRequest, newJsonRequest("RpcLease")) - _ = hdrBuf - _ = requestBuf - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - - hdrBuf, replyBuf = encodeLeaseReply(leaseReply, newJsonReply()) - _ = hdrBuf - _ = replyBuf - - // "validate" the return value - if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReply.LeaseReplyType) - } - } - - b.StopTimer() -} - -// benchmarkRpcLeaseEncode is the guts to benchmark the cost of decoding the -// LeaseRequest and LeaseReply parts of a LeaseRequest RPC. -func benchmarkRpcLeaseDecode(b *testing.B, - encodeLeaseRequest encodeLeaseRequestFunc, - encodeLeaseReply encodeLeaseReplyFunc, - decodeLeaseRequest decodeLeaseRequestFunc, - decodeLeaseReply decodeLeaseReplyFunc) { - - var ( - benchmarkIteration int - leaseReply *LeaseReply - mountByAccountNameReply *MountByAccountNameReply - ) - - mountByAccountNameReply = &MountByAccountNameReply{ - MountID: "6", - } - - // build the over-the-wire buffers that will be decoded later - // - // get lease request first - leaseRequest := &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeExclusive, - } - - // this marshals both the RPCrequest and the header for the request - getLeaseRequestHdrBuf, getLeaseRequestPayloadBuf := - encodeLeaseRequest(leaseRequest, newJsonRequest("RpcLease")) - - // release lease request - leaseRequest.LeaseRequestType = LeaseRequestTypeRelease - releaseLeaseRequestHdrBuf, releaseLeaseRequestPayloadBuf := - encodeLeaseRequest(leaseRequest, newJsonRequest("RpcLease")) - - // now build the replies - // get lease reply - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - getLeaseReplyHdrBuf, getLeaseReplyPayloadBuf := - encodeLeaseReply(leaseReply, newJsonReply()) - - // release lease reply - leaseReply.LeaseReplyType = LeaseReplyTypeReleased - releaseLeaseReplyHdrBuf, releaseLeaseReplyPayloadBuf := - encodeLeaseReply(leaseReply, newJsonReply()) - - b.ResetTimer() - - for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { - - leaseRequest, jRequest := - decodeLeaseRequest(getLeaseRequestHdrBuf, getLeaseRequestPayloadBuf) - _ = leaseRequest - _ = jRequest - - leaseReply, jReply := - decodeLeaseReply(getLeaseReplyHdrBuf, getLeaseReplyPayloadBuf) - _ = jReply - - // "validate" the returned value - if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", - leaseReply.LeaseReplyType) - } - - // release the lock - leaseRequest, jRequest = - decodeLeaseRequest(releaseLeaseRequestHdrBuf, releaseLeaseRequestPayloadBuf) - - leaseReply, jReply = - decodeLeaseReply(releaseLeaseReplyHdrBuf, releaseLeaseReplyPayloadBuf) - - // "validate" the return value - if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", - leaseReply.LeaseReplyType) - } - } - - b.StopTimer() -} - -func newJsonRequest(method string) (jreq *jsonRequest) { - jreq = &jsonRequest{ - MyUniqueID: 77, - RequestID: 88, - HighestReplySeen: 33, - Method: method, - } - - return -} - -func newJsonReply() (jreply *jsonReply) { - jreply = &jsonReply{ - MyUniqueID: 77, - RequestID: 88, - ErrStr: "", - } - - return -} - -// Perform all of the encoding required by the client to send a LeaseRequest. -// This includes encoding the LeaseRequest, the jsonRequest "wrapper", and the -// ioRequest header (which is always a fixed binary encoding). -// -// The encoding of the jsonRequest and LeaseRequest also use a hand-written -// binary encoding. -func encodeLeaseRequestBinary(leaseReq *LeaseRequest, jreq *jsonRequest) (hdrBytes []byte, jsonBytes []byte) { - var err error - - hdrBuf := &bytes.Buffer{} - jsonBuf := &bytes.Buffer{} - - // marshal jsonRequest fields - err = binary.Write(jsonBuf, binary.LittleEndian, jreq.MyUniqueID) - if err != nil { - panic("jreq.MyUniqueID") - } - err = binary.Write(jsonBuf, binary.LittleEndian, jreq.RequestID) - if err != nil { - panic("jreq.RequestID") - } - err = binary.Write(jsonBuf, binary.LittleEndian, jreq.HighestReplySeen) - if err != nil { - panic("jreq.HighestReplySeen") - } - // use a Nul terminated string - methodWithNul := jreq.Method + string([]byte{0}) - _, err = jsonBuf.WriteString(methodWithNul) - if err != nil { - panic("jreq.Method") - } - - // marshal LeaseRequest fields - mountIDWithNul := leaseReq.MountID + MountIDAsString([]byte{0}) - _, err = jsonBuf.WriteString(string(mountIDWithNul)) - if err != nil { - panic("leaseReq.MountID") - } - err = binary.Write(jsonBuf, binary.LittleEndian, leaseReq.InodeNumber) - if err != nil { - panic("leaseReq.LeaseInodeNumber") - } - err = binary.Write(jsonBuf, binary.LittleEndian, leaseReq.LeaseRequestType) - if err != nil { - panic("leaseReq.LeaseRequestType") - } - - // now create the IoRequest header and Marshal it - ioReq := ioRequestRetryRpc{ - Hdr: ioHeader{ - Len: uint32(jsonBuf.Len()), - Protocol: uint16(1), - Version: 1, - Type: 1, - Magic: 0xCAFEFEED, - }, - } - - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Len) - if err != nil { - panic("ioReq.Hdr.Len") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Protocol) - if err != nil { - panic("ioReq.Hdr.Protocol") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Version) - if err != nil { - panic("ioReq.Hdr.Version") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Type) - if err != nil { - panic("ioReq.Hdr.Type") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Magic) - if err != nil { - panic("ioReq.Hdr.Magic") - } - - hdrBytes = hdrBuf.Bytes() - jsonBytes = jsonBuf.Bytes() - return -} - -// Perform all of the encoding required on the server to send a LeaseReply. -// This includes encoding the LeaseReply, the jsonReply "wrapper", and the -// ioReply header (which is always a fixed binary encoding). -// -// The encoding of the jsonReply and LeaseReply also use a hand-written binary -// encoding. -func encodeLeaseReplyBinary(leaseReply *LeaseReply, jreply *jsonReply) (hdrBytes []byte, jsonBytes []byte) { - var err error - - hdrBuf := &bytes.Buffer{} - jsonBuf := &bytes.Buffer{} - - // marshal jsonReply fields - err = binary.Write(jsonBuf, binary.LittleEndian, jreply.MyUniqueID) - if err != nil { - panic("jreply.MyUniqueID") - } - err = binary.Write(jsonBuf, binary.LittleEndian, jreply.RequestID) - if err != nil { - panic("jreply.RequestID") - } - // use a Nul termianted string - errStrWithNul := jreply.ErrStr + string([]byte{0}) - _, err = jsonBuf.WriteString(errStrWithNul) - if err != nil { - panic("jreply.ErrStr") - } - - // marshal LeaseReply fields - err = binary.Write(jsonBuf, binary.LittleEndian, leaseReply.LeaseReplyType) - if err != nil { - panic("leaseReply.LeaseReplyType") - } - - // now create the IoReply header and Marshal it - ioReply := ioReplyRetryRpc{ - Hdr: ioHeader{ - Len: uint32(jsonBuf.Len()), - Protocol: uint16(1), - Version: 1, - Type: 1, - Magic: 0xCAFEFEED, - }, - } - - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Len) - if err != nil { - panic("ioReply.Hdr.Len") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Protocol) - if err != nil { - panic("ioReply.Hdr.Protocol") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Version) - if err != nil { - panic("ioReply.Hdr.Version") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Type) - if err != nil { - panic("ioReply.Hdr.Type") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Magic) - if err != nil { - panic("ioReply.Hdr.Magic") - } - - hdrBytes = hdrBuf.Bytes() - jsonBytes = jsonBuf.Bytes() - return -} - -// Perform all of the decoding required on the server to recieve a LeaseRequest. -// This includes decoding the LeaseRequest, the jsonRequest "wrapper", and the -// ioRequest header (which is always a fixed binary encoding). -// -// The decoding of the jsonRequest and LeaseRequest use a hand-written binary -// decoder. -func decodeLeaseRequestBinary(hdrBytes []byte, jsonBytes []byte) (leaseReq *LeaseRequest, jreq *jsonRequest) { - var err error - - leaseReq = &LeaseRequest{} - jreq = &jsonRequest{} - ioReq := &ioRequestRetryRpc{} - - hdrBuf := bytes.NewBuffer(hdrBytes) - jsonBuf := bytes.NewBuffer(jsonBytes) - - // Unmarshal the IoRequest header - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Len) - if err != nil { - panic("ioReq.Hdr.Len") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Protocol) - if err != nil { - panic("ioReq.Hdr.Protocol") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Version) - if err != nil { - panic("ioReq.Hdr.Version") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Type) - if err != nil { - panic("ioReq.Hdr.Type") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Magic) - if err != nil { - panic("ioReq.Hdr.Magic") - } - - // now unmarshal the jsonRequest fields - err = binary.Read(jsonBuf, binary.LittleEndian, &jreq.MyUniqueID) - if err != nil { - panic("jreq.MyUniqueID") - } - err = binary.Read(jsonBuf, binary.LittleEndian, &jreq.RequestID) - if err != nil { - panic("jreq.RequestID") - } - err = binary.Read(jsonBuf, binary.LittleEndian, &jreq.HighestReplySeen) - if err != nil { - panic("jreq.HighestReplySeen") - } - // method is Nul terminated - method, err := jsonBuf.ReadString(0) - if err != nil { - panic(fmt.Sprintf("jreq.Method '%s' buf '%+v' err: %v", jreq.Method, jsonBuf, err)) - } - jreq.Method = method[0 : len(method)-1] - - // unmarshal LeaseRequest fields (MountID is Nul terminated) - mountID, err := jsonBuf.ReadString(0) - if err != nil { - panic("leaseReq.MountID") - } - leaseReq.MountID = MountIDAsString(mountID[0 : len(mountID)-1]) - err = binary.Read(jsonBuf, binary.LittleEndian, &leaseReq.InodeNumber) - if err != nil { - panic("leaseReq.LeaseInodeNumber") - } - err = binary.Read(jsonBuf, binary.LittleEndian, &leaseReq.LeaseRequestType) - if err != nil { - panic("leaseReq.LeaseRequestType") - } - - hdrBytes = hdrBuf.Bytes() - jsonBytes = jsonBuf.Bytes() - return -} - -// Perform all of the decoding required on the server to recieve a LeaseReply. -// This includes decoding the LeaseReply, the jsonReply "wrapper", and the -// ioReply header (which is always a fixed binary encoding). -// -// The decoding of the jsonReply and LeaseRequest are also a hand-written binary -// decoder. -func decodeLeaseReplyBinary(hdrBytes []byte, jsonBytes []byte) (leaseReply *LeaseReply, jreply *jsonReply) { - var err error - - leaseReply = &LeaseReply{} - jreply = &jsonReply{} - ioReply := &ioReplyRetryRpc{} - - hdrBuf := bytes.NewBuffer(hdrBytes) - jsonBuf := bytes.NewBuffer(jsonBytes) - - // first unmarshal the IoReply - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Len) - if err != nil { - panic("ioReply.Hdr.Len") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Protocol) - if err != nil { - panic("ioReply.Hdr.Protocol") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Version) - if err != nil { - panic("ioReply.Hdr.Version") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Type) - if err != nil { - panic("ioReply.Hdr.Type") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Magic) - if err != nil { - panic("ioReply.Hdr.Magic") - } - - // now unmarshal jsonReply fields - err = binary.Read(jsonBuf, binary.LittleEndian, &jreply.MyUniqueID) - if err != nil { - panic("jreply.MyUniqueID") - } - err = binary.Read(jsonBuf, binary.LittleEndian, &jreply.RequestID) - if err != nil { - panic("jreply.RequestID") - } - // ErrStr is Nul terminated - errStr, err := jsonBuf.ReadString(0) - if err != nil { - panic("jreply.ErrStr") - } - jreply.ErrStr = errStr[0 : len(errStr)-1] - - // unmarshal LeaseReply fields - err = binary.Read(jsonBuf, binary.LittleEndian, &leaseReply.LeaseReplyType) - if err != nil { - panic("leaseReply.LeaseReplyType") - } - - hdrBytes = hdrBuf.Bytes() - jsonBytes = jsonBuf.Bytes() - return -} - -// Perform all of the encoding required by the client to send a LeaseRequest. -// This includes encoding the LeaseRequest, the jsonRequest "wrapper", and the -// ioRequest header (which is always a fixed binary encoding). -// -// The encoding of the jsonRequest and LeaseRequest uses json.Marshal(). -func encodeLeaseRequestJson(leaseReq *LeaseRequest, jreq *jsonRequest) (hdrBytes []byte, jsonBytes []byte) { - var err error - - hdrBuf := &bytes.Buffer{} - - // the Lease Request is part of the JSON request - jreq.Params[0] = leaseReq - - // marshal the json request (and lease request) - jsonBytes, err = json.Marshal(jreq) - if err != nil { - panic("json.Marshal") - } - - // now create the IoRequest header and Marshal it - // (this is always binary) - ioReq := ioRequestRetryRpc{ - Hdr: ioHeader{ - Len: uint32(len(jsonBytes)), - Protocol: uint16(1), - Version: 1, - Type: 1, - Magic: 0xCAFEFEED, - }, - } - - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Len) - if err != nil { - panic("ioReq.Hdr.Len") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Protocol) - if err != nil { - panic("ioReq.Hdr.Protocol") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Version) - if err != nil { - panic("ioReq.Hdr.Version") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Type) - if err != nil { - panic("ioReq.Hdr.Type") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Magic) - if err != nil { - panic("ioReq.Hdr.Magic") - } - - hdrBytes = hdrBuf.Bytes() - return -} - -// Perform all of the decoding required on the server to recieve a LeaseRequest. -// This includes decoding the LeaseRequest, the jsonRequest "wrapper", and the -// ioRequest header (which is always a fixed binary encoding). -// -// The decoding of the jsonRequest and LeaseRequest use json.Unmarshal(). -func decodeLeaseRequestJson(hdrBytes []byte, jsonBytes []byte) (leaseReq *LeaseRequest, jreq *jsonRequest) { - var err error - - leaseReq = &LeaseRequest{} - jreq = &jsonRequest{} - jreq.Params[0] = leaseReq - ioReq := &ioRequestRetryRpc{} - - hdrBuf := bytes.NewBuffer(hdrBytes) - - // Unmarshal the IoRequest header (always binary) - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Len) - if err != nil { - panic("ioReq.Hdr.Len") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Protocol) - if err != nil { - panic("ioReq.Hdr.Protocol") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Version) - if err != nil { - panic("ioReq.Hdr.Version") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Type) - if err != nil { - panic("ioReq.Hdr.Type") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Magic) - if err != nil { - panic("ioReq.Hdr.Magic") - } - - // now unmarshal the jsonRequest fields - err = json.Unmarshal(jsonBytes, jreq) - if err != nil { - panic("json.Unmarshal of json") - } - - return -} - -// Perform all of the encoding required on the server to send a LeaseReply. -// This includes encoding the LeaseReply, the jsonReply "wrapper", and the -// ioReply header (which is always a fixed binary encoding). -// -// The encoding of the jsonReply and LeaseReply use json.Marshal(). -func encodeLeaseReplyJson(leaseReply *LeaseReply, jreply *jsonReply) (hdrBytes []byte, jsonBytes []byte) { - var err error - - hdrBuf := &bytes.Buffer{} - - // the Lease Reply is part of the JSON reply - jreply.Result = leaseReply - - // marshal the json reply (and lease reply) - jsonBytes, err = json.Marshal(jreply) - if err != nil { - panic("json.Marshal") - } - - // now create the IoReply header and Marshal it - // (this is always binary) - ioReply := ioReplyRetryRpc{ - Hdr: ioHeader{ - Len: uint32(len(jsonBytes)), - Protocol: uint16(1), - Version: 1, - Type: 1, - Magic: 0xCAFEFEED, - }, - } - - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Len) - if err != nil { - panic("ioReply.Hdr.Len") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Protocol) - if err != nil { - panic("ioReply.Hdr.Protocol") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Version) - if err != nil { - panic("ioReply.Hdr.Version") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Type) - if err != nil { - panic("ioReply.Hdr.Type") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Magic) - if err != nil { - panic("ioReply.Hdr.Magic") - } - - hdrBytes = hdrBuf.Bytes() - return -} - -// Perform all of the decoding required on the server to recieve a LeaseReply. -// This includes decoding the LeaseReply, the jsonReply "wrapper", and the -// ioReply header (which is always a fixed binary encoding). -// -// The decoding of the jsonReply and LeaseReply use a json.Unmarshal(). -func decodeLeaseReplyJson(hdrBytes []byte, jsonBytes []byte) (leaseReply *LeaseReply, jreply *jsonReply) { - var err error - - leaseReply = &LeaseReply{} - jreply = &jsonReply{} - jreply.Result = leaseReply - ioReply := &ioReplyRetryRpc{} - - hdrBuf := bytes.NewBuffer(hdrBytes) - - // Unmarshal the IoReply header (always binary) - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Len) - if err != nil { - panic("ioReply.Hdr.Len") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Protocol) - if err != nil { - panic("ioReply.Hdr.Protocol") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Version) - if err != nil { - panic("ioReply.Hdr.Version") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Type) - if err != nil { - panic("ioReply.Hdr.Type") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Magic) - if err != nil { - panic("ioReply.Hdr.Magic") - } - - // now unmarshal the jsonReply fields - err = json.Unmarshal(jsonBytes, jreply) - if err != nil { - panic("json.Unmarshal of json") - } - - leaseReply = jreply.Result.(*LeaseReply) - return -} - -// Create gob LeaseRequest and LeaseReply encoders and decoders that are -// globally available. -var ( - encodeLeaseRequestGobBuffer *bytes.Buffer = &bytes.Buffer{} - decodeLeaseRequestGobBuffer *bytes.Buffer = &bytes.Buffer{} - encodeLeaseRequestGobEncoder *gob.Encoder = gob.NewEncoder(encodeLeaseRequestGobBuffer) - decodeLeaseRequestGobDecoder *gob.Decoder = gob.NewDecoder(decodeLeaseRequestGobBuffer) - - encodeLeaseReplyGobBuffer *bytes.Buffer = &bytes.Buffer{} - decodeLeaseReplyGobBuffer *bytes.Buffer = &bytes.Buffer{} - encodeLeaseReplyGobEncoder *gob.Encoder = gob.NewEncoder(encodeLeaseReplyGobBuffer) - decodeLeaseReplyGobDecoder *gob.Decoder = gob.NewDecoder(decodeLeaseReplyGobBuffer) -) - -// Perform all of the encoding required by the client to send a LeaseRequest. -// This includes encoding the LeaseRequest, the jsonRequest "wrapper", and the -// ioRequest header (which is always a fixed binary encoding). -// -// The encoding of the jsonRequest and LeaseRequest use a pre-defined gob -// encoding. -func encodeLeaseRequestGob(leaseReq *LeaseRequest, jreq *jsonRequest) (hdrBytes []byte, gobBytes []byte) { - var err error - - hdrBuf := &bytes.Buffer{} - - // the Lease Request is part of the gob request - jreq.Params[0] = leaseReq - - // marshal jreq (and lease request) - err = encodeLeaseRequestGobEncoder.Encode(jreq) - if err != nil { - panic("encodeLeaseRequestGobEncoder") - } - - // consume the results encoded in the (global) buffer - gobBytes = make([]byte, encodeLeaseRequestGobBuffer.Len()) - n, err := encodeLeaseRequestGobBuffer.Read(gobBytes) - if n != cap(gobBytes) { - panic("didn't read enough bytes") - } - - // now create the IoRequest header and Marshal it - // (this is always binary) - ioReq := ioRequestRetryRpc{ - Hdr: ioHeader{ - Len: uint32(len(gobBytes)), - Protocol: uint16(1), - Version: 1, - Type: 1, - Magic: 0xCAFEFEED, - }, - } - - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Len) - if err != nil { - panic("ioReq.Hdr.Len") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Protocol) - if err != nil { - panic("ioReq.Hdr.Protocol") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Version) - if err != nil { - panic("ioReq.Hdr.Version") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Type) - if err != nil { - panic("ioReq.Hdr.Type") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReq.Hdr.Magic) - if err != nil { - panic("ioReq.Hdr.Magic") - } - - hdrBytes = hdrBuf.Bytes() - return -} - -// Perform all of the decoding required on the server to receive a LeaseRequest. -// This includes decoding the LeaseRequest, the jsonRequest "wrapper", and the -// ioRequest header (which is always a fixed binary encoding). -// -// The decoding of the jsonRequest and LeaseRequest use a pre-defined gob decoder. -func decodeLeaseRequestGob(hdrBytes []byte, gobBytes []byte) (leaseReq *LeaseRequest, jreq *jsonRequest) { - var err error - - jreq = &jsonRequest{} - ioReq := &ioRequestRetryRpc{} - - hdrBuf := bytes.NewBuffer(hdrBytes) - - // Unmarshal the IoRequest header (always binary) - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Len) - if err != nil { - panic("ioReq.Hdr.Len") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Protocol) - if err != nil { - panic("ioReq.Hdr.Protocol") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Version) - if err != nil { - panic("ioReq.Hdr.Version") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Type) - if err != nil { - panic("ioReq.Hdr.Type") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReq.Hdr.Magic) - if err != nil { - panic("ioReq.Hdr.Magic") - } - - // now unmarshal the jsonRequest fields using gob (can't fail) - _, _ = decodeLeaseRequestGobBuffer.Write(gobBytes) - err = decodeLeaseRequestGobDecoder.Decode(jreq) - if err != nil { - panic("decodeLeaseRequestGobDecoder.Decode") - } - leaseReq = jreq.Params[0].(*LeaseRequest) - - return -} - -// Perform all of the encoding required on the server to send a LeaseReply. -// This includes encoding the LeaseReply, the jsonReply "wrapper", and the -// ioReply header (which is always a fixed binary encoding). -// -// The encoding of the jsonReply and LeaseReply use a pre-defined gob encoder. -func encodeLeaseReplyGob(leaseReply *LeaseReply, jreply *jsonReply) (hdrBytes []byte, gobBytes []byte) { - var err error - - hdrBuf := &bytes.Buffer{} - - // the Lease Reply is part of the JSON reply - jreply.Result = leaseReply - - // marshal jreq (and lease request) - err = encodeLeaseReplyGobEncoder.Encode(jreply) - if err != nil { - panic("encodeLeaseReplyGobEncoder") - } - - // consume the results encoded in the (global) buffer - gobBytes = make([]byte, encodeLeaseReplyGobBuffer.Len()) - n, err := encodeLeaseReplyGobBuffer.Read(gobBytes) - if n != cap(gobBytes) { - panic("didn't read enough bytes") - } - - // now create the IoReply header and Marshal it - // (this is always binary) - ioReply := ioReplyRetryRpc{ - Hdr: ioHeader{ - Len: uint32(len(gobBytes)), - Protocol: uint16(1), - Version: 1, - Type: 1, - Magic: 0xCAFEFEED, - }, - } - - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Len) - if err != nil { - panic("ioReply.Hdr.Len") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Protocol) - if err != nil { - panic("ioReply.Hdr.Protocol") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Version) - if err != nil { - panic("ioReply.Hdr.Version") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Type) - if err != nil { - panic("ioReply.Hdr.Type") - } - err = binary.Write(hdrBuf, binary.LittleEndian, ioReply.Hdr.Magic) - if err != nil { - panic("ioReply.Hdr.Magic") - } - - hdrBytes = hdrBuf.Bytes() - return -} - -// Perform all of the decoding required on the server to recieve a LeaseReply. -// This includes decoding the LeaseReply, the jsonReply "wrapper", and the -// ioReply header (which is always a fixed binary encoding). -// -// The decoding of the jsonReply and LeaseReply use a pre-defined gob decoder. -func decodeLeaseReplyGob(hdrBytes []byte, gobBytes []byte) (leaseReply *LeaseReply, jreply *jsonReply) { - var err error - - leaseReply = &LeaseReply{} - jreply = &jsonReply{} - jreply.Result = leaseReply - ioReply := &ioReplyRetryRpc{} - - hdrBuf := bytes.NewBuffer(hdrBytes) - - // Unmarshal the IoReply header (always binary) - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Len) - if err != nil { - panic("ioReply.Hdr.Len") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Protocol) - if err != nil { - panic("ioReply.Hdr.Protocol") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Version) - if err != nil { - panic("ioReply.Hdr.Version") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Type) - if err != nil { - panic("ioReply.Hdr.Type") - } - err = binary.Read(hdrBuf, binary.LittleEndian, &ioReply.Hdr.Magic) - if err != nil { - panic("ioReply.Hdr.Magic") - } - - // now unmarshal the jsonReply fields using gob (can't fail) - _, _ = decodeLeaseReplyGobBuffer.Write(gobBytes) - err = decodeLeaseReplyGobDecoder.Decode(jreply) - if err != nil { - panic("decodeLeaseReplyGobDecoder.Decode") - } - leaseReply = jreply.Result.(*LeaseReply) - - return -} diff --git a/jrpcfs/filesystem.go b/jrpcfs/filesystem.go deleted file mode 100644 index deec4598..00000000 --- a/jrpcfs/filesystem.go +++ /dev/null @@ -1,2251 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// JSON RPC Server on top of FS package. -package jrpcfs - -import ( - "container/list" - "encoding/base64" - "fmt" - "net" - "net/rpc" - "net/rpc/jsonrpc" - "path/filepath" - "strconv" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/utils" -) - -// RPC server handle -var srv *rpc.Server -var jrpcListener net.Listener - -// Local server handle, used to track jrpc-related ops -var jserver *Server - -func jsonRpcServerUp(ipAddr string, portString string) { - var err error - - jserver = NewServer() - - srv = rpc.NewServer() - err = srv.Register(jserver) - if err != nil { - logger.ErrorfWithError(err, "failed to register RPC handler") - return - } - - jrpcListener, err = net.Listen("tcp", net.JoinHostPort(ipAddr, portString)) - if err != nil { - logger.ErrorfWithError(err, "net.Listen %s:%s failed", ipAddr, portString) - return - } - - globals.connLock.Lock() - globals.listeners = append(globals.listeners, jrpcListener) - globals.connLock.Unlock() - - //logger.Infof("Starting to listen on %s:%s", ipAddr, portString) - globals.listenersWG.Add(1) - go jrpcServerLoop() -} - -func jsonRpcServerDown() { - _ = jrpcListener.Close() - DumpIfNecessary(jserver) - stopServerProfiling(jserver) -} - -func jrpcServerLoop() { - for { - conn, err := jrpcListener.Accept() - if err != nil { - if !globals.halting { - logger.ErrorfWithError(err, "net.Accept failed for JRPC listener\n") - } - globals.listenersWG.Done() - return - } - - globals.connWG.Add(1) - - globals.connLock.Lock() - elm := globals.connections.PushBack(conn) - globals.connLock.Unlock() - - go func(myConn net.Conn, myElm *list.Element) { - srv.ServeCodec(jsonrpc.NewServerCodec(myConn)) - globals.connLock.Lock() - globals.connections.Remove(myElm) - - // There is a race condition where the connection could have been - // closed in Down(). However, closing it twice is okay. - myConn.Close() - globals.connLock.Unlock() - globals.connWG.Done() - }(conn, elm) - } -} - -// Enumeration of operations, used for stats-related things -type OpType int - -// NOTE: When you add a new OpType here, be sure to add its string -// at the appropriate spot in opTypeStrs below. -const ( - PingOp OpType = iota - ReadOp - WriteOp - ProvisionObjectOp - WroteOp - FetchExtentMapChunkOp - FlushOp - LookupOp - LookupPathOp - GetStatOp - GetStatPathOp - GetXattrOp - GetXattrPathOp - ReaddirOp - ReaddirByLocOp - ReaddirPlusOp - ReaddirPlusByLocOp - InvalidOp // Keep this as the last entry! -) - -var opTypeStrs = []string{ - "Ping", - "Read", - "Write", - "ProvisionObject", - "Wrote", - "FetchExtentMapChunk", - "Flush", - "Lookup", - "LookupPath", - "GetStat", - "GetStatPath", - "GetXattr", - "GetXattrPath", - "Readdir", - "ReaddirByLoc", - "ReaddirPlus", - "ReaddirPlusByLoc", - "InvalidOp", -} - -// Return operation type as a string -func (op OpType) String() string { - return opTypeStrs[op] -} - -// Get operation type from string -func strToOpType(opStr string) OpType { - for opType, strVal := range opTypeStrs { - if opStr == strVal { - return OpType(opType) - } - } - return InvalidOp -} - -// Map of profilers -type opProfiles map[int]*utils.Profiler - -// All of our RPC methods are called on/passed this Server struct. -type Server struct { - opStatsLock sync.Mutex - maxSavedProfilesPerOp int - allProfiles map[OpType]opProfiles - saveLock sync.Mutex - saveChannel chan *utils.Profiler - internalSaveChannel chan *utils.Profiler - saveProfilerProfiles opProfiles - dumpRunning bool - dumpLock sync.Mutex -} - -func NewOpProfiles() opProfiles { - return make(opProfiles, maxSavedStatProfiles) -} - -func AllocOpProfiles() (theOpProfiles map[OpType]opProfiles) { - if !doProfiling { - return theOpProfiles - } - - theOpProfiles = make(map[OpType]opProfiles, int(InvalidOp)) - - // Cheater range check on opTypeStrs versus number of OpTypes - if len(opTypeStrs) != int(InvalidOp)+1 { - // TODO: Find a way to make this a compile-time check instead - panic("len(opTypeStrs) != int(InvalidOp)+1!") - } - - // Allocat profile space for each op type - for i, _ := range opTypeStrs { - theOpProfiles[OpType(i)] = NewOpProfiles() - } - - return theOpProfiles -} - -// Stats storage -var maxSavedStatProfiles int = 4000 - -// Threshold of when to trigger stats dump/free -var statsCleanupThreshold int = 100 - -// Size of each nonblocking channel -var saveChannelSize int = 1000 - -// Default values here are false -var loggedOutOfStatsRoom map[OpType]bool = make(map[OpType]bool) - -func lookupVolumeHandleByMountIDAsByteArray(mountIDAsByteArray MountIDAsByteArray) (volumeHandle fs.VolumeHandle, err error) { - var ( - mount *mountStruct - ok bool - ) - - globals.volumesLock.Lock() - mount, ok = globals.mountMapByMountIDAsByteArray[mountIDAsByteArray] - globals.volumesLock.Unlock() - - if ok { - volumeHandle = mount.volume.volumeHandle - err = nil - } else { - err = fmt.Errorf("MountID %v not found in jrpcfs globals.mountMapByMountIDAsByteArray", mountIDAsByteArray) - err = blunder.AddError(err, blunder.BadMountIDError) - } - - return -} - -func lookupVolumeHandleByMountIDAsString(mountIDAsString MountIDAsString) (volumeHandle fs.VolumeHandle, err error) { - var ( - mount *mountStruct - ok bool - ) - - globals.volumesLock.Lock() - mount, ok = globals.mountMapByMountIDAsString[mountIDAsString] - globals.volumesLock.Unlock() - - if ok { - volumeHandle = mount.volume.volumeHandle - err = nil - } else { - err = fmt.Errorf("MountID %s not found in jrpcfs globals.mountMapByMountIDAsString", mountIDAsString) - err = blunder.AddError(err, blunder.BadMountIDError) - } - - return -} - -func NewServer() *Server { - s := Server{} - - // Set profiling-related variables - if doProfiling { - s.opStatsLock = sync.Mutex{} - s.maxSavedProfilesPerOp = maxSavedStatProfiles - s.allProfiles = AllocOpProfiles() - s.saveLock = sync.Mutex{} - s.saveChannel = make(chan *utils.Profiler, saveChannelSize) - s.internalSaveChannel = make(chan *utils.Profiler, saveChannelSize) - s.saveProfilerProfiles = make(opProfiles, maxSavedStatProfiles) - s.dumpRunning = false - s.dumpLock = sync.Mutex{} - } - - // Kick off goroutines for saving profiles - if doProfiling { - go opProfileSaver(&s) - go intProfileSaver(&s) - } - - return &s -} - -func stopServerProfiling(s *Server) { - if doProfiling { - close(s.saveChannel) - close(s.internalSaveChannel) - } -} - -// Read profiles off the channel and save them -func opProfileSaver(s *Server) { - for { - p, more := <-s.saveChannel - if more { - // Got a profile to save - op := strToOpType(p.Name()) - - var profiles opProfiles = s.allProfiles[op] - success := saveProfilerInternal(&profiles, p, &s.opStatsLock, s.maxSavedProfilesPerOp) - if !success { - // Couldn't save, no room - if !loggedOutOfStatsRoom[op] { - logger.Infof("Can't save stats for op %v, no more room!", op) - loggedOutOfStatsRoom[op] = true - } - } - - // If saved stats > threshold, kick off a goroutine to dump stats and free up some room - if len(s.allProfiles[op]) >= statsCleanupThreshold { - //logger.Infof("Calling DumpIfNecessary; num %v stats is %v, threshold is %v.", op.String(), len(s.allProfiles[op]), statsCleanupThreshold) - DumpIfNecessary(s) - } - } else { - // Got channel close - //logger.Infof("Profiler save channel closed.") - return - } - } -} - -func intProfileSaver(s *Server) { - for { - p, more := <-s.internalSaveChannel - if more { - // Got a profile to save - var saveProfiles opProfiles = s.saveProfilerProfiles - saveProfilerInternal(&saveProfiles, p, &s.saveLock, s.maxSavedProfilesPerOp) - } else { - // Got channel close - //logger.Infof("Profiler internal save channel closed.") - return - } - } -} - -// Saves a profile, if there is room. Returns true on success and false if out of room -func saveProfilerInternal(profiles *opProfiles, profiler *utils.Profiler, profLock *sync.Mutex, maxProfiles int) bool { - rtnVal := true // success - - profLock.Lock() - statsIndex := len(*profiles) - if statsIndex < maxProfiles { - (*profiles)[statsIndex] = profiler - } else { - // Don't save, no room. Indicate failed to save - rtnVal = false - } - profLock.Unlock() - - return rtnVal -} - -// This is purposely not a method in Server, since if is is one then -// net/rpc/server.go complains that it has the wrong number of ins/outs, -// from a check in suitableMethods(). -func SaveProfiler(s *Server, op OpType, profiler *utils.Profiler) { - if (op == InvalidOp) || (profiler == nil) { - return - } - - // Set op name, in case it wasn't known when the profiler was created - profiler.SetName(op.String()) - - // Profile myself... - var profile_myself *utils.Profiler = nil - if doInternalProfiling { - profile_myself = utils.NewProfilerIf(doProfiling, "SaveProfiler") - } - - // Save profiler by writing its pointer to the save channel - s.saveChannel <- profiler - - // Save my own profiling - if doInternalProfiling { - profile_myself.Close() - // Save profiler by writing its pointer to the internal save channel - s.internalSaveChannel <- profile_myself - } -} - -// TODO: create conf setting for this? -var dumpStatsProfilesToLog bool = true - -func DumpProfileMap(profilerMap *opProfiles) { - - if len(*profilerMap) > 0 { - stats := utils.GetStats(*profilerMap) - if dumpStatsProfilesToLog { - statsStr := stats.DumpToString(true) - logger.Infof("Periodic stats: %s", statsStr) - } else { - stats.DumpToStdout() - } - } -} - -var doInternalProfiling bool = false -var doProfiling bool = false -var dumpInternalStats bool = false - -func dumpStats(s *Server) { - - // Dump stats; looping over stats for each op - for _, opProfs := range s.allProfiles { - DumpProfileMap(&opProfs) - } - - if dumpInternalStats { - DumpProfileMap(&s.saveProfilerProfiles) - } -} - -func DumpIfNecessary(s *Server) { - if !doProfiling { - return - } - - // First check to see if running already - s.dumpLock.Lock() - alreadyRunning := s.dumpRunning - s.dumpLock.Unlock() - - if !alreadyRunning { - //logger.Infof("Triggering stats dump and free.") - go DumpAndFreeStats(s) - } -} - -// How to clean up? Move profiler to some stale list? Just delete? -func DumpAndFreeStats(s *Server) { - // Prevent multiple instances from running at the same time (?) - // If so, should we check to see if there's anything to do first? - s.dumpLock.Lock() - alreadyRunning := s.dumpRunning - if !alreadyRunning { - s.dumpRunning = true - } - s.dumpLock.Unlock() - - if alreadyRunning { - //logger.Infof("Dump is already running; return.") - } - - var profiler = utils.NewProfilerIf(doProfiling, "DumpAndFreeStats") - - // Lock - s.opStatsLock.Lock() - profiler.AddEventNow("after lock") - - // TODO: Is it faster to copy/free and then dump later, - // or just to dump/free and return? - - // TODO: do the makes outside the lock, just make it max size - // could make things faster in here... - - // Copy profiles to a local map - savedOpProfiles := make(map[OpType]opProfiles, len(s.allProfiles)) - - // Do a an element-by-element copy, since otherwise we get a reference to the - // original and that is not what we want - for op, opProfMap := range s.allProfiles { - // Alloc space, same size as source - savedOpProfiles[op] = make(opProfiles, len(opProfMap)) - - // Deep copy - for index, entry := range opProfMap { - savedOpProfiles[op][index] = entry - } - } - profiler.AddEventNow("after copy") - - // Free saved profiles; doing this by creating a new map, therefore removing references - // to anything that was in the previous one. - - // NOTE: this will clear out the save and dump profiles as well! - s.allProfiles = AllocOpProfiles() - profiler.AddEventNow("after free") - - // Unlock - s.opStatsLock.Unlock() - profiler.AddEventNow("after unlock") - - // Mark ourselves as done since we're not accessing the common data structure any more. - // TODO: Is this a race condition waiting to happen? - s.dumpLock.Lock() - s.dumpRunning = false - s.dumpLock.Unlock() - - // Dump stats; looping over stats for each op - for _, opProfMap := range savedOpProfiles { - //logger.Infof("op %v: num stats=%v\n", op, len(opProfMap)) - DumpProfileMap(&opProfMap) - } - profiler.AddEventNow("after dump") - - if dumpInternalStats { - DumpProfileMap(&s.saveProfilerProfiles) - } - - // Save my own profiling - profiler.Close() - // TODO: Disabled dumping of this data; it's for debug only - // profiler.Dump() // TODO: There isn't a dump to string for this one yet -} - -// RESPONSE ERROR FORMATTING -// -// NOTE: The format of the response error is defined in section 5.1 of the -// JSON-RPC 2.0 Specification (http://www.jsonrpc.org/specification), -// which looks something like this: -// -//" 5.1 Error object -// When a rpc call encounters an error, the Response Object MUST contain the error -// member with a value that is a Object with the following members: -// -// code -// A Number that indicates the error type that occurred. -// This MUST be an integer. -// message -// A String providing a short description of the error. -// The message SHOULD be limited to a concise single sentence. -// data -// A Primitive or Structured value that contains additional information about the error. -// This may be omitted. -// The value of this member is defined by the Server (e.g. detailed error information, -// nested errors etc.). -// -// The error codes from and including -32768 to -32000 are reserved for pre-defined errors. -// Any code within this range, but not defined explicitly below is reserved for future use. -//" -// Editorial comment: -// It looks that the "code" here is intended for errors specific to the RPC infrastructure -// itself. RPC/JSON-related error values in the range of -32768 to -32000 are defined in -// the specification. It also says that -32000 to -32099 is reserved for -// "implementation-defined server-errors". I assume this means JSON RPC server errors. -// Then it says "The remainder of the space is available for application defined errors". -// -// I did an experiment with sending an RPC for a method that does not exist, to see if our -// JSON RPC go server returns any of these error codes (-32601 is Method not found). It did -// not send back an error code, just an error with the following text: -// "rpc: can't find method Server.RpcGetStatStatStat" -// -// An example from random.org, a response for a request for a method that doesn't exist: -// { -// "jsonrpc": "2.0", -// "error": { -// "code": -32601, -// "message": "Method not found", -// "data": null -// }, -// "id": 18197 -// } - -// ERROR HANDLING/PARAMETER CHECKING -// -// Because Go does runtime type checking, it's possible that we can trigger a panic by -// passing bad/incorrect parameters into fs functions. It's important that we add the -// proper checks here or in fs code so that a bad parameter in an RPC call cannot ever -// crash proxyfsd. - -// rpcEncodeError converts an error into a format that can be parsed by the client -// to extract error details. -// -// It extracts errno out of the error (if present) and encodes it into the error -// returned by the function. This will be put into the JSON RPC error field by -// the JSON RPC server, to be sent to the far end. -// -// Note that we are discarding ProxyFS-specific error text and other details here; -// this information is not useful to a remote client. If desired, one could log -// the details before overwriting them. The code to do so would look something -// like this: -// -// // Log the file/line where the error was set -// logger.Infof("discarding error %v from %s\n", err, blunder.SourceLine(err)) -// -// // The stacktrace can be printed, if it has one (null string is returned if not) -// logger.Infof("discarding error %v with stacktrace %s\n", err, blunder.Stacktrace(err)) -// -// // Alternately, one could print out all the error details at once -// logger.Infof("discarding error %v\n", err, blunder.Details(err)) -// -// Error format sent to clients: -// -// Currently that format is key/value pairs separated by newline, in conf-like format: -// : \n -// -// More specifically, we are passing: -// errno: -// -// TODO: The format of how this information will be conveyed over JSON RPC has not -// been determined; this is just an experimental implementation. -// -// TODO: We should probably encode in JSON instead, since that is the way everything else -// we send is being encoded. -// -// NOTE: e needs to be pointer to error so that we can modify it -// -func rpcEncodeError(e *error) { - if *e != nil { - *e = fmt.Errorf("errno: %d", blunder.Errno(*e)) - } -} - -// Shorthand for our internal API debug log id; global to the package -const internalDebug = logger.DbgInternal - -func splitPath(fullpath string) (parentDir string, basename string) { - // Split fullpath into parent dir and new basename - parentDir = filepath.Dir(fullpath) - basename = filepath.Base(fullpath) - return -} - -func (s *Server) RpcChown(in *ChownRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // NOTE: We currently just store and return per-inode ownership info. - // We do not check/enforce it; that is the caller's responsibility. - - stat := make(fs.Stat) - if in.UserID != -1 { - stat[fs.StatUserID] = uint64(in.UserID) - } - if in.GroupID != -1 { - stat[fs.StatGroupID] = uint64(in.GroupID) - } - err = volumeHandle.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), stat) - return -} - -func (s *Server) RpcChownPath(in *ChownPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // NOTE: We currently just store and return per-inode ownership info. - // We do not check/enforce it; that is the caller's responsibility. - - // Get the inode - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - if err != nil { - return - } - - // Do the Setstat - stat := make(fs.Stat) - if in.UserID != -1 { - stat[fs.StatUserID] = uint64(in.UserID) - } - if in.GroupID != -1 { - stat[fs.StatGroupID] = uint64(in.GroupID) - } - err = volumeHandle.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, stat) - return -} - -func (s *Server) RpcChmod(in *ChmodRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - // NOTE: We currently just store and return per-inode ownership info. - // We do not check/enforce it; that is the caller's responsibility. - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Samba includes the file mode in in.FileMode, but only the permssion - // bits can be changed by SetStat(). - stat := make(fs.Stat) - stat[fs.StatMode] = uint64(in.FileMode) & 07777 - err = volumeHandle.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), stat) - return -} - -func (s *Server) RpcChmodPath(in *ChmodPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // NOTE: We currently just store and return per-inode ownership info. - // We do not check/enforce it; that is the caller's responsibility. - - // Get the inode - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - if err != nil { - return - } - - // Do the Setstat - // - // Samba includes the file mode in in.FileMode, but only the permssion - // bits can be changed by SetStat(). - stat := make(fs.Stat) - stat[fs.StatMode] = uint64(in.FileMode) & 07777 - err = volumeHandle.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, stat) - return -} - -func (s *Server) RpcCreate(in *CreateRequest, reply *InodeReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - fino, err := volumeHandle.Create(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, inode.InodeNumber(in.InodeNumber), in.Basename, inode.InodeMode(in.FileMode)) - reply.InodeNumber = int64(uint64(fino)) - return -} - -func (s *Server) RpcCreatePath(in *CreatePathRequest, reply *InodeReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Ideally we would like all name/fullpath checking logic to be in the fs package, - // however since the fs.Create() and fs.Mkdir() APIs are inode-based, once we are - // inside those functions the fullpath is no longer available. The simplest solution is to - // just do the checking here. - err = fs.ValidateFullPath(in.Fullpath) - if err != nil { - return err - } - - // Split fullpath into parent dir and new basename - parentDir, basename := splitPath(in.Fullpath) - - // Get the inode for the parent dir - ino, err := volumeHandle.LookupPath(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, parentDir) - if err != nil { - return - } - - // Do the create - fino, err := volumeHandle.Create(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, ino, basename, inode.InodeMode(in.FileMode)) - reply.InodeNumber = int64(uint64(fino)) - return -} - -func (s *Server) RpcFlock(in *FlockRequest, reply *FlockReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - var flock fs.FlockStruct - flock.Type = in.FlockType - flock.Whence = in.FlockWhence - flock.Start = in.FlockStart - flock.Len = in.FlockLen - flock.Pid = in.FlockPid - - lockStruct, err := volumeHandle.Flock(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.FlockCmd, &flock) - if lockStruct != nil { - reply.FlockType = lockStruct.Type - reply.FlockWhence = lockStruct.Whence - reply.FlockStart = lockStruct.Start - reply.FlockLen = lockStruct.Len - reply.FlockPid = lockStruct.Pid - } - return -} - -func UnixSec(t time.Time) (sec int64) { - return t.Unix() -} - -func UnixNanosec(t time.Time) (ns int64) { - return t.UnixNano() - t.Unix()*int64(time.Second) -} - -func (s *Server) RpcProvisionObject(in *ProvisionObjectRequest, reply *ProvisionObjectReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil == err { - reply.PhysPath, err = volumeHandle.CallInodeToProvisionObject() - } - - return -} - -func (s *Server) RpcWrote(in *WroteRequest, reply *WroteReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil == err { - err = volumeHandle.Wrote(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.ContainerName, in.ObjectName, in.FileOffset, in.ObjectOffset, in.Length, in.WroteTimeNs) - } - - return -} - -func (s *Server) RpcFetchExtentMapChunk(in *FetchExtentMapChunkRequest, reply *FetchExtentMapChunkReply) (err error) { - var ( - extentMapChunk *inode.ExtentMapChunkStruct - ) - - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil == err { - extentMapChunk, err = volumeHandle.FetchExtentMapChunk(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.FileOffset, in.MaxEntriesFromFileOffset, in.MaxEntriesBeforeFileOffset) - if nil == err { - reply.FileOffsetRangeStart = extentMapChunk.FileOffsetRangeStart - reply.FileOffsetRangeEnd = extentMapChunk.FileOffsetRangeEnd - reply.FileSize = extentMapChunk.FileSize - reply.ExtentMapEntry = extentMapChunk.ExtentMapEntry - } - } - - return -} - -func (s *Server) RpcFlush(in *FlushRequest, reply *Reply) (err error) { - var profiler = utils.NewProfilerIf(doProfiling, "flush") - - enterGate() - defer leaveGate() - - sendTime := time.Unix(in.SendTimeSec, in.SendTimeNsec) - requestRecTime := time.Now() - deliveryLatency := requestRecTime.Sub(sendTime) - deliveryLatencyUsec := deliveryLatency.Nanoseconds() - var flog logger.FuncCtx - - if globals.dataPathLogging { - flog = logger.TraceEnter("in.", in, "deliveryLatencyUsec:"+strconv.FormatInt(deliveryLatencyUsec, 10)) - } - - stopwatch := utils.NewStopwatch() - defer func() { - _ = stopwatch.Stop() - if globals.dataPathLogging { - flog.TraceExitErr("reply.", err, reply, "duration:"+stopwatch.ElapsedMsString()) - } - }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - profiler.AddEventNow("before fs.Flush()") - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil == err { - err = volumeHandle.Flush(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber)) - } - profiler.AddEventNow("after fs.Flush()") - - reply.RequestTimeSec = UnixSec(requestRecTime) - reply.RequestTimeNsec = UnixNanosec(requestRecTime) - - replySendTime := time.Now() - reply.SendTimeSec = replySendTime.Unix() - reply.SendTimeNsec = (replySendTime.UnixNano() - (reply.SendTimeSec * int64(time.Second))) - - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, FlushOp, profiler) - - return -} - -func (stat *StatStruct) fsStatToStatStruct(fsStat fs.Stat) { - stat.CRTimeNs = fsStat[fs.StatCRTime] - stat.CTimeNs = fsStat[fs.StatCTime] - stat.MTimeNs = fsStat[fs.StatMTime] - stat.ATimeNs = fsStat[fs.StatATime] - stat.Size = fsStat[fs.StatSize] - stat.NumLinks = fsStat[fs.StatNLink] - stat.StatInodeNumber = int64(fsStat[fs.StatINum]) - stat.FileMode = uint32(fsStat[fs.StatMode]) - stat.UserID = uint32(fsStat[fs.StatUserID]) - stat.GroupID = uint32(fsStat[fs.StatGroupID]) -} - -func (s *Server) RpcGetStat(in *GetStatRequest, reply *StatStruct) (err error) { - enterGate() - defer leaveGate() - - var stat fs.Stat - var profiler = utils.NewProfilerIf(doProfiling, "getstat") - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - profiler.AddEventNow("before fs.Getstat()") - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil == err { - stat, err = volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber)) - } - profiler.AddEventNow("after fs.Getstat()") - if err == nil { - reply.fsStatToStatStruct(stat) - } - - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, GetStatOp, profiler) - - return -} - -func (s *Server) RpcGetStatPath(in *GetStatPathRequest, reply *StatStruct) (err error) { - var profiler = utils.NewProfilerIf(doProfiling, "getstat_path") - - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Get the inode - profiler.AddEventNow("before fs.LookupPath()") - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - profiler.AddEventNow("after fs.LookupPath()") - if err != nil { - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, GetStatPathOp, profiler) - return - } - - // Do the GetStat - profiler.AddEventNow("before fs.Getstat()") - stat, err := volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(ino)) - profiler.AddEventNow("after fs.Getstat()") - if err == nil { - reply.fsStatToStatStruct(stat) - } - - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, GetStatPathOp, profiler) - - return -} - -func (s *Server) RpcGetXAttr(in *GetXAttrRequest, reply *GetXAttrReply) (err error) { - var profiler = utils.NewProfilerIf(doProfiling, "getxattr") - - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - profiler.AddEventNow("before fs.GetXAttr()") - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil == err { - reply.AttrValue, err = volumeHandle.GetXAttr(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.AttrName) - } - profiler.AddEventNow("after fs.GetXAttr()") - - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, GetXattrOp, profiler) - - return -} - -func (s *Server) RpcGetXAttrPath(in *GetXAttrPathRequest, reply *GetXAttrReply) (err error) { - var profiler = utils.NewProfilerIf(doProfiling, "getxattr_path") - - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - profiler.AddEventNow("before fs.LookupPath()") - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - profiler.AddEventNow("after fs.LookupPath()") - if err != nil { - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, GetXattrPathOp, profiler) - return - } - - profiler.AddEventNow("before fs.GetXAttr()") - reply.AttrValue, err = volumeHandle.GetXAttr(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(ino), in.AttrName) - profiler.AddEventNow("after fs.GetXAttr()") - if err == nil { - reply.AttrValueSize = uint64(len(reply.AttrValue)) - } - - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, GetXattrPathOp, profiler) - - return -} - -func (s *Server) RpcLog(in *LogRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - logger.Info(in.Message) - return -} - -func (s *Server) RpcLookupPath(in *LookupPathRequest, reply *InodeReply) (err error) { - var profiler = utils.NewProfilerIf(doProfiling, "lookup_path") - - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - profiler.AddEventNow("before fs.LookupPath()") - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - profiler.AddEventNow("after fs.LookupPath()") - if err == nil { - reply.InodeNumber = int64(uint64(ino)) - } - - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, LookupPathOp, profiler) - - return -} - -func (s *Server) RpcLink(in *LinkRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - err = volumeHandle.Link(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.Basename, inode.InodeNumber(in.TargetInodeNumber)) - return -} - -func (s *Server) RpcLinkPath(in *LinkPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Split fullpath into parent dir and basename - parentDir, basename := splitPath(in.Fullpath) - - // Get the inode for the (source) parent dir - srcIno, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDir) - if err != nil { - return - } - - // Get the inode for the target - tgtIno, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.TargetFullpath) - if err != nil { - return - } - - // Do the link - err = volumeHandle.Link(inode.InodeRootUserID, inode.InodeGroupID(0), nil, srcIno, basename, tgtIno) - return -} - -func (s *Server) RpcListXAttr(in *ListXAttrRequest, reply *ListXAttrReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - reply.AttrNames, err = volumeHandle.ListXAttr(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber)) - return -} - -func (s *Server) RpcListXAttrPath(in *ListXAttrPathRequest, reply *ListXAttrReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - if err != nil { - return - } - - reply.AttrNames, err = volumeHandle.ListXAttr(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(ino)) - if err != nil { - return - } - - return -} - -func (s *Server) RpcLookup(in *LookupRequest, reply *InodeReply) (err error) { - enterGate() - defer leaveGate() - - var profiler = utils.NewProfilerIf(doProfiling, "lookup") - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - profiler.AddEventNow("before fs.Lookup()") - ino, err := volumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.Basename) - profiler.AddEventNow("after fs.Lookup()") - // line below is for testing fault injection - //err = blunder.AddError(err, blunder.TryAgainError) - if err == nil { - reply.InodeNumber = int64(uint64(ino)) - } - - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, LookupOp, profiler) - - return -} - -func (s *Server) RpcLookupPlus(in *LookupPlusRequest, reply *LookupPlusReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - ino, err := volumeHandle.Lookup(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.Basename) - if nil != err { - return - } - - stat, err := volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino) - if nil != err { - return - } - - reply.InodeNumber = int64(uint64(ino)) - reply.StatStruct.fsStatToStatStruct(stat) - - return -} - -func (s *Server) RpcAccess(in *AccessRequest, reply *InodeReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - ok := volumeHandle.Access(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, inode.InodeNumber(in.InodeNumber), inode.InodeMode(in.AccessMode)) - if ok { - err = nil - } else { - err = blunder.NewError(blunder.PermDeniedError, "EACCES") - } - - return -} - -func (s *Server) RpcMkdir(in *MkdirRequest, reply *InodeReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - ino, err := volumeHandle.Mkdir(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, inode.InodeNumber(in.InodeNumber), in.Basename, inode.InodeMode(in.FileMode)) - reply.InodeNumber = int64(uint64(ino)) - return -} - -func (s *Server) RpcMkdirPath(in *MkdirPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Ideally we would like all name/fullpath checking logic to be in the fs package, - // however since the fs.Create() and fs.Mkdir() APIs are inode-based, once we are - // inside those functions the fullpath is no longer available. The simplest solution is to - // just do the checking here. - err = fs.ValidateFullPath(in.Fullpath) - if err != nil { - return err - } - - // Split fullpath into parent dir and new basename - parentDir, basename := splitPath(in.Fullpath) - - // Get the inode for the parent dir - ino, err := volumeHandle.LookupPath(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, parentDir) - if err != nil { - return - } - - // Do the mkdir - _, err = volumeHandle.Mkdir(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, ino, basename, inode.InodeMode(in.FileMode)) - return -} - -func performMount(volumeHandle fs.VolumeHandle, authToken string, clientID uint64) (mountIDAsByteArray MountIDAsByteArray, mountIDAsString MountIDAsString, err error) { - var ( - i int - keepTrying bool - mount *mountStruct - ok bool - randByteSlice []byte - volume *volumeStruct - volumeName string - ) - - globals.volumesLock.Lock() - - volumeName = volumeHandle.VolumeName() - - volume, ok = globals.volumeMap[volumeName] - if !ok { - globals.volumesLock.Unlock() - err = fmt.Errorf("performMount(volumeHandle.VolumeName==\"%s\") cannot be found in globals.volumeMap", volumeName) - return - } - - if !volume.acceptingMountsAndLeaseRequests { - globals.volumesLock.Unlock() - err = fmt.Errorf("performMount(volumeHandle.VolumeName==\"%s\") called for dismounting volume", volumeName) - return - } - - keepTrying = true - for keepTrying { - randByteSlice = utils.FetchRandomByteSlice(len(mountIDAsByteArray)) - for i = 0; i < len(mountIDAsByteArray); i++ { - if i != 0 { - keepTrying = false // At least one of the bytes is non-zero... so it's a valid MountID - } - mountIDAsByteArray[i] = randByteSlice[i] - } - if !keepTrying { - _, keepTrying = globals.mountMapByMountIDAsByteArray[mountIDAsByteArray] - } - } - - mountIDAsString = MountIDAsString(base64.StdEncoding.EncodeToString(mountIDAsByteArray[:])) - - mount = &mountStruct{ - volume: volume, - mountIDAsByteArray: mountIDAsByteArray, - mountIDAsString: mountIDAsString, - authToken: authToken, - retryRpcUniqueID: clientID, - acceptingLeaseRequests: true, - leaseRequestMap: make(map[inode.InodeNumber]*leaseRequestStruct), - } - - volume.mountMapByMountIDAsByteArray[mountIDAsByteArray] = mount - volume.mountMapByMountIDAsString[mountIDAsString] = mount - - globals.mountMapByMountIDAsByteArray[mountIDAsByteArray] = mount - globals.mountMapByMountIDAsString[mountIDAsString] = mount - - globals.volumesLock.Unlock() - - return -} - -func (s *Server) RpcMountByAccountName(clientID uint64, in *MountByAccountNameRequest, reply *MountByAccountNameReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := fs.FetchVolumeHandleByAccountName(in.AccountName) - if nil == err { - _, reply.MountID, err = performMount(volumeHandle, in.AuthToken, clientID) - } - - return -} - -func (s *Server) RpcMountByVolumeName(clientID uint64, in *MountByVolumeNameRequest, reply *MountByVolumeNameReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := fs.FetchVolumeHandleByVolumeName(in.VolumeName) - if nil == err { - _, reply.MountID, err = performMount(volumeHandle, in.AuthToken, clientID) - } - - return -} - -func (s *Server) RpcUnmount(in *UnmountRequest, reply *Reply) (err error) { - var ( - leaseReleaseFinishedWG sync.WaitGroup - leaseReleaseStartWG sync.WaitGroup - mount *mountStruct - ok bool - volume *volumeStruct - ) - - enterGate() - defer leaveGate() - - globals.volumesLock.Lock() - - mount, ok = globals.mountMapByMountIDAsString[in.MountID] - if !ok { - globals.volumesLock.Unlock() - err = fmt.Errorf("RpcUnmount(in.MountID==\"%s\",) cannot be found in globals.volumeMap", in.MountID) - return - } - - leaseReleaseStartWG.Add(1) - mount.armReleaseOfAllLeasesWhileLocked(&leaseReleaseStartWG, &leaseReleaseFinishedWG) - - volume = mount.volume - - delete(volume.mountMapByMountIDAsByteArray, mount.mountIDAsByteArray) - delete(volume.mountMapByMountIDAsString, mount.mountIDAsString) - - delete(globals.mountMapByMountIDAsByteArray, mount.mountIDAsByteArray) - delete(globals.mountMapByMountIDAsString, mount.mountIDAsString) - - globals.volumesLock.Unlock() - - leaseReleaseStartWG.Done() - leaseReleaseFinishedWG.Wait() - - err = nil - return -} - -func (dirEnt *DirEntry) fsDirentToDirEntryStruct(fsDirent inode.DirEntry) { - dirEnt.InodeNumber = int64(uint64(fsDirent.InodeNumber)) - dirEnt.Basename = fsDirent.Basename - dirEnt.FileType = uint16(fsDirent.Type) - dirEnt.NextDirLocation = int64(fsDirent.NextDirLocation) -} - -func (s *Server) RpcReaddir(in *ReaddirRequest, reply *ReaddirReply) (err error) { - profiler := utils.NewProfilerIf(doProfiling, "readdir") - err = s.rpcReaddirInternal(in, reply, profiler) - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, ReaddirOp, profiler) - return -} - -func (s *Server) RpcReaddirByLoc(in *ReaddirByLocRequest, reply *ReaddirReply) (err error) { - profiler := utils.NewProfilerIf(doProfiling, "readdirByLoc") - err = s.rpcReaddirInternal(in, reply, profiler) - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, ReaddirByLocOp, profiler) - return -} - -func (s *Server) rpcReaddirInternal(in interface{}, reply *ReaddirReply, profiler *utils.Profiler) (err error) { - var ( - dirEnts []inode.DirEntry - flog logger.FuncCtx - i int - iH InodeHandle - inByLoc *ReaddirByLocRequest - inByName *ReaddirRequest - maxEntries uint64 - okByName bool - prevMarker interface{} - volumeHandle fs.VolumeHandle - ) - - inByName, okByName = in.(*ReaddirRequest) - if okByName { - iH = inByName.InodeHandle - maxEntries = inByName.MaxEntries - prevMarker = inByName.PrevDirEntName - flog = logger.TraceEnter("in.", inByName) - } else { - inByLoc, _ = in.(*ReaddirByLocRequest) - iH = inByLoc.InodeHandle - maxEntries = inByLoc.MaxEntries - prevMarker = inode.InodeDirLocation(inByLoc.PrevDirEntLocation) - flog = logger.TraceEnter("in.", inByLoc) - } - - enterGate() - defer leaveGate() - - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err = lookupVolumeHandleByMountIDAsString(iH.MountID) - if nil != err { - return - } - - profiler.AddEventNow("before fs.Readdir()") - dirEnts, _, _, err = volumeHandle.Readdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(iH.InodeNumber), maxEntries, prevMarker) - profiler.AddEventNow("after fs.Readdir()") - - if nil == err { - reply.DirEnts = make([]DirEntry, len(dirEnts)) - for i = range dirEnts { - reply.DirEnts[i].fsDirentToDirEntryStruct(dirEnts[i]) - } - } - - return -} - -func (s *Server) RpcReaddirPlus(in *ReaddirPlusRequest, reply *ReaddirPlusReply) (err error) { - profiler := utils.NewProfilerIf(doProfiling, "readdir_plus") - err = s.rpcReaddirPlusInternal(in, reply, profiler) - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, ReaddirPlusOp, profiler) - return -} - -func (s *Server) RpcReaddirPlusByLoc(in *ReaddirPlusByLocRequest, reply *ReaddirPlusReply) (err error) { - profiler := utils.NewProfilerIf(doProfiling, "readdir_plus_by_loc") - err = s.rpcReaddirPlusInternal(in, reply, profiler) - // Save profiler with server op stats - profiler.Close() - SaveProfiler(s, ReaddirPlusByLocOp, profiler) - return -} - -func (s *Server) rpcReaddirPlusInternal(in interface{}, reply *ReaddirPlusReply, profiler *utils.Profiler) (err error) { - var ( - dirEnts []inode.DirEntry - flog logger.FuncCtx - i int - iH InodeHandle - inByLoc *ReaddirPlusByLocRequest - inByName *ReaddirPlusRequest - maxEntries uint64 - okByName bool - prevMarker interface{} - statEnts []fs.Stat - volumeHandle fs.VolumeHandle - ) - - inByName, okByName = in.(*ReaddirPlusRequest) - if okByName { - iH = inByName.InodeHandle - maxEntries = inByName.MaxEntries - prevMarker = inByName.PrevDirEntName - flog = logger.TraceEnter("in.", inByName) - } else { - inByLoc, _ = in.(*ReaddirPlusByLocRequest) - iH = inByLoc.InodeHandle - maxEntries = inByLoc.MaxEntries - prevMarker = inode.InodeDirLocation(inByLoc.PrevDirEntLocation) - flog = logger.TraceEnter("in.", inByLoc) - } - - enterGate() - defer leaveGate() - - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err = lookupVolumeHandleByMountIDAsString(iH.MountID) - if err != nil { - return - } - - profiler.AddEventNow("before fs.ReaddirPlus()") - dirEnts, statEnts, _, _, err = volumeHandle.ReaddirPlus(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(iH.InodeNumber), maxEntries, prevMarker) - profiler.AddEventNow("after fs.ReaddirPlus()") - - if nil == err { - reply.DirEnts = make([]DirEntry, len(dirEnts)) - reply.StatEnts = make([]StatStruct, len(dirEnts)) // Assuming len(dirEnts) == len(statEnts) - for i = range dirEnts { - reply.DirEnts[i].fsDirentToDirEntryStruct(dirEnts[i]) - reply.StatEnts[i].fsStatToStatStruct(statEnts[i]) - } - } - - return -} - -func (s *Server) RpcReadSymlink(in *ReadSymlinkRequest, reply *ReadSymlinkReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - target, err := volumeHandle.Readsymlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber)) - reply.Target = target - return -} - -func (s *Server) RpcReadSymlinkPath(in *ReadSymlinkPathRequest, reply *ReadSymlinkReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Get the inode - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - if err != nil { - return - } - - target, err := volumeHandle.Readsymlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino) - reply.Target = target - return -} - -func (s *Server) RpcRemoveXAttr(in *RemoveXAttrRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - err = volumeHandle.RemoveXAttr(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.AttrName) - return -} - -func (s *Server) RpcRemoveXAttrPath(in *RemoveXAttrPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - if err != nil { - return - } - - err = volumeHandle.RemoveXAttr(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(ino), in.AttrName) - return -} - -func (s *Server) RpcRename(in *RenameRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - err = volumeHandle.Rename(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.SrcDirInodeNumber), in.SrcBasename, inode.InodeNumber(in.DstDirInodeNumber), in.DstBasename) - return -} - -func (s *Server) RpcRenamePath(in *RenamePathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Split fullpath into (source) parent dir and new basename - srcParentDir, srcBasename := splitPath(in.Fullpath) - - // Get the inode for the (source) parent dir - srcIno, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, srcParentDir) - if err != nil { - return - } - - // Split DstFullpath into (source) parent dir and new basename - dstParentDir, dstBasename := splitPath(in.DstFullpath) - - // Get the inode for the dest parent dir - dstIno, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dstParentDir) - if err != nil { - return - } - - // Do the rename - err = volumeHandle.Rename(inode.InodeRootUserID, inode.InodeGroupID(0), nil, srcIno, srcBasename, dstIno, dstBasename) - return -} - -func (s *Server) RpcMove(in *MoveRequest, reply *MoveReply) (err error) { - var ( - toDestroyInodeNumber inode.InodeNumber - ) - - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - toDestroyInodeNumber, err = volumeHandle.Move(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.SrcDirInodeNumber), in.SrcBasename, inode.InodeNumber(in.DstDirInodeNumber), in.DstBasename) - reply.ToDestroyInodeNumber = int64(toDestroyInodeNumber) - return -} - -func (s *Server) RpcDestroy(in *DestroyRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - err = volumeHandle.Destroy(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber)) - return -} - -func (s *Server) RpcResize(in *ResizeRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - err = volumeHandle.Resize(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.NewSize) - return -} - -func (s *Server) RpcRmdir(in *UnlinkRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - err = volumeHandle.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.Basename) - return -} - -func (s *Server) RpcRmdirPath(in *UnlinkPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Split fullpath into parent dir and new basename - parentDir, basename := splitPath(in.Fullpath) - - // Get the inode for the parent dir - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDir) - if err != nil { - return - } - - // Do the rmdir - err = volumeHandle.Rmdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, basename) - return -} - -func (s *Server) RpcSetstat(in *SetstatRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - stat := make(fs.Stat) - stat[fs.StatCRTime] = in.CRTimeNs - stat[fs.StatCTime] = in.CTimeNs - stat[fs.StatMTime] = in.MTimeNs - stat[fs.StatATime] = in.ATimeNs - stat[fs.StatSize] = in.Size - stat[fs.StatNLink] = in.NumLinks - // TODO: add in mode/userid/groupid? - err = volumeHandle.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), stat) - return -} - -func (s *Server) RpcSetTime(in *SetTimeRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - stat := make(fs.Stat) - - if uint64(0) != in.MTimeNs { - stat[fs.StatMTime] = in.MTimeNs - } - if uint64(0) != in.ATimeNs { - stat[fs.StatATime] = in.ATimeNs - } - - err = volumeHandle.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), stat) - - return -} - -func (s *Server) RpcSetTimePath(in *SetTimePathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Get the inode - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - if err != nil { - return - } - - stat := make(fs.Stat) - - if uint64(0) != in.MTimeNs { - stat[fs.StatMTime] = in.MTimeNs - } - if uint64(0) != in.ATimeNs { - stat[fs.StatATime] = in.ATimeNs - } - - err = volumeHandle.Setstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, stat) - - return -} - -func (s *Server) RpcSetXAttr(in *SetXAttrRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - err = volumeHandle.SetXAttr(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.AttrName, in.AttrValue, in.AttrFlags) - return -} - -func (s *Server) RpcSetXAttrPath(in *SetXAttrPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, in.Fullpath) - if err != nil { - return - } - - err = volumeHandle.SetXAttr(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(ino), in.AttrName, in.AttrValue, in.AttrFlags) - return -} - -func (s *Server) RpcStatVFS(in *StatVFSRequest, reply *StatVFS) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - statvfs, err := volumeHandle.StatVfs() - if err != nil { - return - } - - // Fill out the stats in the reply - reply.BlockSize = statvfs[fs.StatVFSBlockSize] - reply.FragmentSize = statvfs[fs.StatVFSFragmentSize] - reply.TotalBlocks = statvfs[fs.StatVFSTotalBlocks] - reply.FreeBlocks = statvfs[fs.StatVFSFreeBlocks] - reply.AvailBlocks = statvfs[fs.StatVFSAvailBlocks] - reply.TotalInodes = statvfs[fs.StatVFSTotalInodes] - reply.FreeInodes = statvfs[fs.StatVFSFreeInodes] - reply.AvailInodes = statvfs[fs.StatVFSAvailInodes] - reply.FileSystemID = statvfs[fs.StatVFSFilesystemID] - reply.MountFlags = statvfs[fs.StatVFSMountFlags] - reply.MaxFilenameLen = statvfs[fs.StatVFSMaxFilenameLen] - - return -} - -func (s *Server) RpcSymlink(in *SymlinkRequest, reply *InodeReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - ino, err := volumeHandle.Symlink(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, inode.InodeNumber(in.InodeNumber), in.Basename, in.Target) - reply.InodeNumber = int64(uint64(ino)) - return -} - -func (s *Server) RpcSymlinkPath(in *SymlinkPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Split fullpath into (source) parent dir and new basename - srcParentDir, srcBasename := splitPath(in.Fullpath) - - // Get the inode for the (source) parent dir - srcIno, err := volumeHandle.LookupPath(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, srcParentDir) - if err != nil { - return - } - - _, err = volumeHandle.Symlink(inode.InodeUserID(in.UserID), inode.InodeGroupID(in.GroupID), nil, srcIno, srcBasename, in.TargetFullpath) - return -} - -func (s *Server) RpcType(in *TypeRequest, reply *TypeReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - ftype, err := volumeHandle.GetType(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber)) - // Cast as a uint16 here to get the underlying DT_* constant - reply.FileType = uint16(ftype) - return -} - -func (s *Server) RpcUnlink(in *UnlinkRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - err = volumeHandle.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(in.InodeNumber), in.Basename) - return -} - -func (s *Server) RpcUnlinkPath(in *UnlinkPathRequest, reply *Reply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - volumeHandle, err := lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - - // Split fullpath into parent dir and new basename - parentDir, basename := splitPath(in.Fullpath) - - // Get the inode for the parent dir - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDir) - if err != nil { - return - } - - // Do the unlink - err = volumeHandle.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, basename) - return -} - -func (s *Server) RpcSnapShotCreate(in *SnapShotCreateRequest, reply *SnapShotCreateReply) (err error) { - var ( - fsVolumeHandle fs.VolumeHandle - inodeVolumeHandle inode.VolumeHandle - ) - - fsVolumeHandle, err = lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - inodeVolumeHandle, err = inode.FetchVolumeHandle(fsVolumeHandle.VolumeName()) - if nil != err { - return - } - - reply.SnapShotID, err = inodeVolumeHandle.SnapShotCreate(in.Name) - - return -} - -func (s *Server) RpcSnapShotDelete(in *SnapShotDeleteRequest, reply *SnapShotDeleteReply) (err error) { - var ( - fsVolumeHandle fs.VolumeHandle - inodeVolumeHandle inode.VolumeHandle - ) - - fsVolumeHandle, err = lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - inodeVolumeHandle, err = inode.FetchVolumeHandle(fsVolumeHandle.VolumeName()) - if nil != err { - return - } - - err = inodeVolumeHandle.SnapShotDelete(in.SnapShotID) - - return -} - -func (s *Server) RpcSnapShotListByID(in *SnapShotListRequest, reply *SnapShotListReply) (err error) { - var ( - fsVolumeHandle fs.VolumeHandle - headhunterVolumeHandle headhunter.VolumeHandle - ) - - fsVolumeHandle, err = lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(fsVolumeHandle.VolumeName()) - if nil != err { - return - } - - reply.List = headhunterVolumeHandle.SnapShotListByID(in.Reversed) - - return -} - -func (s *Server) RpcSnapShotListByName(in *SnapShotListRequest, reply *SnapShotListReply) (err error) { - var ( - fsVolumeHandle fs.VolumeHandle - headhunterVolumeHandle headhunter.VolumeHandle - ) - - fsVolumeHandle, err = lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(fsVolumeHandle.VolumeName()) - if nil != err { - return - } - - reply.List = headhunterVolumeHandle.SnapShotListByName(in.Reversed) - - return -} - -func (s *Server) RpcSnapShotListByTime(in *SnapShotListRequest, reply *SnapShotListReply) (err error) { - var ( - fsVolumeHandle fs.VolumeHandle - headhunterVolumeHandle headhunter.VolumeHandle - ) - - fsVolumeHandle, err = lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(fsVolumeHandle.VolumeName()) - if nil != err { - return - } - - reply.List = headhunterVolumeHandle.SnapShotListByTime(in.Reversed) - - return -} - -func (s *Server) RpcSnapShotLookupByName(in *SnapShotLookupByNameRequest, reply *SnapShotLookupByNameReply) (err error) { - var ( - fsVolumeHandle fs.VolumeHandle - headhunterVolumeHandle headhunter.VolumeHandle - ok bool - ) - - fsVolumeHandle, err = lookupVolumeHandleByMountIDAsString(in.MountID) - if nil != err { - return - } - headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(fsVolumeHandle.VolumeName()) - if nil != err { - return - } - - reply.SnapShot, ok = headhunterVolumeHandle.SnapShotLookupByName(in.Name) - if ok { - err = nil - } else { - err = blunder.NewError(blunder.NotFoundError, "ENOENT") - } - - return -} diff --git a/jrpcfs/io.go b/jrpcfs/io.go deleted file mode 100644 index 55a17bce..00000000 --- a/jrpcfs/io.go +++ /dev/null @@ -1,484 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "container/list" - "fmt" - "io" - "net" - "sync" - "time" - "unsafe" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/utils" -) - -// Server handle, used to track io-related ops -var qserver *Server - -var ioListener net.Listener - -func ioServerUp(ipAddr string, fastPortString string) { - var err error - - qserver = NewServer() - - ioListener, err = net.Listen("tcp", net.JoinHostPort(ipAddr, fastPortString)) - if err != nil { - logger.ErrorfWithError(err, "net.Listen %s:%s failed", ipAddr, fastPortString) - return - } - - globals.connLock.Lock() - globals.listeners = append(globals.listeners, ioListener) - globals.connLock.Unlock() - - //logger.Infof("Starting to listen on %s:%s", ipAddr, fastPortString) - globals.listenersWG.Add(1) - go ioServerLoop() -} - -func ioServerDown() { - _ = ioListener.Close() - DumpIfNecessary(qserver) - dumpRunningWorkers() - stopServerProfiling(qserver) -} - -func ioServerLoop() { - for { - conn, err := ioListener.Accept() - if err != nil { - if !globals.halting { - logger.ErrorfWithError(err, "net.Accept failed for IO listener\n") - } - globals.listenersWG.Done() - return - } - - globals.connWG.Add(1) - - globals.connLock.Lock() - elm := globals.connections.PushBack(conn) - globals.connLock.Unlock() - - go func(myConn net.Conn, myElm *list.Element) { - ioHandle(myConn) - globals.connLock.Lock() - globals.connections.Remove(myElm) - - // There is a race condition where the connection could have been - // closed in Down(). However, closing it twice is okay. - myConn.Close() - globals.connLock.Unlock() - globals.connWG.Done() - }(conn, elm) - } -} - -var debugConcurrency = false - -var concWorkerLock sync.Mutex -var numConcWorkers int = 0 // number of concurrent workers -var hwmConcWorkers int = 0 // high water mark of concurrent workers - -var timesIncWorkers int = 0 -var timesDecWorkers int = 0 -var timesLevelEntered []int = make([]int, 128) -var timesLevelExited []int = make([]int, 128) - -var concDuration []time.Duration = make([]time.Duration, 128) -var concStopwatch []utils.Stopwatch = make([]utils.Stopwatch, 128) - -func enterWorkerLevel(level int) { - timesLevelEntered[level]++ - - concStopwatch[level].Restart() -} - -func exitWorkerLevel(level int) { - timesLevelExited[level]++ - - if concStopwatch[level].IsRunning { - concStopwatch[level].Stop() - concDuration[level] += concStopwatch[level].Elapsed() - } -} - -func incRunningWorkers() { - enterGate() - - if debugConcurrency { - concWorkerLock.Lock() - timesIncWorkers++ - - // Record how long we spent in the previous level - exitWorkerLevel(numConcWorkers) - - numConcWorkers++ - - // Time how long we are at this level start the clock - enterWorkerLevel(numConcWorkers) - - if numConcWorkers > hwmConcWorkers { - hwmConcWorkers = numConcWorkers - } - concWorkerLock.Unlock() - } -} - -func decRunningWorkers() { - if debugConcurrency { - concWorkerLock.Lock() - timesDecWorkers++ - - // Record how long we spent in the previous level - exitWorkerLevel(numConcWorkers) - - numConcWorkers-- - - // Time how long we are at this level start the clock - enterWorkerLevel(numConcWorkers) - - concWorkerLock.Unlock() - } - - leaveGate() -} - -// It'd be nice if the time package supported a check for nil duration, but it doesn't. -var nilDuration time.Duration - -func dumpRunningWorkers() { - if !debugConcurrency { - return - } - - var totalTimeMs int64 = 0 - - fmt.Printf("running workers: %d, max running workers: %d\n", numConcWorkers, hwmConcWorkers) - - for i := range concDuration { - if concDuration[i] != nilDuration { - timeMs := concDuration[i].Nanoseconds() / int64(time.Millisecond) - totalTimeMs += int64(i) * timeMs - fmt.Printf(" %v workers: %v ms\n", i, timeMs) - } - } - - fmt.Printf(" total worker-thread runtime: %v ms\n", totalTimeMs) - - fmt.Printf(" times incWorkers called: %v decWorkers called: %v\n", timesIncWorkers, timesDecWorkers) - for i := range timesLevelEntered { - if timesLevelEntered[i] > 0 { - fmt.Printf(" level %v entered: %d exited %v\n", i, timesLevelEntered[i], timesLevelExited[i]) - } - } -} - -// Variable to control debug output -var printDebugLogs bool = false -var debugPutGet bool = false - -func getRequest(conn net.Conn, ctx *ioContext) (err error) { - // "cast" request to bytes before reading into it - reqBytes := makeBytesReq(&ctx.req) - - bytesRead, err := io.ReadFull(conn, reqBytes) - if err != nil { - if err != io.EOF { - logger.Errorf("Failed to read request from the socket: %v", err) - } - return err - } - - // "cast" bytes to request - makeReq(reqBytes, &ctx.req) - - if debugPutGet { - logger.Infof("Got %v bytes, request: %+v", bytesRead, ctx.req) - } - - if ctx.req.opType == 1001 { - // Write op - ctx.op = WriteOp - } else if ctx.req.opType == 1002 { - // Read op - ctx.op = ReadOp - } else { - return fmt.Errorf("getRequest: unsupported op %v!", ctx.req.opType) - } - - // For writes, get write data - if ctx.op == WriteOp { - if debugPutGet { - logger.Infof("Reading %v bytes of write data, ctx.data len is %v.", ctx.req.length, len(ctx.data)) - } - - ctx.data = make([]byte, ctx.req.length) - - _, err = io.ReadFull(conn, ctx.data) - if err != nil { - logger.Infof("Failed to read write buffer from the socket uint64_t.") - return err - } - // NOTE: Suppress for now, will be counted in next event - //profiler.AddEventNow("after get write buf") - } - - return nil -} - -func putResponseWrite(conn net.Conn, buf []byte) (err error) { - var ( - currentIndex = int(0) - limitIndex = len(buf) - numBytesWritten int - ) - - err = nil // default... in case cur == len(buf) already - - for currentIndex < limitIndex { - numBytesWritten, err = conn.Write(buf[currentIndex:]) - if nil != err { - return - } - currentIndex += numBytesWritten - } - - return -} - -func putResponse(conn net.Conn, ctx *ioContext) (err error) { - var ( - respBytes []byte - ) - - if (ctx.op != ReadOp) && (ctx.op != WriteOp) { - // We only support read and write - return fmt.Errorf("putResponse: unsupported opType %v", ctx.op) - } - - // NOTE: the far end expects errno, ioSize, (if a read), a buffer - - // "cast" response to bytes and send them - respBytes = makeBytesResp(&ctx.resp) - - // Send response header - err = putResponseWrite(conn, respBytes) - if nil != err { - logger.Infof("putResponse() failed to send ctx.resp: %v", err) - return - } - - // If (non-zero length) Read Payload, send it as well - if (ctx.op == ReadOp) && (len(ctx.data) > 0) { - err = putResponseWrite(conn, ctx.data) - if nil != err { - logger.Infof("putResponse() failed to send ctx.data: %v", err) - return - } - } - - err = nil - return -} - -func putUint64(conn net.Conn, fieldName string, value uint64, field []byte) (err error) { - return putUint64Profiled(conn, fieldName, value, field, nil) -} - -// Rather than allocate byte slice to write from each time we are called, we -// try to be more efficient by reusing a byte slice that is passed in by the caller. -func putUint64Profiled(conn net.Conn, fieldName string, value uint64, field []byte, profiler *utils.Profiler) (err error) { - - // "cast" value to bytes before sending it - valBytes := makeBytesUint64(value) - bytesWanted := len(valBytes) - - nBytes, err := conn.Write(valBytes) - profiler.AddEventNow("after conn.Write") - if err != nil { - logger.Infof("Failed to write %v", fieldName) - return - } - if nBytes != bytesWanted { - logger.Infof("Error, wanted %v bytes, got %v", bytesWanted, nBytes) - return fmt.Errorf("Error, wanted %v bytes, got %v", bytesWanted, nBytes) - } - - if debugPutGet { - logger.Infof("Wrote %v bytes, %v = 0x%x.", nBytes, fieldName, value) - } - return -} - -type ioRequest struct { - opType uint64 - mountID MountIDAsByteArray - inodeID uint64 - offset uint64 - length uint64 -} - -type ioResponse struct { - errno uint64 //out - ioSize uint64 //out -} - -type ioContext struct { - op OpType - req ioRequest - resp ioResponse - // read/writeData buf* (in: write; out: read) - // Ideally this would be a pointer (?) - data []byte -} - -const ioRequestSize int = 8 + 16 + 8 + 8 + 8 -const ioResponseSize int = 8 + 8 - -func makeBytesReq(req *ioRequest) []byte { - mem := *(*[ioRequestSize]byte)(unsafe.Pointer(req)) - return mem[:] -} - -func makeBytesResp(resp *ioResponse) []byte { - mem := *(*[ioResponseSize]byte)(unsafe.Pointer(resp)) - return mem[:] -} - -func makeBytesUint64(value uint64) []byte { - mem := *(*[8]byte)(unsafe.Pointer(&value)) - return mem[:] -} - -func makeReq(bytes []byte, req *ioRequest) { - *req = *(*ioRequest)(unsafe.Pointer(&bytes[0])) -} - -func ioHandle(conn net.Conn) { - var ( - volumeHandle fs.VolumeHandle - ) - - // NOTE: This function runs in a goroutine and only processes - // one request at a time. - ctxStorage := ioContext{op: InvalidOp} - ctx := &ctxStorage - - if printDebugLogs { - logger.Infof("got a connection - starting read/write io thread") - } - - for { - if printDebugLogs { - logger.Infof("Waiting for RPC request; ctx.data size is %v.", len(ctx.data)) - } - - // Get RPC request - err := getRequest(conn, ctx) - // NOTE: Suppress this for now, we're seeing not much time spent up to here - //profiler.AddEventNow("after get request") - if err != nil { - //logger.Infof("Connection terminated; returning.") - return - } - - // Wait until here to increment this as a worker; else we count wait time as work time. - incRunningWorkers() - - // Taking stats *after* socket read, because otherwise we unintentionally count wait time. - profiler := utils.NewProfilerIf(doProfiling, "") // We don't know the op type yet, gets set by SaveProfiler(). - - if debugPutGet { - logger.Infof("Got request: %+v", ctx.req) - } - - switch ctx.op { - case WriteOp: - if globals.dataPathLogging || printDebugLogs { - logger.Tracef(">> ioWrite in.{InodeHandle:{MountID:%v InodeNumber:%v} Offset:%v Buf.size:%v Buf.", - ctx.req.mountID, ctx.req.inodeID, ctx.req.offset, len(ctx.data)) - } - - profiler.AddEventNow("before fs.Write()") - volumeHandle, err = lookupVolumeHandleByMountIDAsByteArray(ctx.req.mountID) - if err == nil { - ctx.resp.ioSize, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(ctx.req.inodeID), ctx.req.offset, ctx.data, profiler) - } - profiler.AddEventNow("after fs.Write()") - - stats.IncrementOperationsAndBucketedBytes(stats.JrpcfsIoWrite, ctx.resp.ioSize) - - if globals.dataPathLogging || printDebugLogs { - logger.Tracef("<< ioWrite errno:%v out.Size:%v", ctx.resp.errno, ctx.resp.ioSize) - } - - case ReadOp: - if globals.dataPathLogging || printDebugLogs { - logger.Tracef(">> ioRead in.{InodeHandle:{MountID:%v InodeNumber:%v} Offset:%v Length:%v}", ctx.req.mountID, ctx.req.inodeID, ctx.req.offset, ctx.req.length) - } - - profiler.AddEventNow("before fs.Read()") - volumeHandle, err = lookupVolumeHandleByMountIDAsByteArray(ctx.req.mountID) - if err == nil { - ctx.data, err = volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(ctx.req.inodeID), ctx.req.offset, ctx.req.length, profiler) - } - profiler.AddEventNow("after fs.Read()") - - // Set io size in response - ctx.resp.ioSize = uint64(len(ctx.data)) - - stats.IncrementOperationsAndBucketedBytes(stats.JrpcfsIoRead, ctx.resp.ioSize) - - if globals.dataPathLogging || printDebugLogs { - logger.Tracef("<< ioRead errno:%v out.Buf.size:%v out.Buf.", ctx.resp.errno, len(ctx.data)) - } - - default: - // Hmmm, this should have been caught by getRequest... - logger.Errorf("Error, unsupported op %v", ctx.op) - decRunningWorkers() - return - } - - // Set error in context - ctx.resp.errno = uint64(blunder.Errno(err)) - - // Write response - err = putResponse(conn, ctx) - // TODO: Enable if we want to see this event specifically. - // Otherwise this will show up under "remaining time". - //profiler.AddEventNow("after rpc send response") - if err != nil { - decRunningWorkers() - return - } - - // Save profiler with server op stats. Close it first so that save time isn't counted. - profiler.Close() - SaveProfiler(qserver, ctx.op, profiler) - - // TODO: No sync.Pool for now, just alloc on the stack - // Return context struct to pool - // ioContextPool.Put(ctx) - - // Reset data buffer size in ctx to its full size - // ctx.data = dataStorage[:0] - ctx.data = nil - - if printDebugLogs { - logger.Infof("Done with op, back to beginning") - } - - decRunningWorkers() - } -} diff --git a/jrpcfs/io_stats b/jrpcfs/io_stats deleted file mode 100755 index 19373453..00000000 --- a/jrpcfs/io_stats +++ /dev/null @@ -1,138 +0,0 @@ -#! /usr/bin/awk -f -# -# Command to parse periodic stats logs from proxyfsd.log -# and count the average duration. -# - -BEGIN { - PROCINFO["sorted_in"] = "@ind_num_asc" - defaultEventString = 36 - maxEventString = 36 -} -{ - # Skip lines that we don't care about - if ( $0 ~ /Periodic stats:/ ) - { - # op is the 8th field. Strip the comma from the end. - split($8, tempOp, ",") - key = tempOp[1] - - #print "*** op is " key - - #print "Number of stats found: " $9 " " $10 " " $11 " " $12 - split($12, tempArray, ":") - numberOfOps = tempArray[1] - #print "Number of stats found: " numberOfOps - numOps[key] += numberOfOps - - # Count fields, total duration is the ninth-last one - #print "op is " key ", num ops is " numberOfOps ", duration is " $(NF-8) - totalOpDurationsMs[key] += $(NF-8) - - # Parse out individual events, if present - eventNumber = 1 - split($0, events, ";") - for (ev in events) - { - #print "raw event " ev " is " events[ev] - - # Look for "+" that each event has, but skip the "total duration" one, since it isn't an event - if ((events[ev] ~ /\+/) && (events[ev] !~ /total duration/)) - { - #print "event " eventNumber " is " events[ev] - - # Event name is everything on left of "+", duration stuff on the right - split(events[ev], eventDetail, "+") - - eventName = eventDetail[1] - - # Keep track of event name size, for print formatting later - if ( length(eventName) > maxEventString ) { - # +3 is for event number plus space - maxEventString = length(eventName) + 3 - } - - # Event duration is the first field on rhs - split(eventDetail[2], timingDetail, " ") - #print " event duration: " timingDetail[1] - - #print "storing OpEventMs["key"]["eventNumber " " eventName"] += " timingDetail[1] - OpEventMs[key][eventNumber " " eventName] += timingDetail[1] - - # Number of events is also on the rhs - if (timingDetail[6] ~ /\[numEvents/) - { - # get the ] off the end of the number of events - split(timingDetail[7], numEventsDetail, "]") - #print "number of events is ", numEventsDetail[1] - OpEventCount[key][eventNumber " " eventName] += numEventsDetail[1] - } - - eventNumber = eventNumber + 1 - } - } - } -} -END { - allOpsDurationMs = 0 - allOpsCount = 0 - addlEventString = 0 - - if (maxEventString > defaultEventString) - { - addlEventString = maxEventString - defaultEventString - } - - printf "\n" - printf " %" 24 + addlEventString "s\n", - " Total Average" - printf "Operation Quantity %" 30 + addlEventString "s\n", - "Duration (ms) duration (ms)" - printf "--------- -------- %" 30 + addlEventString "s\n", - "------------- -------------" - for (op in numOps) - { - if ( numOps[op] > 0 ) - { - if (totalOpDurationsMs[op] > 0) - { - printf "%-12s %5d %" 9 + addlEventString ".0f %9.2f %8.2f %%\n", - op, numOps[op], totalOpDurationsMs[op], totalOpDurationsMs[op]/numOps[op], - totalOpDurationsMs[op]/totalOpDurationsMs[op] * 100 - - allOpsDurationMs += totalOpDurationsMs[op] - allOpsCount += numOps[op] - - if ( length(OpEventMs[op]) != 0 ) - { - countedDurationMs = 0 - for (event in OpEventMs[op]) - { - #print "OpEventMs event is " event " = " OpEventMs[key][event] - printf " (%-" maxEventString "s) %9.0f %9.2f %8.2f %% (hits %5d/%5d)\n", - event, OpEventMs[op][event], OpEventMs[op][event]/numOps[op], - OpEventMs[op][event]/totalOpDurationsMs[op] * 100, - OpEventCount[op][event], numOps[op] - countedDurationMs += OpEventMs[op][event] - } - - # Unaccounted doesn't make sense with these stats since we always have - # all the data. Comment this out. - #remainingDurationMs = totalOpDurationsMs[op] - countedDurationMs - #printf " (%-36s) %9.0f %9.2f %8.2f %%\n", - # " unaccounted", remainingDurationMs, remainingDurationMs/numOps[op], - # remainingDurationMs/totalOpDurationsMs[op] * 100 - } - } else { - printf "%-12s %5d %" 9 + addlEventString ".0f %9.2f %8.2f %%\n", - op, numOps[op], totalOpDurationsMs[op], totalOpDurationsMs[op]/numOps[op], 100 - } - - print " " - } - } - - printf "%-12s %5d %" 9 + addlEventString ".0f\n", - "All ops", allOpsCount, allOpsDurationMs - printf "\n" -} diff --git a/jrpcfs/lease.go b/jrpcfs/lease.go deleted file mode 100644 index c0217779..00000000 --- a/jrpcfs/lease.go +++ /dev/null @@ -1,2112 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "container/list" - "encoding/json" - "fmt" - "runtime" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/sortedmap" -) - -// RpcLease is called to either request a Shared|Exclusive Lease or to -// Promote|Demote|Release a granted Shared|Exclusive|either Lease. -// -func (s *Server) RpcLease(in *LeaseRequest, reply *LeaseReply) (err error) { - var ( - additionalEvictionsInitiated uint64 - additionalEvictionsNeeded uint64 - delayedUnmount *mountStruct - delayedUnmountLeaseReleaseFinishedWG sync.WaitGroup - delayedUnmountLeaseReleaseStartWG sync.WaitGroup - delayedUnmountListElement *list.Element - inodeLease *inodeLeaseStruct - inodeLeaseFromLRU *inodeLeaseStruct - inodeLeaseLRUElement *list.Element - inodeNumber inode.InodeNumber - leaseRequestOperation *leaseRequestOperationStruct - mount *mountStruct - ok bool - replyToReturn *LeaseReply - volume *volumeStruct - ) - - enterGate() - defer leaveGate() - - switch in.LeaseRequestType { - case LeaseRequestTypeShared: - case LeaseRequestTypePromote: - case LeaseRequestTypeExclusive: - case LeaseRequestTypeDemote: - case LeaseRequestTypeRelease: - default: - reply.LeaseReplyType = LeaseReplyTypeDenied - err = fmt.Errorf("LeaseRequestType %v not supported", in.LeaseRequestType) - err = blunder.AddError(err, blunder.BadLeaseRequest) - return - } - - inodeNumber = inode.InodeNumber(in.InodeHandle.InodeNumber) - - globals.volumesLock.Lock() - - mount, ok = globals.mountMapByMountIDAsString[in.MountID] - if !ok { - globals.volumesLock.Unlock() - reply.LeaseReplyType = LeaseReplyTypeDenied - err = fmt.Errorf("MountID %s not found in jrpcfs globals.mountMapByMountIDAsString", in.MountID) - err = blunder.AddError(err, blunder.BadMountIDError) - return - } - - volume = mount.volume - - evtlog.Record(evtlog.FormatLeaseRequest, volume.volumeName, string(in.InodeHandle.MountID), uint64(inode.InodeNumber(in.InodeHandle.InodeNumber)), uint32(in.LeaseRequestType)) - defer func() { - evtlog.Record(evtlog.FormatLeaseReply, volume.volumeName, string(in.InodeHandle.MountID), uint64(inode.InodeNumber(in.InodeHandle.InodeNumber)), uint32(in.LeaseRequestType), uint32(reply.LeaseReplyType)) - }() - - if (in.LeaseRequestType == LeaseRequestTypeShared) || (in.LeaseRequestType == LeaseRequestTypeExclusive) { - if !volume.acceptingMountsAndLeaseRequests || !mount.acceptingLeaseRequests { - globals.volumesLock.Unlock() - reply.LeaseReplyType = LeaseReplyTypeDenied - err = fmt.Errorf("LeaseRequestType %v not allowed while dismounting Volume", in.LeaseRequestType) - err = blunder.AddError(err, blunder.BadLeaseRequest) - return - } - inodeLease, ok = volume.inodeLeaseMap[inodeNumber] - if !ok { - inodeLease = &inodeLeaseStruct{ - volume: volume, - InodeNumber: inodeNumber, - leaseState: inodeLeaseStateNone, - beingEvicted: false, - requestChan: make(chan *leaseRequestOperationStruct), - stopChan: make(chan struct{}), - sharedHoldersList: list.New(), - promotingHolder: nil, - exclusiveHolder: nil, - releasingHoldersList: list.New(), - requestedList: list.New(), - lastGrantTime: time.Time{}, - lastInterruptTime: time.Time{}, - interruptsSent: 0, - longAgoTimer: &time.Timer{}, - interruptTimer: &time.Timer{}, - } - - volume.inodeLeaseMap[inodeNumber] = inodeLease - inodeLease.lruElement = volume.inodeLeaseLRU.PushBack(inodeLease) - - volume.leaseHandlerWG.Add(1) - go inodeLease.handler() - - if uint64(volume.inodeLeaseLRU.Len()) >= volume.activeLeaseEvictHighLimit { - additionalEvictionsNeeded = (volume.activeLeaseEvictHighLimit - volume.activeLeaseEvictLowLimit) - volume.ongoingLeaseEvictions - additionalEvictionsInitiated = 0 - inodeLeaseLRUElement = volume.inodeLeaseLRU.Front() - for (nil != inodeLeaseLRUElement) && (additionalEvictionsInitiated < additionalEvictionsNeeded) { - inodeLeaseFromLRU = inodeLeaseLRUElement.Value.(*inodeLeaseStruct) - if !inodeLeaseFromLRU.beingEvicted { - inodeLeaseFromLRU.beingEvicted = true - close(inodeLeaseFromLRU.stopChan) - additionalEvictionsInitiated++ - } - inodeLeaseLRUElement = inodeLeaseLRUElement.Next() - } - volume.ongoingLeaseEvictions += additionalEvictionsInitiated - } - } - } else { // in.LeaseRequestType is one of LeaseRequestType{Promote|Demote|Release} - inodeLease, ok = volume.inodeLeaseMap[inodeNumber] - if !ok { - globals.volumesLock.Unlock() - reply.LeaseReplyType = LeaseReplyTypeDenied - err = fmt.Errorf("LeaseRequestType %v not allowed for non-existent Lease [case 1]", in.LeaseRequestType) - err = blunder.AddError(err, blunder.BadLeaseRequest) - return - } - } - - // Send Lease Request Operation to *inodeLeaseStruct.handler() - // - // Note that we still hold the volumesLock, so inodeLease can't disappear out from under us - - leaseRequestOperation = &leaseRequestOperationStruct{ - mount: mount, - inodeLease: inodeLease, - LeaseRequestType: in.LeaseRequestType, - replyChan: make(chan *LeaseReply), - } - - inodeLease.requestChan <- leaseRequestOperation - - globals.volumesLock.Unlock() - - replyToReturn = <-leaseRequestOperation.replyChan - reply.LeaseReplyType = replyToReturn.LeaseReplyType - - globals.volumesLock.Lock() - - delayedUnmountLeaseReleaseStartWG.Add(1) - - delayedUnmountListElement = volume.delayedUnmountList.Front() - - for nil != delayedUnmountListElement { - delayedUnmount = delayedUnmountListElement.Value.(*mountStruct) - _ = volume.delayedUnmountList.Remove(delayedUnmountListElement) - - delayedUnmount.armReleaseOfAllLeasesWhileLocked(&delayedUnmountLeaseReleaseStartWG, &delayedUnmountLeaseReleaseFinishedWG) - - delete(volume.mountMapByMountIDAsByteArray, delayedUnmount.mountIDAsByteArray) - delete(volume.mountMapByMountIDAsString, delayedUnmount.mountIDAsString) - - delete(globals.mountMapByMountIDAsByteArray, delayedUnmount.mountIDAsByteArray) - delete(globals.mountMapByMountIDAsString, delayedUnmount.mountIDAsString) - - delayedUnmountListElement = volume.delayedUnmountList.Front() - } - - globals.volumesLock.Unlock() - - delayedUnmountLeaseReleaseStartWG.Done() - delayedUnmountLeaseReleaseFinishedWG.Wait() - - err = nil - return -} - -func (inodeLease *inodeLeaseStruct) handler() { - var ( - leaseRequestOperation *leaseRequestOperationStruct - ) - - for { - select { - case leaseRequestOperation = <-inodeLease.requestChan: - inodeLease.handleOperation(leaseRequestOperation) - case _ = <-inodeLease.longAgoTimer.C: - inodeLease.handleLongAgoTimerPop() - case _ = <-inodeLease.interruptTimer.C: - inodeLease.handleInterruptTimerPop() - case _, _ = <-inodeLease.stopChan: - inodeLease.handleStopChanClose() // will not return - } - } -} - -func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *leaseRequestOperationStruct) { - var ( - err error - leaseReply *LeaseReply - leaseRequest *leaseRequestStruct - leaseRequestElement *list.Element - ok bool - rpcInterrupt *RPCInterrupt - rpcInterruptBuf []byte - sharedHolderLeaseRequest *leaseRequestStruct - sharedHolderListElement *list.Element - ) - - globals.volumesLock.Lock() - defer globals.volumesLock.Unlock() - - inodeLease.volume.inodeLeaseLRU.MoveToBack(inodeLease.lruElement) - - switch leaseRequestOperation.LeaseRequestType { - case LeaseRequestTypeShared: - _, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] - if ok { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] returned !ok - leaseRequest = &leaseRequestStruct{ - mount: leaseRequestOperation.mount, - inodeLease: inodeLease, - requestState: leaseRequestStateSharedRequested, - replyChan: leaseRequestOperation.replyChan, - } - leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] = leaseRequest - switch inodeLease.leaseState { - case inodeLeaseStateNone: - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequest.replyChan <- leaseReply - case inodeLeaseStateSharedGrantedRecently: - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - inodeLease.lastGrantTime = time.Time{} - inodeLease.longAgoTimer = &time.Timer{} - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequest.replyChan <- leaseReply - case inodeLeaseStateSharedGrantedLongAgo: - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequest.replyChan <- leaseReply - case inodeLeaseStateSharedPromoting: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedReleasing: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveGrantedLongAgo: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - inodeLease.leaseState = inodeLeaseStateExclusiveDemoting - inodeLease.demotingHolder = inodeLease.exclusiveHolder - inodeLease.demotingHolder.requestState = leaseRequestStateExclusiveDemoting - inodeLease.exclusiveHolder = nil - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeDemote, - InodeNumber: int64(inodeLease.InodeNumber), - } - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) - } - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.demotingHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(inodeLease.demotingHolder.mount.retryRpcUniqueID, rpcInterruptBuf) - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - case inodeLeaseStateExclusiveDemoting: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveReleasing: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeShared, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) - } - } - case LeaseRequestTypePromote: - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] - if ok { - if leaseRequestStateSharedGranted == leaseRequest.requestState { - switch inodeLease.leaseState { - case inodeLeaseStateNone: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateNone") - case inodeLeaseStateSharedGrantedRecently: - if nil == inodeLease.promotingHolder { - _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - if 0 == inodeLease.sharedHoldersList.Len() { - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypePromoted, - } - leaseRequestOperation.replyChan <- leaseReply - inodeLease.exclusiveHolder = leaseRequest - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } else { - inodeLease.promotingHolder = leaseRequest - leaseRequest.replyChan = leaseRequestOperation.replyChan - } - } else { // nil != inodeLease.promotingHolder - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateSharedGrantedLongAgo: - _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - if 0 == inodeLease.sharedHoldersList.Len() { - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypePromoted, - } - leaseRequestOperation.replyChan <- leaseReply - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } else { - inodeLease.leaseState = inodeLeaseStateSharedPromoting - inodeLease.promotingHolder = leaseRequest - leaseRequest.requestState = leaseRequestStateSharedPromoting - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) - } - for nil != inodeLease.sharedHoldersList.Front() { - leaseRequestElement = inodeLease.sharedHoldersList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.sharedHoldersList.Remove(leaseRequestElement) - leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) - leaseRequest.requestState = leaseRequestStateSharedReleasing - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - } - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - } - case inodeLeaseStateSharedPromoting: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedPromoting") - case inodeLeaseStateSharedReleasing: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedReleasing") - case inodeLeaseStateSharedExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveGrantedRecently") - case inodeLeaseStateExclusiveGrantedLongAgo: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveGrantedLongAgo") - case inodeLeaseStateExclusiveDemoting: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveDemoting") - case inodeLeaseStateExclusiveReleasing: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveReleasing") - case inodeLeaseStateExclusiveExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypePromote, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) - } - } else { // leaseRequestStateSharedGranted != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] returned !ok - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case LeaseRequestTypeExclusive: - _, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] - if ok { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] returned !ok - leaseRequest = &leaseRequestStruct{ - mount: leaseRequestOperation.mount, - inodeLease: inodeLease, - requestState: leaseRequestStateExclusiveRequested, - replyChan: leaseRequestOperation.replyChan, - } - leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] = leaseRequest - switch inodeLease.leaseState { - case inodeLeaseStateNone: - leaseRequest.requestState = leaseRequestStateExclusiveGranted - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = leaseRequest - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - leaseRequest.replyChan <- leaseReply - case inodeLeaseStateSharedGrantedRecently: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedGrantedLongAgo: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - inodeLease.leaseState = inodeLeaseStateSharedReleasing - for nil != inodeLease.sharedHoldersList.Front() { - sharedHolderListElement = inodeLease.sharedHoldersList.Front() - sharedHolderLeaseRequest = sharedHolderListElement.Value.(*leaseRequestStruct) - _ = inodeLease.sharedHoldersList.Remove(sharedHolderListElement) - sharedHolderLeaseRequest.requestState = leaseRequestStateSharedReleasing - sharedHolderLeaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(sharedHolderLeaseRequest) - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) - } - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(sharedHolderLeaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(sharedHolderLeaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - } - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - case inodeLeaseStateSharedPromoting: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedReleasing: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateSharedExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveGrantedLongAgo: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - inodeLease.leaseState = inodeLeaseStateExclusiveReleasing - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveReleasing - inodeLease.exclusiveHolder.listElement = inodeLease.releasingHoldersList.PushBack(inodeLease.exclusiveHolder) - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) - } - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.exclusiveHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(inodeLease.exclusiveHolder.mount.retryRpcUniqueID, rpcInterruptBuf) - inodeLease.exclusiveHolder = nil - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - case inodeLeaseStateExclusiveDemoting: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveReleasing: - leaseRequest.listElement = inodeLease.requestedList.PushBack(leaseRequest) - case inodeLeaseStateExclusiveExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeExclusive, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) - } - } - case LeaseRequestTypeDemote: - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] - if ok { - switch inodeLease.leaseState { - case inodeLeaseStateNone: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - case inodeLeaseStateSharedGrantedRecently: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - case inodeLeaseStateSharedGrantedLongAgo: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - case inodeLeaseStateSharedPromoting: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - case inodeLeaseStateSharedReleasing: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - case inodeLeaseStateSharedExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unexpected inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - if leaseRequestStateExclusiveGranted == leaseRequest.requestState { - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.exclusiveHolder = nil - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDemoted, - } - leaseRequestOperation.replyChan <- leaseReply - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateSharedGranted - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - leaseRequestElement = nil - } - } - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } else { // leaseRequestStateExclusiveGranted == leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveGrantedLongAgo: - if leaseRequestStateExclusiveGranted == leaseRequest.requestState { - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.requestState = leaseRequestStateSharedGranted - inodeLease.exclusiveHolder = nil - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDemoted, - } - leaseRequestOperation.replyChan <- leaseReply - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveDemoting: - if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - inodeLease.demotingHolder = nil - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDemoted, - } - leaseRequestOperation.replyChan <- leaseReply - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateSharedGranted - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateSharedRequested != leaseRequest.requestState - leaseRequestElement = nil - } - } - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } else { // leaseRequestStateExclusiveDemoting == leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveReleasing: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - case inodeLeaseStateExclusiveExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unexpected inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeDemote, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) - } - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] returned !ok - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case LeaseRequestTypeRelease: - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] - if ok { - switch inodeLease.leaseState { - case inodeLeaseStateNone: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - case inodeLeaseStateSharedGrantedRecently: - if leaseRequestStateSharedGranted == leaseRequest.requestState { - _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - if 0 == inodeLease.sharedHoldersList.Len() { - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - if nil == inodeLease.promotingHolder { - leaseRequestElement = inodeLease.requestedList.Front() - if nil == leaseRequestElement { - inodeLease.leaseState = inodeLeaseStateNone - } else { // nil != leaseRequestElement - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequestElement) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - if 0 == inodeLease.requestedList.Len() { - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - leaseRequest.replyChan <- leaseReply - } else { // 0 < inodeLease.requestedList.Len() - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState { - leaseRequestElement = nil - } - } - } - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - leaseRequest.replyChan <- leaseReply - } - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } - } else { // nil != inodeLease.promotingHolder - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = inodeLease.promotingHolder - inodeLease.promotingHolder = nil - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypePromoted, - } - inodeLease.exclusiveHolder.replyChan <- leaseReply - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } - } - } else { // leaseRequestStateSharedGranted != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateSharedGrantedLongAgo: - if leaseRequestStateSharedGranted == leaseRequest.requestState { - _ = inodeLease.sharedHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - if 0 == inodeLease.sharedHoldersList.Len() { - inodeLease.leaseState = inodeLeaseStateNone - } - } else { // leaseRequestStateSharedGranted != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateSharedPromoting: - if leaseRequestStateSharedReleasing == leaseRequest.requestState { - _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - if 0 == inodeLease.releasingHoldersList.Len() { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = inodeLease.promotingHolder - inodeLease.promotingHolder = nil - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypePromoted, - } - inodeLease.exclusiveHolder.replyChan <- leaseReply - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } - } else { // leaseRequestStateSharedReleasing != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateSharedReleasing: - if leaseRequestStateSharedReleasing == leaseRequest.requestState { - _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - if 0 == inodeLease.releasingHoldersList.Len() { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - leaseRequestElement = inodeLease.requestedList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - inodeLease.exclusiveHolder = leaseRequest - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - leaseRequest.replyChan <- leaseReply - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } - } else { // leaseRequestStateSharedReleasing != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateSharedExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState inodeLeaseStateSharedExpired") - case inodeLeaseStateExclusiveGrantedRecently: - if leaseRequestStateExclusiveGranted == leaseRequest.requestState { - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - if nil == leaseRequestElement { - inodeLease.leaseState = inodeLeaseStateNone - inodeLease.exclusiveHolder = nil - inodeLease.lastGrantTime = time.Time{} - inodeLease.longAgoTimer = &time.Timer{} - } else { // nil != leaseRequestElement - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequestElement) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - _ = inodeLease.requestedList.Remove(leaseRequestElement) - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - leaseRequestElement = nil - } - } - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - leaseRequest.requestState = leaseRequestStateExclusiveGranted - inodeLease.exclusiveHolder = leaseRequest - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - leaseRequest.replyChan <- leaseReply - } - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } - } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveGrantedLongAgo: - if leaseRequestStateExclusiveGranted == leaseRequest.requestState { - inodeLease.leaseState = inodeLeaseStateNone - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - inodeLease.exclusiveHolder = nil - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - } else { // leaseRequestStateExclusiveGranted != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveDemoting: - if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - inodeLease.demotingHolder = nil - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - if (nil == inodeLease.requestedList.Front()) || (leaseRequestStateExclusiveRequested == inodeLease.requestedList.Front().Value.(*leaseRequestStruct).requestState) { - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseRequest.listElement = nil - inodeLease.exclusiveHolder = leaseRequest - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - leaseRequest.replyChan <- leaseReply - } else { - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - _ = inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - leaseRequestElement = nil - } - } - } - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } else { // leaseRequestStateExclusiveDemoting != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveReleasing: - if leaseRequestStateExclusiveReleasing == leaseRequest.requestState { - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - inodeLease.interruptTimer = &time.Timer{} - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - _ = inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - leaseRequest.requestState = leaseRequestStateExclusiveGranted - _ = inodeLease.requestedList.Remove(leaseRequestElement) - leaseRequest.listElement = nil - inodeLease.exclusiveHolder = leaseRequest - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - leaseRequest.replyChan <- leaseReply - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - } else { // leaseRequestStateExclusiveReleasing != leaseRequest.requestState - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveExpired: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState inodeLeaseStateExclusiveExpired") - default: - logger.Fatalf("(*inodeLeaseStruct).handleOperation(), while in leaseRequestOperation.LeaseRequestType LeaseRequestTypeRelease, found unknown inodeLease.leaseState: %v", inodeLease.leaseState) - } - } else { // leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] returned !ok - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - default: - logger.Fatalf("(*inodeLeaseStruct).handleOperation() found unexpected leaseRequestOperation.LeaseRequestType: %v", leaseRequestOperation.LeaseRequestType) - } -} - -func (inodeLease *inodeLeaseStruct) handleLongAgoTimerPop() { - var ( - err error - leaseRequest *leaseRequestStruct - leaseRequestElement *list.Element - rpcInterrupt *RPCInterrupt - rpcInterruptBuf []byte - ) - - globals.volumesLock.Lock() - - inodeLease.lastGrantTime = time.Time{} - inodeLease.longAgoTimer = &time.Timer{} - - switch inodeLease.leaseState { - case inodeLeaseStateSharedGrantedRecently: - inodeLease.leaseState = inodeLeaseStateSharedGrantedLongAgo - - if (nil != inodeLease.promotingHolder) || (0 != inodeLease.requestedList.Len()) { - if nil != inodeLease.promotingHolder { - inodeLease.leaseState = inodeLeaseStateSharedPromoting - inodeLease.promotingHolder.requestState = leaseRequestStateSharedPromoting - } else { - inodeLease.leaseState = inodeLeaseStateSharedReleasing - } - - leaseRequestElement = inodeLease.sharedHoldersList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - leaseRequest.requestState = leaseRequestStateSharedReleasing - _ = inodeLease.sharedHoldersList.Remove(leaseRequestElement) - leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - leaseRequestElement = inodeLease.sharedHoldersList.Front() - } - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - } - case inodeLeaseStateExclusiveGrantedRecently: - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedLongAgo - - leaseRequestElement = inodeLease.requestedList.Front() - if nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - switch leaseRequest.requestState { - case leaseRequestStateSharedRequested: - inodeLease.leaseState = inodeLeaseStateExclusiveDemoting - - inodeLease.demotingHolder = inodeLease.exclusiveHolder - inodeLease.demotingHolder.requestState = leaseRequestStateExclusiveDemoting - inodeLease.exclusiveHolder = nil - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeDemote, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.demotingHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(inodeLease.demotingHolder.mount.retryRpcUniqueID, rpcInterruptBuf) - case leaseRequestStateExclusiveRequested: - inodeLease.leaseState = inodeLeaseStateExclusiveReleasing - - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveReleasing - inodeLease.exclusiveHolder.listElement = inodeLease.releasingHoldersList.PushBack(inodeLease.exclusiveHolder) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.exclusiveHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(inodeLease.exclusiveHolder.mount.retryRpcUniqueID, rpcInterruptBuf) - - inodeLease.exclusiveHolder = nil - default: - logger.Fatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() found requestedList with unexpected leaseRequest.requestState: %v", leaseRequest.requestState) - } - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - } - default: - logger.Fatalf("(*inodeLeaseStruct).handleLongAgoTimerPop() called while in wrong state (%v)", inodeLease.leaseState) - } - - globals.volumesLock.Unlock() -} - -func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { - var ( - err error - leaseReply *LeaseReply - leaseRequest *leaseRequestStruct - leaseRequestElement *list.Element - rpcInterrupt *RPCInterrupt - rpcInterruptBuf []byte - ) - - globals.volumesLock.Lock() - - if globals.leaseInterruptLimit <= inodeLease.interruptsSent { - switch inodeLease.leaseState { - case inodeLeaseStateSharedPromoting: - inodeLease.leaseState = inodeLeaseStateSharedExpired - - leaseRequestElement = inodeLease.releasingHoldersList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 1]") - } - - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - - if leaseRequest.mount.acceptingLeaseRequests { - leaseRequest.mount.acceptingLeaseRequests = false - _ = leaseRequest.mount.volume.delayedUnmountList.PushBack(leaseRequest.mount) - } - - inodeLease.releasingHoldersList.Remove(leaseRequestElement) - leaseRequest.listElement = nil - - leaseRequestElement = inodeLease.releasingHoldersList.Front() - } - - inodeLease.exclusiveHolder = inodeLease.promotingHolder - inodeLease.promotingHolder = nil - - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - - inodeLease.exclusiveHolder.replyChan <- leaseReply - - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - case inodeLeaseStateSharedReleasing: - inodeLease.leaseState = inodeLeaseStateSharedExpired - - leaseRequestElement = inodeLease.releasingHoldersList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 2]") - } - - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - - if leaseRequest.mount.acceptingLeaseRequests { - leaseRequest.mount.acceptingLeaseRequests = false - _ = leaseRequest.mount.volume.delayedUnmountList.PushBack(leaseRequest.mount) - } - - inodeLease.releasingHoldersList.Remove(leaseRequestElement) - leaseRequest.listElement = nil - - leaseRequestElement = inodeLease.releasingHoldersList.Front() - } - - leaseRequestElement = inodeLease.requestedList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty requestedList [case 1]") - } - - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateExclusiveRequested != leaseRequest.requestState { - logger.Fatalf("(inodeLeaseStruct).handleInterruptTimerPop() found unexpected requestedList.Front().requestState: %v [case 1]", leaseRequest.requestState) - } - - inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - inodeLease.exclusiveHolder = leaseRequest - - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - - inodeLease.exclusiveHolder.replyChan <- leaseReply - - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - case inodeLeaseStateExclusiveReleasing: - inodeLease.leaseState = inodeLeaseStateExclusiveExpired - - leaseRequestElement = inodeLease.releasingHoldersList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 3]") - } - - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - - if leaseRequest.mount.acceptingLeaseRequests { - leaseRequest.mount.acceptingLeaseRequests = false - _ = leaseRequest.mount.volume.delayedUnmountList.PushBack(leaseRequest.mount) - } - - inodeLease.releasingHoldersList.Remove(leaseRequestElement) - leaseRequest.listElement = nil - - if nil != inodeLease.releasingHoldersList.Front() { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found releasingHoldersList unexpectedly with >1 leaseRequestElements") - } - - leaseRequestElement = inodeLease.requestedList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty requestedList [case 2]") - } - - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateExclusiveRequested != leaseRequest.requestState { - logger.Fatalf("(inodeLeaseStruct).handleInterruptTimerPop() found unexpected requestedList.Front().requestState: %v [case 2]", leaseRequest.requestState) - } - - inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - inodeLease.exclusiveHolder = leaseRequest - - inodeLease.exclusiveHolder.requestState = leaseRequestStateExclusiveGranted - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - - inodeLease.exclusiveHolder.replyChan <- leaseReply - - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - case inodeLeaseStateExclusiveDemoting: - inodeLease.leaseState = inodeLeaseStateExclusiveExpired - - if nil == inodeLease.demotingHolder { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty demotingHolder [case 1]") - } - - delete(inodeLease.demotingHolder.mount.leaseRequestMap, inodeLease.InodeNumber) - - if inodeLease.demotingHolder.mount.acceptingLeaseRequests { - inodeLease.demotingHolder.mount.acceptingLeaseRequests = false - _ = inodeLease.demotingHolder.mount.volume.delayedUnmountList.PushBack(inodeLease.demotingHolder.mount) - } - - inodeLease.demotingHolder = nil - - leaseRequestElement = inodeLease.requestedList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty requestedList [case 3]") - } - - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested != leaseRequest.requestState { - logger.Fatalf("(inodeLeaseStruct).handleInterruptTimerPop() found unexpected requestedList.Front().requestState: %v [case 3]", leaseRequest.requestState) - } - - for { - inodeLease.requestedList.Remove(leaseRequestElement) - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - - leaseRequest.requestState = leaseRequestStateSharedGranted - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - - leaseRequest.replyChan <- leaseReply - - leaseRequestElement = inodeLease.requestedList.Front() - if nil == leaseRequestElement { - break - } - - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateExclusiveRequested == leaseRequest.requestState { - break - } - } - - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - default: - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found unexpected leaseState: %v [case 1]", inodeLease.leaseState) - } - - inodeLease.lastGrantTime = time.Now() - inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) - - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - - inodeLease.interruptTimer = &time.Timer{} - } else { // globals.leaseInterruptLimit > inodeLease.interruptsSent - switch inodeLease.leaseState { - case inodeLeaseStateSharedPromoting: - leaseRequestElement = inodeLease.releasingHoldersList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 4]") - } - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - leaseRequestElement = leaseRequestElement.Next() - } - case inodeLeaseStateSharedReleasing: - leaseRequestElement = inodeLease.releasingHoldersList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 5]") - } - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - leaseRequestElement = leaseRequestElement.Next() - } - case inodeLeaseStateExclusiveReleasing: - leaseRequestElement = inodeLease.releasingHoldersList.Front() - if nil == leaseRequestElement { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty releasingHoldersList [case 6]") - } - - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - case inodeLeaseStateExclusiveDemoting: - if nil == inodeLease.demotingHolder { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty demotingHolder [case 2]") - } - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeDemote, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.demotingHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(inodeLease.demotingHolder.mount.retryRpcUniqueID, rpcInterruptBuf) - default: - logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found unexpected leaseState: %v [case 2]", inodeLease.leaseState) - } - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent++ - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - } - - globals.volumesLock.Unlock() -} - -func (inodeLease *inodeLeaseStruct) handleStopChanClose() { - var ( - err error - leaseReply *LeaseReply - leaseRequest *leaseRequestStruct - leaseRequestElement *list.Element - leaseRequestOperation *leaseRequestOperationStruct - ok bool - rpcInterrupt *RPCInterrupt - rpcInterruptBuf []byte - ) - - // Deny all pending requests: - - globals.volumesLock.Lock() - - for nil != inodeLease.requestedList.Front() { - leaseRequestElement = inodeLease.requestedList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - inodeLease.requestedList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequest.replyChan <- leaseReply - } - - // If inodeLease.leaseState is inodeLeaseStateSharedPromoting: - // Reject inodeLease.promotingHolder's LeaseRequestTypePromote - // Ensure formerly inodeLease.promotingHolder is also now releasing - - if inodeLeaseStateSharedPromoting == inodeLease.leaseState { - leaseRequest = inodeLease.promotingHolder - inodeLease.promotingHolder = nil - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - - leaseRequest.replyChan <- leaseReply - - leaseRequest.requestState = leaseRequestStateSharedReleasing - - leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - inodeLease.leaseState = inodeLeaseStateSharedReleasing - } - - // Ensure that inodeLease.leaseState is not inodeLeaseState{Shared|Exclusive}GrantedRecently - - switch inodeLease.leaseState { - case inodeLeaseStateSharedGrantedRecently: - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - inodeLease.lastGrantTime = time.Time{} - inodeLease.longAgoTimer = &time.Timer{} - - inodeLease.leaseState = inodeLeaseStateSharedGrantedLongAgo - case inodeLeaseStateExclusiveGrantedRecently: - if !inodeLease.longAgoTimer.Stop() { - <-inodeLease.longAgoTimer.C - } - inodeLease.lastGrantTime = time.Time{} - inodeLease.longAgoTimer = &time.Timer{} - - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedLongAgo - default: - // Nothing to do here - } - - // If necessary, transition inodeLease.leaseState from inodeLeaseState{Shared|Exclusive}GrantedLongAgo - // to inodeLeaseState{Shared|Exclusive}Releasing - - switch inodeLease.leaseState { - case inodeLeaseStateSharedGrantedLongAgo: - for nil != inodeLease.sharedHoldersList.Front() { - leaseRequestElement = inodeLease.sharedHoldersList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - inodeLease.sharedHoldersList.Remove(leaseRequestElement) - - leaseRequest.requestState = leaseRequestStateSharedReleasing - - leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 2]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - } - - inodeLease.leaseState = inodeLeaseStateSharedReleasing - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - case inodeLeaseStateExclusiveGrantedLongAgo: - leaseRequest = inodeLease.exclusiveHolder - inodeLease.exclusiveHolder = nil - - leaseRequest.requestState = leaseRequestStateExclusiveReleasing - - leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - inodeLease.leaseState = inodeLeaseStateExclusiveReleasing - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - default: - // Nothing to do here - } - - // Loop until inodeLease.leaseState is inodeLeaseStateNone - - for inodeLeaseStateNone != inodeLease.leaseState { - globals.volumesLock.Unlock() - - select { - case leaseRequestOperation = <-inodeLease.requestChan: - globals.volumesLock.Lock() - - switch leaseRequestOperation.LeaseRequestType { - case LeaseRequestTypeShared: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - - case LeaseRequestTypePromote: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - - case LeaseRequestTypeExclusive: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - - case LeaseRequestTypeDemote: - if inodeLeaseStateExclusiveDemoting == inodeLease.leaseState { - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[leaseRequestOperation.inodeLease.InodeNumber] - if ok { - if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { - if leaseRequest == inodeLease.demotingHolder { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDemoted, - } - leaseRequestOperation.replyChan <- leaseReply - - inodeLease.demotingHolder = nil - - leaseRequest.requestState = leaseRequestStateExclusiveReleasing - - leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - inodeLease.leaseState = inodeLeaseStateExclusiveReleasing - - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent = 1 - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - } else { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - } else { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - } else { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - } else { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - - case LeaseRequestTypeRelease: - leaseRequest, ok = leaseRequestOperation.mount.leaseRequestMap[leaseRequestOperation.inodeLease.InodeNumber] - if ok { - switch inodeLease.leaseState { - case inodeLeaseStateSharedReleasing: - if leaseRequestStateSharedReleasing == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - - if 0 == inodeLease.releasingHoldersList.Len() { - inodeLease.leaseState = inodeLeaseStateNone - - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - - inodeLease.interruptTimer = &time.Timer{} - } - } else { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveDemoting: - if leaseRequestStateExclusiveDemoting == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - inodeLease.demotingHolder = nil - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - - inodeLease.leaseState = inodeLeaseStateNone - - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - - inodeLease.interruptTimer = &time.Timer{} - } else { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - case inodeLeaseStateExclusiveReleasing: - if leaseRequestStateExclusiveReleasing == leaseRequest.requestState { - leaseRequest.requestState = leaseRequestStateNone - delete(leaseRequest.mount.leaseRequestMap, inodeLease.InodeNumber) - inodeLease.releasingHoldersList.Remove(leaseRequest.listElement) - leaseRequest.listElement = nil - - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeReleased, - } - leaseRequestOperation.replyChan <- leaseReply - - inodeLease.leaseState = inodeLeaseStateNone - - if !inodeLease.interruptTimer.Stop() { - <-inodeLease.interruptTimer.C - } - - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - - inodeLease.interruptTimer = &time.Timer{} - } else { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - default: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - } else { - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - } - - default: - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() read unexected leaseRequestOperationLeaseRequestType: %v", leaseRequestOperation.LeaseRequestType) - } - - case _ = <-inodeLease.interruptTimer.C: - globals.volumesLock.Lock() - - switch inodeLease.leaseState { - case inodeLeaseStateSharedGrantedLongAgo: - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() hit an interruptTimer pop while unexpectedly in inodeLeaseStateSharedGrantedLongAgo") - case inodeLeaseStateSharedReleasing: - if globals.leaseInterruptLimit <= inodeLease.interruptsSent { - inodeLease.leaseState = inodeLeaseStateSharedExpired - - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - - inodeLease.interruptTimer = &time.Timer{} - - for nil != inodeLease.releasingHoldersList.Front() { - leaseRequestElement = inodeLease.releasingHoldersList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - inodeLease.releasingHoldersList.Remove(leaseRequestElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - } - - inodeLease.leaseState = inodeLeaseStateNone - } else { // globals.leaseInterruptLimit > inodeLease.interruptsSent { - leaseRequestElement = inodeLease.releasingHoldersList.Front() - - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 5]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - leaseRequestElement = leaseRequestElement.Next() - } - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent++ - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - } - case inodeLeaseStateExclusiveGrantedLongAgo: - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() hit an interruptTimer pop while unexpectedly in inodeLeaseStateExclusiveGrantedLongAgo") - case inodeLeaseStateExclusiveDemoting: - if globals.leaseInterruptLimit <= inodeLease.interruptsSent { - inodeLease.leaseState = inodeLeaseStateExclusiveExpired - - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - - inodeLease.interruptTimer = &time.Timer{} - - leaseRequest = inodeLease.demotingHolder - inodeLease.demotingHolder = nil - leaseRequest.requestState = leaseRequestStateNone - - inodeLease.leaseState = inodeLeaseStateNone - } else { // globals.leaseInterruptLimit > inodeLease.interruptsSent { - leaseRequest = inodeLease.demotingHolder - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 6]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent++ - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - } - case inodeLeaseStateExclusiveReleasing: - if globals.leaseInterruptLimit <= inodeLease.interruptsSent { - inodeLease.leaseState = inodeLeaseStateExclusiveExpired - - inodeLease.lastInterruptTime = time.Time{} - inodeLease.interruptsSent = 0 - - inodeLease.interruptTimer = &time.Timer{} - - leaseRequestElement = inodeLease.releasingHoldersList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - inodeLease.releasingHoldersList.Remove(leaseRequestElement) - leaseRequest.listElement = nil - leaseRequest.requestState = leaseRequestStateNone - - inodeLease.leaseState = inodeLeaseStateNone - } else { // globals.leaseInterruptLimit > inodeLease.interruptsSent { - leaseRequestElement = inodeLease.releasingHoldersList.Front() - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - - rpcInterrupt = &RPCInterrupt{ - RPCInterruptType: RPCInterruptTypeRelease, - InodeNumber: int64(inodeLease.InodeNumber), - } - - rpcInterruptBuf, err = json.Marshal(rpcInterrupt) - if nil != err { - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() unable to json.Marshal(rpcInterrupt: %#v): %v [case 7]", rpcInterrupt, err) - } - - evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - - globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) - - inodeLease.lastInterruptTime = time.Now() - inodeLease.interruptsSent++ - - inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) - } - default: - logger.Fatalf("(*inodeLeaseStruct).handleStopChanClose() hit an interruptTimer pop while unexpectedly in unknown inodeLease.leaseState: %v", inodeLease.leaseState) - } - } - } - - // Drain requestChan before exiting - - for { - select { - case leaseRequestOperation = <-inodeLease.requestChan: - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeDenied, - } - leaseRequestOperation.replyChan <- leaseReply - default: - goto RequestChanDrained - } - } - -RequestChanDrained: - - delete(inodeLease.volume.inodeLeaseMap, inodeLease.InodeNumber) - _ = inodeLease.volume.inodeLeaseLRU.Remove(inodeLease.lruElement) - - if inodeLease.beingEvicted { - inodeLease.volume.ongoingLeaseEvictions-- - } - - inodeLease.volume.leaseHandlerWG.Done() - - globals.volumesLock.Unlock() - - runtime.Goexit() -} - -// armReleaseOfAllLeasesWhileLocked is called to schedule releasing of all held leases -// for a specific mountStruct. It is called while globals.volumesLock is locked. The -// leaseReleaseStartWG is assumed to be a sync.WaitGroup with a count of 1 such that -// the actual releasing of each lease will occur once leaseReleaseStartWG is signaled -// by calling Done() once. The caller would presumably do this after having released -// globals.volumesLock and then await completion by calling leaseReleaseFinishedWG.Wait(). -// -func (mount *mountStruct) armReleaseOfAllLeasesWhileLocked(leaseReleaseStartWG *sync.WaitGroup, leaseReleaseFinishedWG *sync.WaitGroup) { - var ( - leaseRequest *leaseRequestStruct - leaseRequestOperation *leaseRequestOperationStruct - ) - - for _, leaseRequest = range mount.leaseRequestMap { - if nil != leaseRequest.inodeLease { - leaseRequestOperation = &leaseRequestOperationStruct{ - mount: mount, - inodeLease: leaseRequest.inodeLease, - LeaseRequestType: LeaseRequestTypeRelease, - replyChan: make(chan *LeaseReply), - } - - leaseReleaseFinishedWG.Add(1) - - go func(leaseRequestOperation *leaseRequestOperationStruct) { - leaseReleaseStartWG.Wait() - leaseRequestOperation.inodeLease.requestChan <- leaseRequestOperation - _ = <-leaseRequestOperation.replyChan - leaseReleaseFinishedWG.Done() - }(leaseRequestOperation) - } - } -} - -func fetchLeaseReport(volumeName string) (leaseReport []*LeaseReportElementStruct, err error) { - var ( - annotatedMountID string - inodeLease *inodeLeaseStruct - leaseReportElement *LeaseReportElementStruct - leaseReportLLRBTree sortedmap.LLRBTree - leaseReportLLRBTreeElement sortedmap.Value - leaseReportLLRBTreeIndex int - leaseReportLLRBTreeLen int - leaseRequest *leaseRequestStruct - leaseRequestListElement *list.Element - ok bool - volume *volumeStruct - ) - - enterGate() - defer leaveGate() - - globals.volumesLock.Lock() - defer globals.volumesLock.Unlock() - - volume, ok = globals.volumeMap[volumeName] - if !ok { - err = fmt.Errorf("fetchLeaseReport(volumeName: \"%s\") could not find Volume", volumeName) - return - } - - // leaseReport = make([]*LeaseReportElementStruct, 0) - - leaseReportLLRBTree = sortedmap.NewLLRBTree(sortedmap.CompareUint64, &globals) - - for _, inodeLease = range volume.inodeLeaseMap { - leaseReportElement = &LeaseReportElementStruct{ - InodeNumber: fmt.Sprintf("%016X", inodeLease.InodeNumber), - SharedHoldersList: make([]MountIDAsString, 0), - PromotingHolder: MountIDAsString(""), - ExclusiveHolder: MountIDAsString(""), - DemotingHolder: MountIDAsString(""), - ReleasingHoldersList: make([]MountIDAsString, 0), - RequestedList: make([]string, 0), - } - - for leaseRequestListElement = inodeLease.sharedHoldersList.Front(); leaseRequestListElement != nil; leaseRequestListElement = leaseRequestListElement.Next() { - leaseRequest = leaseRequestListElement.Value.(*leaseRequestStruct) - leaseReportElement.SharedHoldersList = append(leaseReportElement.SharedHoldersList, leaseRequest.mount.mountIDAsString) - } - - if nil != inodeLease.promotingHolder { - leaseReportElement.PromotingHolder = inodeLease.promotingHolder.mount.mountIDAsString - } - - if nil != inodeLease.exclusiveHolder { - leaseReportElement.ExclusiveHolder = inodeLease.exclusiveHolder.mount.mountIDAsString - } - - if nil != inodeLease.demotingHolder { - leaseReportElement.DemotingHolder = inodeLease.demotingHolder.mount.mountIDAsString - } - - for leaseRequestListElement = inodeLease.releasingHoldersList.Front(); leaseRequestListElement != nil; leaseRequestListElement = leaseRequestListElement.Next() { - leaseRequest = leaseRequestListElement.Value.(*leaseRequestStruct) - leaseReportElement.ReleasingHoldersList = append(leaseReportElement.ReleasingHoldersList, leaseRequest.mount.mountIDAsString) - } - - for leaseRequestListElement = inodeLease.requestedList.Front(); leaseRequestListElement != nil; leaseRequestListElement = leaseRequestListElement.Next() { - leaseRequest = leaseRequestListElement.Value.(*leaseRequestStruct) - switch leaseRequest.requestState { - case leaseRequestStateSharedRequested: - annotatedMountID = fmt.Sprintf("S(%s)", leaseRequest.mount.mountIDAsString) - case leaseRequestStateExclusiveRequested: - annotatedMountID = fmt.Sprintf("E(%s)", leaseRequest.mount.mountIDAsString) - default: - logger.Fatalf("leaseRequest.requestState (%v) unexpected", leaseRequest.requestState) - } - leaseReportElement.RequestedList = append(leaseReportElement.RequestedList, annotatedMountID) - } - - ok, err = leaseReportLLRBTree.Put(uint64(inodeLease.InodeNumber), leaseReportElement) - if nil != err { - logger.Fatalf("leaseReportLLRBTree.Put(0x%016X,) failed: %v", uint64(inodeLease.InodeNumber), err) - } - if !ok { - logger.Fatalf("leaseReportLLRBTree.Put(0x%016X,) returned !ok", uint64(inodeLease.InodeNumber)) - } - } - - leaseReportLLRBTreeLen, err = leaseReportLLRBTree.Len() - if nil != err { - logger.Fatalf("leaseReportLLRBTree.Len() failed: %v", err) - } - - leaseReport = make([]*LeaseReportElementStruct, 0, leaseReportLLRBTreeLen) - - for leaseReportLLRBTreeIndex = 0; leaseReportLLRBTreeIndex < leaseReportLLRBTreeLen; leaseReportLLRBTreeIndex++ { - _, leaseReportLLRBTreeElement, ok, err = leaseReportLLRBTree.GetByIndex(leaseReportLLRBTreeIndex) - if nil != err { - logger.Fatalf("leaseReportLLRBTree.GetByIndex() failed: %v", err) - } - if !ok { - logger.Fatalf("leaseReportLLRBTree.GetByIndex() returned !ok") - } - - leaseReportElement = leaseReportLLRBTreeElement.(*LeaseReportElementStruct) - - leaseReport = append(leaseReport, leaseReportElement) - } - - err = nil - return -} - -func (dummy *globalsStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - err = fmt.Errorf("Not implemented") - return -} - -func (dummy *globalsStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - err = fmt.Errorf("Not implemented") - return -} diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go deleted file mode 100644 index faeae246..00000000 --- a/jrpcfs/lease_test.go +++ /dev/null @@ -1,2158 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "bytes" - "crypto/tls" - "crypto/x509" - "encoding/binary" - "encoding/json" - "fmt" - "io" - "net" - "os" - "runtime" - "strings" - "sync" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/retryrpc" -) - -const ( - testRpcLeaseDelayAfterSendingRequest = 10 * time.Millisecond - testRpcLeaseDelayBeforeSendingRequest = 10 * time.Millisecond - testRpcLeaseRetryRPCDeadlineIO = "60s" - testRpcLeaseRetryRPCKeepAlivePeriod = "60s" - testRpcLeaseMultiFirstInodeNumber int64 = 1 - testRpcLeaseMultiNumInstances int = 5 - testRpcLeaseSingleInodeNumber int64 = 1 - testRpcLeaseSingleNumInstances int = 101 // Must be >= 4 - testRpcLeaseTimeFormat = "15:04:05.000" - testRpcLeaseLocalClientID uint64 = 0 - testRpcLeaseShortcutIPAddrPort = "127.0.0.1:24680" - testRpcLeaseShortcutClientConnMaxRetries = 10 - testRpcLeaseShortcutClientConnRetryDelay = "100ms" -) - -var ( - testRpcLeaseRequestLetters = [5]string{"S", "P", "E", "D", "R"} - testRpcLeaseReplyLetters = [6]string{"D", "S", "P", "E", "D", "R"} - testRpcLeaseInterruptLetters = [3]string{"U", "D", "R"} - testRpcLeaseLogVerbosely bool -) - -type testRpcLeaseClientStruct struct { - instance int - inodeNumber int64 - chIn chan LeaseRequestType // close it to terminate testRpcLeaseClient instance - chOut chan interface{} // either a LeaseReplyType or an RPCInterruptType - alreadyUnmounted bool // if true, no RpcUnmount will be issued - wg *sync.WaitGroup // signaled when testRpcLeaseClient instance exits - t *testing.T -} - -type benchmarkRpcServerStats struct { - UnmarshalRequestUsec bucketstats.BucketLog2Round - MarshalReplyUsec bucketstats.BucketLog2Round -} - -type benchmarkRpcShortcutRequestMethodOnlyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form - Method string -} - -type benchmarkRpcShortcutLeaseRequestStruct struct { // preceeded by a uint32 encoding length in LittleEndian form - Method string - Request *LeaseRequest -} - -type benchmarkRpcShortcutMountByAccountNameRequestStruct struct { // preceeded by a uint32 encoding length in LittleEndian form - Method string - Request *MountByAccountNameRequest -} - -type benchmarkRpcShortcutUnmountRequestStruct struct { // preceeded by a uint32 encoding length in LittleEndian form - Method string - Request *UnmountRequest -} - -type benchmarkRpcShortcutLeaseReplyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form - Err error - Reply *LeaseReply -} - -type benchmarkRpcShortcutMountByAccountNameReplyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form - Err error - Reply *MountByAccountNameReply -} - -type benchmarkRpcShortcutUnmountReplyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form - Err error - Reply *Reply -} - -// benchmarkRpcLeaseCustomRequest's are laid out as follows: -// -// RequestLength uint32 - [LittleEndian] does not include RequestLength nor RequestType -// RequestType uint32 - [LittleEndian] one of benchmarkRpcLeaseCustomRequestType* -// Request []byte - optimally packed request of the corresponding type - -// benchmarkRpcLeaseCustomReply's are laid out as follows: -// -// ReplyLenth uint32 - does not include ReplyLength -// Reply []byte - optimally packed RPC error string followed by the reply of the corresponding type - -// In each of the marshaled forms, strings consist of: -// -// StringLength uint32 - [LittleEndian] length of UTF-8 string -// StringBuf []byte - UTF-8 string - -const ( - benchmarkRpcLeaseCustomRequestTypeMountByAccountName uint32 = iota - benchmarkRpcLeaseCustomRequestTypeLease - benchmarkRpcLeaseCustomRequestTypeUnmount -) - -func benchmarkRpcLeaseCustomMountByAccountNameRequestMarshal(mountByAccountNameRequest *MountByAccountNameRequest) (mountByAccountNameRequestBuf []byte) { - var ( - accountNameLen = uint32(len(mountByAccountNameRequest.AccountName)) - authTokenLen = uint32(len(mountByAccountNameRequest.AuthToken)) - bytesBuffer *bytes.Buffer - err error - n int - ) - - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+accountNameLen+4+authTokenLen)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, accountNameLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, accountNameLen) failed: %v", err)) - } - - n, err = bytesBuffer.WriteString(mountByAccountNameRequest.AccountName) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameRequest.AccountName) failed: %v", err)) - } - if uint32(n) != accountNameLen { - panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameRequest.AccountName) returned n==%v (expected %v)", n, accountNameLen)) - } - - err = binary.Write(bytesBuffer, binary.LittleEndian, authTokenLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, authTokenLen) failed: %v", err)) - } - - n, err = bytesBuffer.WriteString(mountByAccountNameRequest.AuthToken) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameRequest.AuthToken) failed: %v", err)) - } - if uint32(n) != authTokenLen { - panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameRequest.AuthToken) returned n==%v (expected %v)", n, authTokenLen)) - } - - mountByAccountNameRequestBuf = bytesBuffer.Bytes() - - return -} - -func benchmarkRpcLeaseCustomMountByAccountNameRequestUnmarshal(mountByAccountNameRequestBuf []byte) (mountByAccountNameRequest *MountByAccountNameRequest, unmarshalErr error) { - var ( - accountNameLen uint32 - authTokenLen uint32 - mountByAccountNameRequestBufLen = uint32(len(mountByAccountNameRequestBuf)) - mountByAccountNameRequestBufPos uint32 = 0 - ) - - mountByAccountNameRequest = &MountByAccountNameRequest{} - - if mountByAccountNameRequestBufPos+4 > mountByAccountNameRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read accountNameLen") - return - } - - accountNameLen = binary.LittleEndian.Uint32(mountByAccountNameRequestBuf[mountByAccountNameRequestBufPos : mountByAccountNameRequestBufPos+4]) - mountByAccountNameRequestBufPos += 4 - - if mountByAccountNameRequestBufPos+accountNameLen > mountByAccountNameRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read AccountName") - return - } - - mountByAccountNameRequest.AccountName = string(mountByAccountNameRequestBuf[mountByAccountNameRequestBufPos : mountByAccountNameRequestBufPos+accountNameLen]) - mountByAccountNameRequestBufPos += accountNameLen - - if mountByAccountNameRequestBufPos+4 > mountByAccountNameRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read authTokenLen") - return - } - - authTokenLen = binary.LittleEndian.Uint32(mountByAccountNameRequestBuf[mountByAccountNameRequestBufPos : mountByAccountNameRequestBufPos+4]) - mountByAccountNameRequestBufPos += 4 - - if mountByAccountNameRequestBufPos+authTokenLen > mountByAccountNameRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read AuthToken") - return - } - - mountByAccountNameRequest.AuthToken = string(mountByAccountNameRequestBuf[mountByAccountNameRequestBufPos : mountByAccountNameRequestBufPos+authTokenLen]) - mountByAccountNameRequestBufPos += authTokenLen - - unmarshalErr = nil - return -} - -func benchmarkRpcLeaseCustomMountByAccountNameReplyMarshal(rpcErr error, mountByAccountNameReply *MountByAccountNameReply) (mountByAccountNameReplyBuf []byte) { - var ( - bytesBuffer *bytes.Buffer - err error - mountIDLen = uint32(len(mountByAccountNameReply.MountID)) - n int - rpcErrString string - rpcErrStringLen uint32 - ) - - if nil == rpcErr { - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+4+mountIDLen)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) failed: %v", err)) - } - } else { - rpcErrString = rpcErr.Error() - rpcErrStringLen = uint32(len(rpcErrString)) - - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+rpcErrStringLen+4+mountIDLen)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) - } - - n, err = bytesBuffer.WriteString(string(rpcErrString)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) - } - if uint32(n) != rpcErrStringLen { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) - } - } - - err = binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) failed: %v", err)) - } - - n, err = bytesBuffer.WriteString(string(mountByAccountNameReply.MountID)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameReply.MountID) failed: %v", err)) - } - if uint32(n) != mountIDLen { - panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameReply.MountID) returned n==%v (expected %v)", n, mountIDLen)) - } - - mountByAccountNameReplyBuf = bytesBuffer.Bytes() - - return -} - -func benchmarkRpcLeaseCustomMountByAccountNameReplyUnmarshal(mountByAccountNameReplyBuf []byte) (rpcErr error, mountByAccountNameReply *MountByAccountNameReply, unmarshalErr error) { - var ( - mountByAccountNameReplyBufLen = uint32(len(mountByAccountNameReplyBuf)) - mountByAccountNameReplyBufPos uint32 = 0 - mountIDLen uint32 - rpcErrAsString string - rpcErrLen uint32 - ) - - if mountByAccountNameReplyBufPos+4 > mountByAccountNameReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read rpcErrLen") - return - } - - rpcErrLen = binary.LittleEndian.Uint32(mountByAccountNameReplyBuf[mountByAccountNameReplyBufPos : mountByAccountNameReplyBufPos+4]) - mountByAccountNameReplyBufPos += 4 - - if 0 == rpcErrLen { - rpcErr = nil - } else { - if mountByAccountNameReplyBufPos+rpcErrLen > mountByAccountNameReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read rpcErr") - return - } - - rpcErrAsString = string(mountByAccountNameReplyBuf[mountByAccountNameReplyBufPos : mountByAccountNameReplyBufPos+rpcErrLen]) - mountByAccountNameReplyBufPos += rpcErrLen - - rpcErr = fmt.Errorf("%s", rpcErrAsString) - } - - mountByAccountNameReply = &MountByAccountNameReply{} - - if mountByAccountNameReplyBufPos+4 > mountByAccountNameReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read mountIDLen") - return - } - - mountIDLen = binary.LittleEndian.Uint32(mountByAccountNameReplyBuf[mountByAccountNameReplyBufPos : mountByAccountNameReplyBufPos+4]) - mountByAccountNameReplyBufPos += 4 - - if mountByAccountNameReplyBufPos+mountIDLen > mountByAccountNameReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read MountID") - return - } - - mountByAccountNameReply.MountID = MountIDAsString(string(mountByAccountNameReplyBuf[mountByAccountNameReplyBufPos : mountByAccountNameReplyBufPos+mountIDLen])) - mountByAccountNameReplyBufPos += mountIDLen - - unmarshalErr = nil - return -} - -func benchmarkRpcLeaseCustomLeaseRequestMarshal(leaseRequest *LeaseRequest) (leaseRequestBuf []byte) { - var ( - bytesBuffer *bytes.Buffer - err error - mountIDLen = uint32(len(leaseRequest.InodeHandle.MountID)) - n int - ) - - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+mountIDLen+8+4)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) failed: %v", err)) - } - - n, err = bytesBuffer.WriteString(string(leaseRequest.InodeHandle.MountID)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(leaseRequest.InodeHandle.MountID) failed: %v", err)) - } - if uint32(n) != mountIDLen { - panic(fmt.Errorf("bytesBuffer.WriteString(leaseRequest.InodeHandle.MountID) returned n==%v (expected %v)", n, mountIDLen)) - } - - err = binary.Write(bytesBuffer, binary.LittleEndian, uint64(leaseRequest.InodeHandle.InodeNumber)) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, uint64(leaseRequest.InodeHandle.InodeNumber)) failed: %v", err)) - } - - err = binary.Write(bytesBuffer, binary.LittleEndian, leaseRequest.LeaseRequestType) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, leaseRequest.LeaseRequestType) failed: %v", err)) - } - - leaseRequestBuf = bytesBuffer.Bytes() - - return -} - -func benchmarkRpcLeaseCustomLeaseRequestUnmarshal(leaseRequestBuf []byte) (leaseRequest *LeaseRequest, unmarshalErr error) { - var ( - leaseRequestBufLen = uint32(len(leaseRequestBuf)) - leaseRequestBufPos uint32 = 0 - mountIDLen uint32 - ) - - leaseRequest = &LeaseRequest{} - - if leaseRequestBufPos+4 > leaseRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read mountIDLen") - return - } - - mountIDLen = binary.LittleEndian.Uint32(leaseRequestBuf[leaseRequestBufPos : leaseRequestBufPos+4]) - leaseRequestBufPos += 4 - - if leaseRequestBufPos+mountIDLen > leaseRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read MountID") - return - } - - leaseRequest.InodeHandle.MountID = MountIDAsString(string(leaseRequestBuf[leaseRequestBufPos : leaseRequestBufPos+mountIDLen])) - leaseRequestBufPos += mountIDLen - - if leaseRequestBufPos+8 > leaseRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read InodeNumber") - return - } - - leaseRequest.InodeHandle.InodeNumber = int64(binary.LittleEndian.Uint64(leaseRequestBuf[leaseRequestBufPos : leaseRequestBufPos+8])) - leaseRequestBufPos += 8 - - if leaseRequestBufPos+4 > leaseRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read LeaseRequestType") - return - } - - leaseRequest.LeaseRequestType = LeaseRequestType(binary.LittleEndian.Uint32(leaseRequestBuf[leaseRequestBufPos : leaseRequestBufPos+4])) - leaseRequestBufPos += 4 - - unmarshalErr = nil - return -} - -func benchmarkRpcLeaseCustomLeaseReplyMarshal(rpcErr error, leaseReply *LeaseReply) (leaseReplyBuf []byte) { - var ( - bytesBuffer *bytes.Buffer - err error - n int - rpcErrString string - rpcErrStringLen uint32 - ) - - if nil == rpcErr { - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+4)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) failed: %v", err)) - } - } else { - rpcErrString = rpcErr.Error() - rpcErrStringLen = uint32(len(rpcErrString)) - - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+rpcErrStringLen+4)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) - } - - n, err = bytesBuffer.WriteString(string(rpcErrString)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) - } - if uint32(n) != rpcErrStringLen { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) - } - } - - err = binary.Write(bytesBuffer, binary.LittleEndian, leaseReply.LeaseReplyType) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, leaseReply.LeaseReplyType) failed: %v", err)) - } - - leaseReplyBuf = bytesBuffer.Bytes() - - return -} - -func benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf []byte) (rpcErr error, leaseReply *LeaseReply, unmarshalErr error) { - var ( - leaseReplyBufLen = uint32(len(leaseReplyBuf)) - leaseReplyBufPos uint32 = 0 - rpcErrAsString string - rpcErrLen uint32 - ) - - if leaseReplyBufPos+4 > leaseReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read rpcErrLen") - return - } - - rpcErrLen = binary.LittleEndian.Uint32(leaseReplyBuf[leaseReplyBufPos : leaseReplyBufPos+4]) - leaseReplyBufPos += 4 - - if 0 == rpcErrLen { - rpcErr = nil - } else { - if leaseReplyBufPos+rpcErrLen > leaseReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read rpcErr") - return - } - - rpcErrAsString = string(leaseReplyBuf[leaseReplyBufPos : leaseReplyBufPos+rpcErrLen]) - leaseReplyBufPos += rpcErrLen - - rpcErr = fmt.Errorf("%s", rpcErrAsString) - } - - leaseReply = &LeaseReply{} - - if leaseReplyBufPos+4 > leaseReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read LeaseReplyType") - return - } - - leaseReply.LeaseReplyType = LeaseReplyType(binary.LittleEndian.Uint32(leaseReplyBuf[leaseReplyBufPos : leaseReplyBufPos+4])) - leaseReplyBufPos += 4 - - unmarshalErr = nil - return -} - -func benchmarkRpcLeaseCustomUnmountRequestMarshal(unmountRequest *UnmountRequest) (unmountRequestBuf []byte) { - var ( - bytesBuffer *bytes.Buffer - err error - mountIDLen = uint32(len(unmountRequest.MountID)) - n int - ) - - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+mountIDLen)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) failed: %v", err)) - } - - n, err = bytesBuffer.WriteString(string(unmountRequest.MountID)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(unmountRequest.MountID) failed: %v", err)) - } - if uint32(n) != mountIDLen { - panic(fmt.Errorf("bytesBuffer.WriteString(unmountRequest.MountID) returned n==%v (expected %v)", n, mountIDLen)) - } - - unmountRequestBuf = bytesBuffer.Bytes() - - return -} - -func benchmarkRpcLeaseCustomUnmountRequestUnmarshal(unmountRequestBuf []byte) (unmountRequest *UnmountRequest, unmarshalErr error) { - var ( - unmountRequestBufLen = uint32(len(unmountRequestBuf)) - unmountRequestBufPos uint32 = 0 - mountIDLen uint32 - ) - - unmountRequest = &UnmountRequest{} - - if unmountRequestBufPos+4 > unmountRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read mountIDLen") - return - } - - mountIDLen = binary.LittleEndian.Uint32(unmountRequestBuf[unmountRequestBufPos : unmountRequestBufPos+4]) - unmountRequestBufPos += 4 - - if unmountRequestBufPos+mountIDLen > unmountRequestBufLen { - unmarshalErr = fmt.Errorf("couldn't read MountID") - return - } - - unmountRequest.MountID = MountIDAsString(string(unmountRequestBuf[unmountRequestBufPos : unmountRequestBufPos+mountIDLen])) - unmountRequestBufPos += mountIDLen - - unmarshalErr = nil - return -} - -func benchmarkRpcLeaseCustomUnmountReplyMarshal(rpcErr error, unmountReply *Reply) (unmountReplyBuf []byte) { - var ( - bytesBuffer *bytes.Buffer - err error - n int - rpcErrString string - rpcErrStringLen uint32 - ) - - if nil == rpcErr { - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) failed: %v", err)) - } - } else { - rpcErrString = rpcErr.Error() - rpcErrStringLen = uint32(len(rpcErrString)) - - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+rpcErrStringLen)) - - err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) - } - - n, err = bytesBuffer.WriteString(string(rpcErrString)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) - } - if uint32(n) != rpcErrStringLen { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) - } - } - - unmountReplyBuf = bytesBuffer.Bytes() - - return -} - -func benchmarkRpcLeaseCustomUnmountReplyUnmarshal(unmountReplyBuf []byte) (rpcErr error, unmountReply *Reply, unmarshalErr error) { - var ( - unmountReplyBufLen = uint32(len(unmountReplyBuf)) - unmountReplyBufPos uint32 = 0 - rpcErrAsString string - rpcErrLen uint32 - ) - - if unmountReplyBufPos+4 > unmountReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read rpcErrLen") - return - } - - rpcErrLen = binary.LittleEndian.Uint32(unmountReplyBuf[unmountReplyBufPos : unmountReplyBufPos+4]) - unmountReplyBufPos += 4 - - if 0 == rpcErrLen { - rpcErr = nil - } else { - if unmountReplyBufPos+rpcErrLen > unmountReplyBufLen { - unmarshalErr = fmt.Errorf("couldn't read rpcErr") - return - } - - rpcErrAsString = string(unmountReplyBuf[unmountReplyBufPos : unmountReplyBufPos+rpcErrLen]) - unmountReplyBufPos += rpcErrLen - - rpcErr = fmt.Errorf("%s", rpcErrAsString) - } - - unmountReply = &Reply{} - - unmarshalErr = nil - return -} - -func BenchmarkRpcLeaseCustomTCP(b *testing.B) { - benchmarkRpcLeaseCustom(b, false) -} - -func BenchmarkRpcLeaseCustomTLS(b *testing.B) { - benchmarkRpcLeaseCustom(b, true) -} - -func benchmarkRpcLeaseCustom(b *testing.B, useTLS bool) { - var ( - benchmarkIteration int - doneWG sync.WaitGroup - err error - leaseReply *LeaseReply - leaseReplyBuf []byte - leaseRequest *LeaseRequest - leaseRequestBuf []byte - mountByAccountNameReply *MountByAccountNameReply - mountByAccountNameReplyBuf []byte - mountByAccountNameRequest *MountByAccountNameRequest - mountByAccountNameRequestBuf []byte - netConn net.Conn - netConnRetries int - netConnRetryDelay time.Duration - ok bool - rootCACertPool *x509.CertPool - rpcErr error - tlsConfig *tls.Config - tlsConn *tls.Conn - unmarshalErr error - unmountReplyBuf []byte - unmountRequest *UnmountRequest - unmountRequestBuf []byte - ) - - doneWG.Add(1) - go benchmarkRpcLeaseCustomServer(useTLS, &doneWG) - - netConnRetries = 0 - - netConnRetryDelay, err = time.ParseDuration(testRpcLeaseShortcutClientConnRetryDelay) - if nil != err { - b.Fatalf("time.ParseDuration(testRpcLeaseShortcutClientConnRetryDelay=\"%s\") failed: %v", testRpcLeaseShortcutClientConnRetryDelay, err) - } - - if useTLS { - rootCACertPool = x509.NewCertPool() - ok = rootCACertPool.AppendCertsFromPEM(testTLSCerts.caCertPEMBlock) - if !ok { - b.Fatalf("rootCACertPool.AppendCertsFromPEM(testTLSCerts.caCertPEMBlock) returned !ok") - } - tlsConfig = &tls.Config{ - RootCAs: rootCACertPool, - } - } - - for { - if useTLS { - tlsConn, err = tls.Dial("tcp", testRpcLeaseShortcutIPAddrPort, tlsConfig) - if nil == err { - netConn = tlsConn - break - } - } else { - netConn, err = net.Dial("tcp", testRpcLeaseShortcutIPAddrPort) - if nil == err { - break - } - } - - netConnRetries++ - - if netConnRetries > testRpcLeaseShortcutClientConnMaxRetries { - b.Fatalf("netConnRetries exceeded testRpcLeaseShortcutClientConnMaxRetries (%v)", testRpcLeaseShortcutClientConnMaxRetries) - } - - time.Sleep(netConnRetryDelay) - } - - mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - AuthToken: "", - } - - mountByAccountNameRequestBuf = benchmarkRpcLeaseCustomMountByAccountNameRequestMarshal(mountByAccountNameRequest) - - mountByAccountNameReplyBuf = benchmarkRpcLeaseCustomDoRequest(b, netConn, benchmarkRpcLeaseCustomRequestTypeMountByAccountName, mountByAccountNameRequestBuf) - - rpcErr, mountByAccountNameReply, unmarshalErr = benchmarkRpcLeaseCustomMountByAccountNameReplyUnmarshal(mountByAccountNameReplyBuf) - if nil != unmarshalErr { - b.Fatalf("benchmarkRpcLeaseCustomMountByAccountNameReplyUnmarshal(mountByAccountNameReplyBuf) returned unmarshalErr: %v", unmarshalErr) - } - if nil != rpcErr { - b.Fatalf("benchmarkRpcLeaseCustomMountByAccountNameReplyUnmarshal(mountByAccountNameReplyBuf) returned rpcErr: %v", rpcErr) - } - - b.ResetTimer() - - for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeExclusive, - } - - leaseRequestBuf = benchmarkRpcLeaseCustomLeaseRequestMarshal(leaseRequest) - - leaseReplyBuf = benchmarkRpcLeaseCustomDoRequest(b, netConn, benchmarkRpcLeaseCustomRequestTypeLease, leaseRequestBuf) - - rpcErr, leaseReply, unmarshalErr = benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) - if nil != unmarshalErr { - b.Fatalf("benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) returned unmarshalErr: %v", unmarshalErr) - } - if nil != rpcErr { - b.Fatalf("benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) returned rpcErr: %v", rpcErr) - } - - if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReply.LeaseReplyType) - } - - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeRelease, - } - - leaseRequestBuf = benchmarkRpcLeaseCustomLeaseRequestMarshal(leaseRequest) - - leaseReplyBuf = benchmarkRpcLeaseCustomDoRequest(b, netConn, benchmarkRpcLeaseCustomRequestTypeLease, leaseRequestBuf) - - rpcErr, leaseReply, unmarshalErr = benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) - if nil != unmarshalErr { - b.Fatalf("benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) returned unmarshalErr: %v", unmarshalErr) - } - if nil != rpcErr { - b.Fatalf("benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) returned rpcErr: %v", rpcErr) - } - - if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReply.LeaseReplyType) - } - } - - b.StopTimer() - - unmountRequest = &UnmountRequest{ - MountID: mountByAccountNameReply.MountID, - } - - unmountRequestBuf = benchmarkRpcLeaseCustomUnmountRequestMarshal(unmountRequest) - - unmountReplyBuf = benchmarkRpcLeaseCustomDoRequest(b, netConn, benchmarkRpcLeaseCustomRequestTypeUnmount, unmountRequestBuf) - - rpcErr, _, unmarshalErr = benchmarkRpcLeaseCustomUnmountReplyUnmarshal(unmountReplyBuf) - if nil != unmarshalErr { - b.Fatalf("benchmarkRpcLeaseCustomUnmountReplyUnmarshal(unmountReplyBuf) returned unmarshalErr: %v", unmarshalErr) - } - if nil != rpcErr { - b.Fatalf("benchmarkRpcLeaseCustomUnmountReplyUnmarshal(unmountReplyBuf) returned rpcErr: %v", rpcErr) - } - - err = netConn.Close() - if nil != err { - b.Fatalf("netConn.Close() failed: %v", err) - } - - doneWG.Wait() -} - -func benchmarkRpcLeaseCustomServer(useTLS bool, doneWG *sync.WaitGroup) { - var ( - err error - jserver *Server - leaseReply *LeaseReply - leaseRequest *LeaseRequest - mountByAccountNameReply *MountByAccountNameReply - mountByAccountNameRequest *MountByAccountNameRequest - n int - netConn net.Conn - netListener net.Listener - replyBuf []byte - replyLen uint32 - replyLenBuf []byte - requestBuf []byte - requestLen uint32 - requestLenAndRequestTypeBuf []byte - requestType uint32 - tlsConfig *tls.Config - unmountReply *Reply - unmountRequest *UnmountRequest - ) - - jserver = NewServer() - - netListener, err = net.Listen("tcp", testRpcLeaseShortcutIPAddrPort) - if nil != err { - panic(fmt.Errorf("net.Listen(\"tcp\", testRpcLeaseShortcutIPAddrPort) failed: %v", err)) - } - - if useTLS { - tlsConfig = &tls.Config{ - Certificates: []tls.Certificate{testTLSCerts.endpointTLSCert}, - } - - netListener = tls.NewListener(netListener, tlsConfig) - } - - netConn, err = netListener.Accept() - if nil != err { - panic(fmt.Errorf("netListener.Accept() failed: %v", err)) - } - - for { - requestLenAndRequestTypeBuf = make([]byte, 8) - - n, err = netConn.Read(requestLenAndRequestTypeBuf) - if nil != err { - if io.EOF != err { - panic(fmt.Errorf("netConn.Read(requestLenAndRequestTypeBuf) failed: %v", err)) - } - - err = netConn.Close() - if nil != err { - panic(fmt.Errorf("netConn.Close() failed: %v", err)) - } - - err = netListener.Close() - if nil != err { - panic(fmt.Errorf("netListener.Close() failed: %v", err)) - } - - doneWG.Done() - - return - } - if n != 8 { - panic(fmt.Errorf("netConn.Read(requestLenAndRequestTypeBuf) returned n == %v (expected 8)", n)) - } - - requestLen = binary.LittleEndian.Uint32(requestLenAndRequestTypeBuf[:4]) - requestType = binary.LittleEndian.Uint32(requestLenAndRequestTypeBuf[4:]) - - requestBuf = make([]byte, requestLen) - - n, err = netConn.Read(requestBuf) - if nil != err { - panic(fmt.Errorf("netConn.Read(requestBuf) failed: %v", err)) - } - if n != int(requestLen) { - panic(fmt.Errorf("netConn.Read(requestBuf) returned n == %v (expected %v)", n, requestLen)) - } - - switch requestType { - case benchmarkRpcLeaseCustomRequestTypeMountByAccountName: - mountByAccountNameRequest, err = benchmarkRpcLeaseCustomMountByAccountNameRequestUnmarshal(requestBuf) - if nil != err { - panic(fmt.Errorf("benchmarkRpcLeaseCustomMountByAccountNameRequestUnmarshal(requestBuf) failed: %v", err)) - } - - mountByAccountNameReply = &MountByAccountNameReply{} - - err = jserver.RpcMountByAccountName(testRpcLeaseLocalClientID, mountByAccountNameRequest, mountByAccountNameReply) - - replyBuf = benchmarkRpcLeaseCustomMountByAccountNameReplyMarshal(err, mountByAccountNameReply) - case benchmarkRpcLeaseCustomRequestTypeLease: - leaseRequest, err = benchmarkRpcLeaseCustomLeaseRequestUnmarshal(requestBuf) - if nil != err { - panic(fmt.Errorf("benchmarkRpcLeaseCustomLeaseRequestUnmarshal(requestBuf) failed: %v", err)) - } - - leaseReply = &LeaseReply{} - - err = jserver.RpcLease(leaseRequest, leaseReply) - - replyBuf = benchmarkRpcLeaseCustomLeaseReplyMarshal(err, leaseReply) - case benchmarkRpcLeaseCustomRequestTypeUnmount: - unmountRequest, err = benchmarkRpcLeaseCustomUnmountRequestUnmarshal(requestBuf) - if nil != err { - panic(fmt.Errorf("benchmarkRpcLeaseCustomUnmountRequestUnmarshal(requestBuf) failed: %v", err)) - } - - unmountReply = &Reply{} - - err = jserver.RpcUnmount(unmountRequest, unmountReply) - - replyBuf = benchmarkRpcLeaseCustomUnmountReplyMarshal(err, unmountReply) - default: - panic(fmt.Errorf("requestType (%v) not recognized", requestType)) - } - - replyLen = uint32(len(replyBuf)) - replyLenBuf = make([]byte, 4) - binary.LittleEndian.PutUint32(replyLenBuf, replyLen) - - n, err = netConn.Write(replyLenBuf) - if nil != err { - panic(fmt.Errorf("netConn.Write(replyLenBuf) failed: %v", err)) - } - if n != 4 { - panic(fmt.Errorf("netConn.Write(replyLenBuf) returned n == %v (expected 4)", n)) - } - - n, err = netConn.Write(replyBuf) - if nil != err { - panic(fmt.Errorf("netConn.Write(replyBuf) failed: %v", err)) - } - if n != int(replyLen) { - panic(fmt.Errorf("netConn.Write(replyBuf) returned n == %v (expected %v)", n, replyLen)) - } - } -} - -func benchmarkRpcLeaseCustomDoRequest(b *testing.B, netConn net.Conn, requestType uint32, requestBuf []byte) (replyBuf []byte) { - var ( - err error - n int - replyLen uint32 - replyLenBuf []byte - requestLen uint32 = uint32(len(requestBuf)) - requestLenAndRequestTypeBuf []byte - ) - - requestLenAndRequestTypeBuf = make([]byte, 8) - - binary.LittleEndian.PutUint32(requestLenAndRequestTypeBuf[:4], requestLen) - binary.LittleEndian.PutUint32(requestLenAndRequestTypeBuf[4:], requestType) - - n, err = netConn.Write(requestLenAndRequestTypeBuf) - if nil != err { - b.Fatalf("netConn.Write(requestLenAndRequestTypeBuf) failed: %v", err) - } - if n != 8 { - b.Fatalf("netConn.Write(requestLenAndRequestTypeBuf) returned n == %v (expected 8)", n) - } - - n, err = netConn.Write(requestBuf) - if nil != err { - b.Fatalf("netConn.Write(requestBuf) failed: %v", err) - } - if n != int(requestLen) { - b.Fatalf("netConn.Write(requestBuf) returned n == %v (expected %v)", n, requestLen) - } - - replyLenBuf = make([]byte, 4) - - n, err = netConn.Read(replyLenBuf) - if nil != err { - b.Fatalf("netConn.Read(replyLenBuf) failed: %v", err) - } - if n != 4 { - b.Fatalf("netConn.Read(replyLenBuf) returned n == %v (expected 4)", n) - } - - replyLen = binary.LittleEndian.Uint32(replyLenBuf) - replyBuf = make([]byte, replyLen) - - n, err = netConn.Read(replyBuf) - if nil != err { - b.Fatalf("netConn.Read(replyBuf) failed: %v", err) - } - if n != int(replyLen) { - b.Fatalf("netConn.Read(replyBuf) returned n == %v (expected %v)", n, replyLen) - } - - return -} - -func BenchmarkRpcLeaseShortcutTCP(b *testing.B) { - benchmarkRpcLeaseShortcut(b, false) -} - -func BenchmarkRpcLeaseShortcutTLS(b *testing.B) { - benchmarkRpcLeaseShortcut(b, true) -} - -func benchmarkRpcLeaseShortcut(b *testing.B, useTLS bool) { - var ( - benchmarkIteration int - doneWG sync.WaitGroup - err error - leaseReply *LeaseReply - leaseReplyWrapped *benchmarkRpcShortcutLeaseReplyStruct - leaseRequest *LeaseRequest - leaseRequestWrapped *benchmarkRpcShortcutLeaseRequestStruct - mountByAccountNameReply *MountByAccountNameReply - mountByAccountNameReplyWrapped *benchmarkRpcShortcutMountByAccountNameReplyStruct - mountByAccountNameRequest *MountByAccountNameRequest - mountByAccountNameRequestWrapped *benchmarkRpcShortcutMountByAccountNameRequestStruct - netConn net.Conn - netConnRetries int - netConnRetryDelay time.Duration - ok bool - rootCACertPool *x509.CertPool - tlsConfig *tls.Config - tlsConn *tls.Conn - unmountReply *Reply - unmountReplyWrapped *benchmarkRpcShortcutUnmountReplyStruct - unmountRequest *UnmountRequest - unmountRequestWrapped *benchmarkRpcShortcutUnmountRequestStruct - ) - - doneWG.Add(1) - go benchmarkRpcLeaseShortcutServer(useTLS, &doneWG) - - netConnRetries = 0 - - netConnRetryDelay, err = time.ParseDuration(testRpcLeaseShortcutClientConnRetryDelay) - if nil != err { - b.Fatalf("time.ParseDuration(testRpcLeaseShortcutClientConnRetryDelay=\"%s\") failed: %v", testRpcLeaseShortcutClientConnRetryDelay, err) - } - - if useTLS { - rootCACertPool = x509.NewCertPool() - ok = rootCACertPool.AppendCertsFromPEM(testTLSCerts.caCertPEMBlock) - if !ok { - b.Fatalf("rootCACertPool.AppendCertsFromPEM(testTLSCerts.caCertPEMBlock) returned !ok") - } - tlsConfig = &tls.Config{ - RootCAs: rootCACertPool, - } - } - - for { - if useTLS { - tlsConn, err = tls.Dial("tcp", testRpcLeaseShortcutIPAddrPort, tlsConfig) - if nil == err { - netConn = tlsConn - break - } - } else { - netConn, err = net.Dial("tcp", testRpcLeaseShortcutIPAddrPort) - if nil == err { - break - } - } - - netConnRetries++ - - if netConnRetries > testRpcLeaseShortcutClientConnMaxRetries { - b.Fatalf("netConnRetries exceeded testRpcLeaseShortcutClientConnMaxRetries (%v)", testRpcLeaseShortcutClientConnMaxRetries) - } - - time.Sleep(netConnRetryDelay) - } - - mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - AuthToken: "", - } - mountByAccountNameReply = &MountByAccountNameReply{} - - mountByAccountNameRequestWrapped = &benchmarkRpcShortcutMountByAccountNameRequestStruct{Method: "RpcMountByAccountName", Request: mountByAccountNameRequest} - mountByAccountNameReplyWrapped = &benchmarkRpcShortcutMountByAccountNameReplyStruct{Reply: mountByAccountNameReply} - - benchmarkRpcLeaseShortcutDoRequest(b, netConn, mountByAccountNameRequestWrapped, mountByAccountNameReplyWrapped) - if nil != mountByAccountNameReplyWrapped.Err { - b.Fatalf("benchmarkRpcLeaseShortcutDoRequest(mountByAccountNameRequestWrapped, mountByAccountNameReplyWrapped) failed: %v", mountByAccountNameReplyWrapped.Err) - } - - b.ResetTimer() - - for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeExclusive, - } - leaseReply = &LeaseReply{} - - leaseRequestWrapped = &benchmarkRpcShortcutLeaseRequestStruct{Method: "RpcLease", Request: leaseRequest} - leaseReplyWrapped = &benchmarkRpcShortcutLeaseReplyStruct{Reply: leaseReply} - - benchmarkRpcLeaseShortcutDoRequest(b, netConn, leaseRequestWrapped, leaseReplyWrapped) - if nil != leaseReplyWrapped.Err { - b.Fatalf("benchmarkRpcLeaseShortcutDoRequest(leaseRequestWrapped, leaseReplyWrapped) failed: %v", leaseReplyWrapped.Err) - } - - if LeaseReplyTypeExclusive != leaseReplyWrapped.Reply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReplyWrapped.Reply.LeaseReplyType) - } - - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeRelease, - } - leaseReply = &LeaseReply{} - - leaseRequestWrapped = &benchmarkRpcShortcutLeaseRequestStruct{Method: "RpcLease", Request: leaseRequest} - leaseReplyWrapped = &benchmarkRpcShortcutLeaseReplyStruct{Reply: leaseReply} - - benchmarkRpcLeaseShortcutDoRequest(b, netConn, leaseRequestWrapped, leaseReplyWrapped) - if nil != leaseReplyWrapped.Err { - b.Fatalf("benchmarkRpcLeaseShortcutDoRequest(leaseRequestWrapped, leaseReplyWrapped) failed: %v", leaseReplyWrapped.Err) - } - - if LeaseReplyTypeReleased != leaseReplyWrapped.Reply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReplyWrapped.Reply.LeaseReplyType) - } - } - - b.StopTimer() - - unmountRequest = &UnmountRequest{ - MountID: mountByAccountNameReply.MountID, - } - unmountReply = &Reply{} - - unmountRequestWrapped = &benchmarkRpcShortcutUnmountRequestStruct{Method: "RpcUnmount", Request: unmountRequest} - unmountReplyWrapped = &benchmarkRpcShortcutUnmountReplyStruct{Reply: unmountReply} - - benchmarkRpcLeaseShortcutDoRequest(b, netConn, unmountRequestWrapped, unmountReplyWrapped) - if nil != unmountReplyWrapped.Err { - b.Fatalf("benchmarkRpcLeaseShortcutDoRequest(unmountRequestWrapped, unmountReplyWrapped) failed: %v", unmountReplyWrapped.Err) - } - - err = netConn.Close() - if nil != err { - b.Fatalf("netConn.Close() failed: %v", err) - } - - doneWG.Wait() -} - -type ServerStats struct { - RequestUnmarshalUsec bucketstats.BucketLog2Round - RpcMountRequestUnmarshalUsec bucketstats.BucketLog2Round - RpcMountReplyMarshalUsec bucketstats.BucketLog2Round - RpcLeaseRequestUnmarshalUsec bucketstats.BucketLog2Round - RpcLeaseReplyMarshalUsec bucketstats.BucketLog2Round - RpcUnmountRequestUnmarshalUsec bucketstats.BucketLog2Round - RpcUnmountReplyMarshalUsec bucketstats.BucketLog2Round -} - -func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { - var ( - benchmarkRpcShortcutRequestMethodOnly benchmarkRpcShortcutRequestMethodOnlyStruct - err error - jserver *Server - leaseReply *LeaseReply - leaseReplyWrapped *benchmarkRpcShortcutLeaseReplyStruct - leaseRequest *LeaseRequest - leaseRequestWrapped *benchmarkRpcShortcutLeaseRequestStruct - mountByAccountNameReply *MountByAccountNameReply - mountByAccountNameReplyWrapped *benchmarkRpcShortcutMountByAccountNameReplyStruct - mountByAccountNameRequest *MountByAccountNameRequest - mountByAccountNameRequestWrapped *benchmarkRpcShortcutMountByAccountNameRequestStruct - n int - netConn net.Conn - netListener net.Listener - replyBuf []byte - replyLen uint32 - replyLenBuf []byte - requestBuf []byte - requestLen uint32 - requestLenBuf []byte - tlsConfig *tls.Config - unmountReply *Reply - unmountReplyWrapped *benchmarkRpcShortcutUnmountReplyStruct - unmountRequest *UnmountRequest - unmountRequestWrapped *benchmarkRpcShortcutUnmountRequestStruct - serverStats ServerStats - ) - - bucketstats.Register("benchmark", "shortcutServer", &serverStats) - defer func() { - /* TODO - uncomment if the bucketstats should be dumped - fmt.Printf("%s\n", - bucketstats.SprintStats(bucketstats.StatFormatParsable1, "benchmark", "shortcutServer")) - */ - bucketstats.UnRegister("benchmark", "shortcutServer") - }() - - jserver = NewServer() - - netListener, err = net.Listen("tcp", testRpcLeaseShortcutIPAddrPort) - if nil != err { - panic(fmt.Errorf("net.Listen(\"tcp\", testRpcLeaseShortcutIPAddrPort) failed: %v", err)) - } - - if useTLS { - tlsConfig = &tls.Config{ - Certificates: []tls.Certificate{testTLSCerts.endpointTLSCert}, - } - - netListener = tls.NewListener(netListener, tlsConfig) - } - - netConn, err = netListener.Accept() - if nil != err { - panic(fmt.Errorf("netListener.Accept() failed: %v", err)) - } - - for { - requestLenBuf = make([]byte, 4) - - n, err = netConn.Read(requestLenBuf) - if nil != err { - if io.EOF != err { - panic(fmt.Errorf("netConn.Read(requestLenBuf) failed: %v", err)) - } - - err = netConn.Close() - if nil != err { - panic(fmt.Errorf("netConn.Close() failed: %v", err)) - } - - err = netListener.Close() - if nil != err { - panic(fmt.Errorf("netListener.Close() failed: %v", err)) - } - - doneWG.Done() - - return - } - if n != 4 { - panic(fmt.Errorf("netConn.Read(requestLenBuf) returned n == %v (expected 4)", n)) - } - - requestLen = binary.LittleEndian.Uint32(requestLenBuf) - requestBuf = make([]byte, requestLen) - - n, err = netConn.Read(requestBuf) - if nil != err { - panic(fmt.Errorf("netConn.Read(requestBuf) failed: %v", err)) - } - if n != int(requestLen) { - panic(fmt.Errorf("netConn.Read(requestBuf) returned n == %v (expected %v)", n, requestLen)) - } - - startTime := time.Now() - err = json.Unmarshal(requestBuf, &benchmarkRpcShortcutRequestMethodOnly) - serverStats.RequestUnmarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) - - if nil != err { - panic(fmt.Errorf("json.Unmarshal(requestBuf, benchmarkRpcShortcutRequestMethodOnly) failed: %v", err)) - } - - switch benchmarkRpcShortcutRequestMethodOnly.Method { - case "RpcMountByAccountName": - mountByAccountNameRequestWrapped = &benchmarkRpcShortcutMountByAccountNameRequestStruct{} - - startTime = time.Now() - err = json.Unmarshal(requestBuf, mountByAccountNameRequestWrapped) - serverStats.RpcMountRequestUnmarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) - - if nil != err { - panic(fmt.Errorf("json.Unmarshal(requestBuf, mountByAccountNameRequestWrapped) failed: %v", err)) - } - - mountByAccountNameRequest = mountByAccountNameRequestWrapped.Request - mountByAccountNameReply = &MountByAccountNameReply{} - - err = jserver.RpcMountByAccountName(testRpcLeaseLocalClientID, mountByAccountNameRequest, mountByAccountNameReply) - - mountByAccountNameReplyWrapped = &benchmarkRpcShortcutMountByAccountNameReplyStruct{ - Err: err, - Reply: mountByAccountNameReply, - } - - startTime = time.Now() - replyBuf, err = json.Marshal(mountByAccountNameReplyWrapped) - serverStats.RpcMountReplyMarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) - if nil != err { - panic(fmt.Errorf("json.Marshal(mountByAccountNameReplyWrapped) failed")) - } - case "RpcLease": - leaseRequestWrapped = &benchmarkRpcShortcutLeaseRequestStruct{} - - startTime = time.Now() - err = json.Unmarshal(requestBuf, leaseRequestWrapped) - serverStats.RpcLeaseRequestUnmarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) - if nil != err { - panic(fmt.Errorf("json.Unmarshal(requestBuf, leaseRequestWrapped) failed: %v", err)) - } - - leaseRequest = leaseRequestWrapped.Request - leaseReply = &LeaseReply{} - - err = jserver.RpcLease(leaseRequest, leaseReply) - - leaseReplyWrapped = &benchmarkRpcShortcutLeaseReplyStruct{ - Err: err, - Reply: leaseReply, - } - - startTime = time.Now() - replyBuf, err = json.Marshal(leaseReplyWrapped) - serverStats.RpcLeaseReplyMarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) - if nil != err { - panic(fmt.Errorf("json.Marshal(leaseReplyWrapped) failed")) - } - case "RpcUnmount": - unmountRequestWrapped = &benchmarkRpcShortcutUnmountRequestStruct{} - - startTime = time.Now() - err = json.Unmarshal(requestBuf, unmountRequestWrapped) - serverStats.RpcUnmountRequestUnmarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) - if nil != err { - panic(fmt.Errorf("json.Unmarshal(requestBuf, unmountRequestWrapped) failed: %v", err)) - } - - unmountRequest = unmountRequestWrapped.Request - unmountReply = &Reply{} - - err = jserver.RpcUnmount(unmountRequest, unmountReply) - - unmountReplyWrapped = &benchmarkRpcShortcutUnmountReplyStruct{ - Err: err, - Reply: unmountReply, - } - - startTime = time.Now() - replyBuf, err = json.Marshal(unmountReplyWrapped) - serverStats.RpcUnmountReplyMarshalUsec.Add(uint64(time.Since(startTime).Microseconds())) - if nil != err { - panic(fmt.Errorf("json.Marshal(unmountReplyWrapped) failed")) - } - default: - panic(fmt.Errorf("benchmarkRpcShortcutRequestMethodOnly.Method (\"%s\") not recognized", benchmarkRpcShortcutRequestMethodOnly.Method)) - } - - replyLen = uint32(len(replyBuf)) - replyLenBuf = make([]byte, 4) - binary.LittleEndian.PutUint32(replyLenBuf, replyLen) - - n, err = netConn.Write(replyLenBuf) - if nil != err { - panic(fmt.Errorf("netConn.Write(replyLenBuf) failed: %v", err)) - } - if n != 4 { - panic(fmt.Errorf("netConn.Write(replyLenBuf) returned n == %v (expected 4)", n)) - } - - n, err = netConn.Write(replyBuf) - if nil != err { - panic(fmt.Errorf("netConn.Write(replyBuf) failed: %v", err)) - } - if n != int(replyLen) { - panic(fmt.Errorf("netConn.Write(replyBuf) returned n == %v (expected %v)", n, replyLen)) - } - } -} - -func benchmarkRpcLeaseShortcutDoRequest(b *testing.B, netConn net.Conn, request interface{}, reply interface{}) { - var ( - err error - n int - replyBuf []byte - replyLen uint32 - replyLenBuf []byte - requestBuf []byte - requestLen uint32 - requestLenBuf []byte - ) - - requestBuf, err = json.Marshal(request) - if nil != err { - b.Fatalf("json.Marshal(request) failed: %v", err) - } - - requestLen = uint32(len(requestBuf)) - requestLenBuf = make([]byte, 4) - binary.LittleEndian.PutUint32(requestLenBuf, requestLen) - - n, err = netConn.Write(requestLenBuf) - if nil != err { - b.Fatalf("netConn.Write(requestLenBuf) failed: %v", err) - } - if n != 4 { - b.Fatalf("netConn.Write(requestLenBuf) returned n == %v (expected 4)", n) - } - - n, err = netConn.Write(requestBuf) - if nil != err { - b.Fatalf("netConn.Write(requestBuf) failed: %v", err) - } - if n != int(requestLen) { - b.Fatalf("netConn.Write(requestBuf) returned n == %v (expected %v)", n, requestLen) - } - - replyLenBuf = make([]byte, 4) - - n, err = netConn.Read(replyLenBuf) - if nil != err { - b.Fatalf("netConn.Read(replyLenBuf) failed: %v", err) - } - if n != 4 { - b.Fatalf("netConn.Read(replyLenBuf) returned n == %v (expected 4)", n) - } - - replyLen = binary.LittleEndian.Uint32(replyLenBuf) - replyBuf = make([]byte, replyLen) - - n, err = netConn.Read(replyBuf) - if nil != err { - b.Fatalf("netConn.Read(replyBuf) failed: %v", err) - } - if n != int(replyLen) { - b.Fatalf("netConn.Read(replyBuf) returned n == %v (expected %v)", n, replyLen) - } - - err = json.Unmarshal(replyBuf, reply) - if nil != err { - b.Fatalf("json.Unmarshal(replyBuf, reply) failed: %v", err) - } -} - -func BenchmarkRpcLeaseLocal(b *testing.B) { - var ( - benchmarkIteration int - err error - jserver *Server - leaseReply *LeaseReply - leaseRequest *LeaseRequest - mountByAccountNameReply *MountByAccountNameReply - mountByAccountNameRequest *MountByAccountNameRequest - unmountReply *Reply - unmountRequest *UnmountRequest - ) - - jserver = NewServer() - - mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - AuthToken: "", - } - mountByAccountNameReply = &MountByAccountNameReply{} - - err = jserver.RpcMountByAccountName(testRpcLeaseLocalClientID, mountByAccountNameRequest, mountByAccountNameReply) - if nil != err { - b.Fatalf("jserver.RpcMountByAccountName(testRpcLeaseLocalClientID, mountByAccountNameRequest, mountByAccountNameReply) failed: %v", err) - } - - b.ResetTimer() - - for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeExclusive, - } - leaseReply = &LeaseReply{} - - err = jserver.RpcLease(leaseRequest, leaseReply) - if nil != err { - b.Fatalf("jserver.RpcLease(leaseRequest, leaseReply) failed: %v", err) - } - - if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReply.LeaseReplyType) - } - - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeRelease, - } - leaseReply = &LeaseReply{} - - err = jserver.RpcLease(leaseRequest, leaseReply) - if nil != err { - b.Fatalf("jserver.RpcLease(leaseRequest, leaseReply) failed: %v", err) - } - - if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReply.LeaseReplyType) - } - } - - b.StopTimer() - - unmountRequest = &UnmountRequest{ - MountID: mountByAccountNameReply.MountID, - } - unmountReply = &Reply{} - - err = jserver.RpcUnmount(unmountRequest, unmountReply) - if nil != err { - b.Fatalf("jserver.RpcUnmount(unmountRequest, unmountReply) failed: %v", err) - } -} - -func BenchmarkRpcLeaseRemote(b *testing.B) { - var ( - benchmarkIteration int - deadlineIO time.Duration - err error - keepAlivePeriod time.Duration - leaseReply *LeaseReply - leaseRequest *LeaseRequest - mountByAccountNameReply *MountByAccountNameReply - mountByAccountNameRequest *MountByAccountNameRequest - retryRPCClient *retryrpc.Client - retryrpcClientConfig *retryrpc.ClientConfig - testRpcLeaseClient *testRpcLeaseClientStruct - unmountReply *Reply - unmountRequest *UnmountRequest - ) - - deadlineIO, err = time.ParseDuration(testRpcLeaseRetryRPCDeadlineIO) - if nil != err { - b.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRpcLeaseRetryRPCDeadlineIO, err) - } - keepAlivePeriod, err = time.ParseDuration(testRpcLeaseRetryRPCKeepAlivePeriod) - if nil != err { - b.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRpcLeaseRetryRPCKeepAlivePeriod, err) - } - - retryrpcClientConfig = &retryrpc.ClientConfig{ - DNSOrIPAddr: globals.publicIPAddr, - Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, - Callbacks: testRpcLeaseClient, - DeadlineIO: deadlineIO, - KeepAlivePeriod: keepAlivePeriod, - } - - retryRPCClient, err = retryrpc.NewClient(retryrpcClientConfig) - if nil != err { - b.Fatalf("retryrpc.NewClient() failed: %v", err) - } - - mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - AuthToken: "", - } - mountByAccountNameReply = &MountByAccountNameReply{} - - err = retryRPCClient.Send("RpcMountByAccountName", mountByAccountNameRequest, mountByAccountNameReply) - if nil != err { - b.Fatalf("retryRPCClient.Send(\"RpcMountByAccountName\",,) failed: %v", err) - } - - b.ResetTimer() - - for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeExclusive, - } - leaseReply = &LeaseReply{} - - err = retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - b.Fatalf("retryRPCClient.Send(\"RpcLease\",LeaseRequestTypeExclusive) failed: %v", err) - } - - if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReply.LeaseReplyType) - } - - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseSingleInodeNumber, - }, - LeaseRequestType: LeaseRequestTypeRelease, - } - leaseReply = &LeaseReply{} - - err = retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - b.Fatalf("retryRPCClient.Send(\"RpcLease\",LeaseRequestTypeRelease) failed: %v", err) - } - - if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { - b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReply.LeaseReplyType) - } - } - - b.StopTimer() - - fmt.Printf("RetryRPC Bucketstats: %v\n", bucketstats.SprintStats(bucketstats.StatFormatParsable1, "proxyfs.retryrpc", "*")) - - unmountRequest = &UnmountRequest{ - MountID: mountByAccountNameReply.MountID, - } - unmountReply = &Reply{} - - err = retryRPCClient.Send("RpcUnmount", unmountRequest, unmountReply) - if nil != err { - b.Fatalf("retryRPCClient.Send(\"RpcUnmount\",,) failed: %v", err) - } - - retryRPCClient.Close() -} - -func TestRpcLease(t *testing.T) { - var ( - instance int - testRpcLeaseClient []*testRpcLeaseClientStruct - wg sync.WaitGroup - ) - - // Setup Single Lease instances - - wg.Add(testRpcLeaseSingleNumInstances) - - testRpcLeaseClient = make([]*testRpcLeaseClientStruct, testRpcLeaseSingleNumInstances) - - for instance = 0; instance < testRpcLeaseSingleNumInstances; instance++ { - testRpcLeaseClient[instance] = &testRpcLeaseClientStruct{ - instance: instance, - inodeNumber: testRpcLeaseSingleInodeNumber, - chIn: make(chan LeaseRequestType), - chOut: make(chan interface{}), - alreadyUnmounted: false, - wg: &wg, - t: t, - } - - go testRpcLeaseClient[instance].instanceGoroutine() - } - - // Perform Single Lease test cases - - testRpcLeaseLogTestCase("1 Shared", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("2 Shared", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("3 Shared", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("1 Exclusive", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("1 Exclusive then Demote", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeDemoted) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("1 Exclusive then 1 Shared leading to Demotion", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeDemoted, RPCInterruptTypeDemote) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("1 Shared then 1 Exclusive leading to Release", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("1 Exclusive then 1 Exclusive leading to Release", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("2 Shared then Promotion leading to Release", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypePromote) - testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypePromoted) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("1 Exclusive then 2 Shared leading to Demotion", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeDemoted, RPCInterruptTypeDemote) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("2 Shared then 1 Exclusive leading to Release", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("2 Exclusives leading to Release that Expires", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseClient[0].alreadyUnmounted = true - - testRpcLeaseLogTestCase("2 Shared then 2 Promotions leading to Release", true) - - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypePromote) - testRpcLeaseClient[2].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypePromote) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeDenied, RPCInterruptTypeRelease) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypePromoted) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase(fmt.Sprintf("%v Shared", testRpcLeaseSingleNumInstances-1), false) - - for instance = 1; instance < testRpcLeaseSingleNumInstances; instance++ { - testRpcLeaseClient[instance].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[instance].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - } - for instance = 1; instance < testRpcLeaseSingleNumInstances; instance++ { - testRpcLeaseClient[instance].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[instance].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - } - - testRpcLeaseLogTestCase(fmt.Sprintf("%v Exclusives", testRpcLeaseSingleNumInstances-1), false) - - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - for instance = 2; instance < testRpcLeaseSingleNumInstances; instance++ { - testRpcLeaseClient[instance].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[(instance - 1)].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[(instance - 1)].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[(instance-1)].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[instance].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - } - testRpcLeaseClient[(testRpcLeaseSingleNumInstances - 1)].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[(testRpcLeaseSingleNumInstances - 1)].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - // Shutdown Single Lease instances - - for instance = 0; instance < testRpcLeaseSingleNumInstances; instance++ { - close(testRpcLeaseClient[instance].chIn) - } - - wg.Wait() - - // Setup Multi Lease instances - - wg.Add(testRpcLeaseMultiNumInstances) - - testRpcLeaseClient = make([]*testRpcLeaseClientStruct, testRpcLeaseMultiNumInstances) - - for instance = 0; instance < testRpcLeaseMultiNumInstances; instance++ { - testRpcLeaseClient[instance] = &testRpcLeaseClientStruct{ - instance: instance, - inodeNumber: (testRpcLeaseMultiFirstInodeNumber + int64(instance)), - chIn: make(chan LeaseRequestType), - chOut: make(chan interface{}), - alreadyUnmounted: false, - wg: &wg, - t: t, - } - - go testRpcLeaseClient[instance].instanceGoroutine() - } - - // Perform Multi Lease test case - - testRpcLeaseLogTestCase(fmt.Sprintf("%v Unique InodeNumber Exclusives", testRpcLeaseMultiNumInstances), true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeExclusive) - - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - - testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeExclusive) - - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - - testRpcLeaseClient[3].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - - testRpcLeaseClient[4].sendLeaseRequest(LeaseRequestTypeExclusive) - - testRpcLeaseClient[4].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[4].sendLeaseRequest(LeaseRequestTypeRelease) - - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[3].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[4].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - // Shutdown Multi Lease instances - - for instance = 0; instance < testRpcLeaseMultiNumInstances; instance++ { - close(testRpcLeaseClient[instance].chIn) - } - - wg.Wait() -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { - var ( - deadlineIO time.Duration - err error - keepAlivePeriod time.Duration - leaseReply *LeaseReply - leaseRequest *LeaseRequest - leaseRequestType LeaseRequestType - mountByAccountNameRequest *MountByAccountNameRequest - mountByAccountNameReply *MountByAccountNameReply - ok bool - retryRPCClient *retryrpc.Client - retryrpcClientConfig *retryrpc.ClientConfig - unmountReply *Reply - unmountRequest *UnmountRequest - ) - - deadlineIO, err = time.ParseDuration(testRpcLeaseRetryRPCDeadlineIO) - if nil != err { - testRpcLeaseClient.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRpcLeaseRetryRPCDeadlineIO, err) - } - keepAlivePeriod, err = time.ParseDuration(testRpcLeaseRetryRPCKeepAlivePeriod) - if nil != err { - testRpcLeaseClient.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRpcLeaseRetryRPCKeepAlivePeriod, err) - } - - retryrpcClientConfig = &retryrpc.ClientConfig{ - DNSOrIPAddr: globals.publicIPAddr, - Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, - Callbacks: testRpcLeaseClient, - DeadlineIO: deadlineIO, - KeepAlivePeriod: keepAlivePeriod, - } - - retryRPCClient, err = retryrpc.NewClient(retryrpcClientConfig) - if nil != err { - testRpcLeaseClient.Fatalf("retryrpc.NewClient() failed: %v", err) - } - - mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - AuthToken: "", - } - mountByAccountNameReply = &MountByAccountNameReply{} - - err = retryRPCClient.Send("RpcMountByAccountName", mountByAccountNameRequest, mountByAccountNameReply) - if nil != err { - testRpcLeaseClient.Fatalf("retryRPCClient.Send(\"RpcMountByAccountName\",,) failed: %v", err) - } - - for { - leaseRequestType, ok = <-testRpcLeaseClient.chIn - - if ok { - leaseRequest = &LeaseRequest{ - InodeHandle: InodeHandle{ - MountID: mountByAccountNameReply.MountID, - InodeNumber: testRpcLeaseClient.inodeNumber, - }, - LeaseRequestType: leaseRequestType, - } - leaseReply = &LeaseReply{} - - testRpcLeaseClient.logEvent(leaseRequest.LeaseRequestType) - - err = retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - testRpcLeaseClient.Fatalf("retryRPCClient.Send(\"RpcLease\",LeaseRequestType=%d) failed: %v", leaseRequestType, err) - } - - testRpcLeaseClient.logEvent(leaseReply.LeaseReplyType) - - testRpcLeaseClient.chOut <- leaseReply.LeaseReplyType - } else { - unmountRequest = &UnmountRequest{ - MountID: mountByAccountNameReply.MountID, - } - unmountReply = &Reply{} - - err = retryRPCClient.Send("RpcUnmount", unmountRequest, unmountReply) - if testRpcLeaseClient.alreadyUnmounted { - if nil == err { - testRpcLeaseClient.Fatalf("retryRPCClient.Send(\"RpcUnmount\",,) should have failed") - } - } else { - if nil != err { - testRpcLeaseClient.Fatalf("retryRPCClient.Send(\"RpcUnmount\",,) failed: %v", err) - } - } - - retryRPCClient.Close() - - testRpcLeaseClient.wg.Done() - - runtime.Goexit() - } - } -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) Interrupt(rpcInterruptBuf []byte) { - var ( - err error - rpcInterrupt *RPCInterrupt - ) - - rpcInterrupt = &RPCInterrupt{} - - err = json.Unmarshal(rpcInterruptBuf, rpcInterrupt) - if nil != err { - testRpcLeaseClient.Fatalf("json.Unmarshal() failed: %v", err) - } - if rpcInterrupt.InodeNumber != testRpcLeaseClient.inodeNumber { - testRpcLeaseClient.Fatalf("Interrupt() called for InodeNumber %v... expected to be for %v", rpcInterrupt.InodeNumber, testRpcLeaseClient.inodeNumber) - } - - testRpcLeaseClient.logEvent(rpcInterrupt.RPCInterruptType) - - testRpcLeaseClient.chOut <- rpcInterrupt.RPCInterruptType -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) Fatalf(format string, args ...interface{}) { - var ( - argsForPrintf []interface{} - argsIndex int - argsValue interface{} - formatForPrintf string - ) - - formatForPrintf = "Failing testRpcLeaseClient %v: " + format + "\n" - - argsForPrintf = make([]interface{}, len(args)+1) - argsForPrintf[0] = testRpcLeaseClient.instance - for argsIndex, argsValue = range args { - argsForPrintf[argsIndex+1] = argsValue - } - - fmt.Printf(formatForPrintf, argsForPrintf...) - - os.Exit(-1) -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) sendLeaseRequest(leaseRequestType LeaseRequestType) { - time.Sleep(testRpcLeaseDelayBeforeSendingRequest) - testRpcLeaseClient.chIn <- leaseRequestType - time.Sleep(testRpcLeaseDelayAfterSendingRequest) -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) sendLeaseRequestPromptly(leaseRequestType LeaseRequestType) { - testRpcLeaseClient.chIn <- leaseRequestType -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) validateChOutValueIsLeaseReplyType(expectedLeaseReplyType LeaseReplyType) { - var ( - chOutValueAsInterface interface{} - chOutValueAsLeaseReplyType LeaseReplyType - ok bool - ) - - chOutValueAsInterface = <-testRpcLeaseClient.chOut - - chOutValueAsLeaseReplyType, ok = chOutValueAsInterface.(LeaseReplyType) - if !ok { - testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut did not return a LeaseReplyType") - } - if chOutValueAsLeaseReplyType != expectedLeaseReplyType { - testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut returned LeaseReplyType %v... expected %v", chOutValueAsLeaseReplyType, expectedLeaseReplyType) - } -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(expectedLeaseReplyType LeaseReplyType, ignoredRPCInterruptType RPCInterruptType) { - var ( - chOutValueAsInterface interface{} - chOutValueAsRPCInterruptType RPCInterruptType - chOutValueAsLeaseReplyType LeaseReplyType - ok bool - ) - - for { - chOutValueAsInterface = <-testRpcLeaseClient.chOut - - chOutValueAsRPCInterruptType, ok = chOutValueAsInterface.(RPCInterruptType) - if ok { - if chOutValueAsRPCInterruptType != ignoredRPCInterruptType { - testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut did not return an ignored RPCInterruptType") - } - } else { - break - } - } - - chOutValueAsLeaseReplyType, ok = chOutValueAsInterface.(LeaseReplyType) - if !ok { - testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut did not return a LeaseReplyType or ignored RPCInterruptType") - } - if chOutValueAsLeaseReplyType != expectedLeaseReplyType { - testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut returned LeaseReplyType %v... expected %v", chOutValueAsLeaseReplyType, expectedLeaseReplyType) - } -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) validateChOutValueIsRPCInterruptType(expectedRPCInterruptType RPCInterruptType) { - var ( - chOutValueAsInterface interface{} - chOutValueAsRPCInterruptType RPCInterruptType - ok bool - ) - - chOutValueAsInterface = <-testRpcLeaseClient.chOut - - chOutValueAsRPCInterruptType, ok = chOutValueAsInterface.(RPCInterruptType) - if !ok { - testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut did not return a RPCInterruptType") - } - if chOutValueAsRPCInterruptType != expectedRPCInterruptType { - testRpcLeaseClient.t.Fatalf("<-testRpcLeaseClient.chOut returned RPCInterruptType %v... expected %v", chOutValueAsRPCInterruptType, expectedRPCInterruptType) - } -} - -func testRpcLeaseLogTestCase(testCase string, verbose bool) { - fmt.Printf("%v %s\n", time.Now().Format(testRpcLeaseTimeFormat), testCase) - testRpcLeaseLogVerbosely = verbose -} - -func (testRpcLeaseClient *testRpcLeaseClientStruct) logEvent(ev interface{}) { - if testRpcLeaseLogVerbosely { - switch ev.(type) { - case LeaseRequestType: - fmt.Printf("%v %s%s-> \n", time.Now().Format(testRpcLeaseTimeFormat), strings.Repeat(" ", testRpcLeaseClient.instance), testRpcLeaseRequestLetters[ev.(LeaseRequestType)]) - case LeaseReplyType: - fmt.Printf("%v %s <-%s\n", time.Now().Format(testRpcLeaseTimeFormat), strings.Repeat(" ", testRpcLeaseClient.instance), testRpcLeaseReplyLetters[ev.(LeaseReplyType)]) - case RPCInterruptType: - fmt.Printf("%v %s ^^%s\n", time.Now().Format(testRpcLeaseTimeFormat), strings.Repeat(" ", testRpcLeaseClient.instance), testRpcLeaseInterruptLetters[ev.(RPCInterruptType)]) - } - } -} diff --git a/jrpcfs/middleware.go b/jrpcfs/middleware.go deleted file mode 100644 index 1fb422f6..00000000 --- a/jrpcfs/middleware.go +++ /dev/null @@ -1,421 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "fmt" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/utils" -) - -// JSON RPC Server on top of the FS package. -// -// This file handles the RPCs related to Swift middleware support -// -// NOTE: These functions should only verify the arguments and then call -// functions in package fs since there may be fs locking required. - -// parseVirtPath extracts path components and fetches the corresponding fs.VolumeHandle from virtPath -func parseVirtPath(virtPath string) (accountName string, vContainerName string, vObjectName string, volumeName string, volumeHandle fs.VolumeHandle, err error) { - - // Extract Account, vContainer, and vObject from VirtPath - accountName, vContainerName, vObjectName, err = utils.PathToAcctContObj(virtPath) - if nil != err { - return - } - - // Map accountName to volumeHandle - volumeHandle, err = fs.FetchVolumeHandleByAccountName(accountName) - if nil != err { - return - } - - // Map volumeHandle to volumeName - volumeName = volumeHandle.VolumeName() - - return -} - -// RpcCreateContainer is used by Middleware to PUT of a container. -// -// TODO - add update of metadata -// TODO - combine this into one PUT RPC instead of multiple RPCs? -func (s *Server) RpcCreateContainer(in *CreateContainerRequest, reply *CreateContainerReply) (err error) { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - // TODO: Need to determine how we want to pass errors back for RPCs used by middleware. - // By default, jrpcfs code (and rpcEncodeError) use errno-type errors. - // However for RPCs used by middleware, perhaps we want to return HTTP status codes? - // The blunder error package supports this, we just need to add some helper functions. - //defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - accountName, containerName, _, _, volumeHandle, err := parseVirtPath(in.VirtPath) - - // Validate the components of the containerName - err = fs.ValidateFullPath(containerName) - if err != nil { - return err - } - - err = fs.ValidateBaseName(containerName) - if err != nil { - return err - } - - // Make the directory - _, err = volumeHandle.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, containerName, inode.PosixModePerm) - - if err != nil { - logger.DebugfIDWithError(internalDebug, err, "fs.Mkdir() of acct: %v vContainerName: %v failed!", accountName, containerName) - } - return -} - -// RpcDelete is used by Middleware to service a DELETE HTTP request. -// -// This routine has to handle delete of an empty container as well as delete of an objectName -// where objectName could be "file1", "dir1/file1", "dir1/dir2", etc. -func (s *Server) RpcDelete(in *DeleteReq, reply *DeleteReply) (err error) { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - _, containerName, objectName, _, volumeHandle, err := parseVirtPath(in.VirtPath) - - parentDir, baseName := splitPath(containerName + "/" + objectName) - - // objectName empty means we are deleting a container - if objectName == "" { - parentDir = "/" - baseName = containerName - } - - // Call fs to delete the baseName if it is a file or an empty directory. - err = volumeHandle.MiddlewareDelete(parentDir, baseName) - - return err -} - -// RpcGetAccount is used by Middleware to issue a GET on an account and return the results. -func (s *Server) RpcGetAccount(in *GetAccountReq, reply *GetAccountReply) (err error) { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - _, _, _, _, volumeHandle, err := parseVirtPath(in.VirtPath) - if err != nil { - logger.ErrorfWithError(err, "RpcGetAccount: error mounting share for %s", in.VirtPath) - return err - } - - entries, mtime, ctime, err := volumeHandle.MiddlewareGetAccount(in.MaxEntries, in.Marker, in.EndMarker) - if err != nil { - return err - } - reply.AccountEntries = entries - reply.ModificationTime = mtime - reply.AttrChangeTime = ctime - return nil -} - -// RpcHead is used by Middleware to issue a HEAD on a container or object and return the results. -func (s *Server) RpcHead(in *HeadReq, reply *HeadReply) (err error) { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - _, vContainerName, vObjectName, _, volumeHandle, err := parseVirtPath(in.VirtPath) - if err != nil { - logger.ErrorfWithError(err, "RpcHead: error mounting share for %s", in.VirtPath) - return err - } - - entityPath := vContainerName - if vObjectName != "" { - entityPath = entityPath + "/" + vObjectName - } - - resp, err := volumeHandle.MiddlewareHeadResponse(entityPath) - if err != nil { - if !blunder.Is(err, blunder.NotFoundError) { - logger.ErrorfWithError(err, "RpcHead: error retrieving metadata for %s", in.VirtPath) - } - return err - } - - reply.Metadata = resp.Metadata - reply.FileSize = resp.FileSize - reply.ModificationTime = resp.ModificationTime - reply.AttrChangeTime = resp.AttrChangeTime - reply.InodeNumber = int64(uint64(resp.InodeNumber)) - reply.NumWrites = resp.NumWrites - reply.IsDir = resp.IsDir - - return nil -} - -// RpcGetContainer is used by Middleware to issue a GET on a container and return the results. -func (s *Server) RpcGetContainer(in *GetContainerReq, reply *GetContainerReply) (err error) { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - _, vContainerName, _, _, volumeHandle, err := parseVirtPath(in.VirtPath) - if err != nil { - logger.ErrorfWithError(err, "RpcGetContainer: error mounting share for %s", in.VirtPath) - return err - } - - entries, err := volumeHandle.MiddlewareGetContainer(vContainerName, in.MaxEntries, in.Marker, in.EndMarker, in.Prefix, in.Delimiter) - if err != nil { - return err - } - resp, err := volumeHandle.MiddlewareHeadResponse(vContainerName) - if err != nil { - return err - } - reply.ContainerEntries = entries - reply.Metadata = resp.Metadata - reply.ModificationTime = resp.ModificationTime - return nil -} - -// RpcGetObject is used by GET HTTP request to retrieve the read plan for an object. -func (s *Server) RpcGetObject(in *GetObjectReq, reply *GetObjectReply) (err error) { - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - _, vContainerName, objectName, _, volumeHandle, err := parseVirtPath(in.VirtPath) - - mountRelativePath := vContainerName + "/" + objectName - - resp, err := volumeHandle.MiddlewareGetObject(mountRelativePath, in.ReadEntsIn, &reply.ReadEntsOut) - if err != nil { - if !blunder.Is(err, blunder.NotFoundError) { - logger.ErrorfWithError(err, "RpcGetObject(): error retrieving metadata for %s", in.VirtPath) - } - return err - } - - reply.Metadata = resp.Metadata - reply.FileSize = resp.FileSize - reply.ModificationTime = resp.ModificationTime - reply.AttrChangeTime = resp.AttrChangeTime - reply.InodeNumber = uint64(resp.InodeNumber) - reply.NumWrites = resp.NumWrites - reply.IsDir = resp.IsDir - - return err -} - -// RpcPost handles a POST command from middleware for an account, container or object. -func (s *Server) RpcPost(in *MiddlewarePostReq, reply *MiddlewarePostReply) (err error) { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - - accountName, containerName, objectName, _, volumeHandle, err := parseVirtPath(in.VirtPath) - - // Don't allow a POST on an invalid account or on just an account - if accountName == "" || containerName == "" { - err = fmt.Errorf("%s: Can't modify an account, AccountName: %v is invalid or ContainerName: %v is invalid.", utils.GetFnName(), accountName, containerName) - logger.ErrorWithError(err) - err = blunder.AddError(err, blunder.AccountNotModifiable) - return err - } - - var parentDir, baseName string - if objectName != "" { - parentDir, baseName = splitPath(containerName + "/" + objectName) - } else { - parentDir, baseName = splitPath(containerName) - } - - err = volumeHandle.MiddlewarePost(parentDir, baseName, in.NewMetaData, in.OldMetaData) - - return err - -} - -// Makes a directory. Unlike RpcMkdir, one can invoke this with just a path. -func (s *Server) RpcMiddlewareMkdir(in *MiddlewareMkdirReq, reply *MiddlewareMkdirReply) (err error) { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - - _, containerName, objectName, _, volumeHandle, err := parseVirtPath(in.VirtPath) - - // Require a reference to an object; you can't create a container with this method. - if objectName == "" { - err = blunder.NewError(blunder.NotAnObjectError, "%s: VirtPath must reference an object, not container or account (%s)", utils.GetFnName(), in.VirtPath) - // This is worth logging; a correct middleware will never send such a path. - logger.ErrorWithError(err) - return err - } - - mtime, ctime, inodeNumber, numWrites, err := volumeHandle.MiddlewareMkdir(containerName, objectName, in.Metadata) - reply.ModificationTime = mtime - reply.AttrChangeTime = ctime - reply.InodeNumber = int64(uint64(inodeNumber)) - reply.NumWrites = numWrites - return -} - -// RpcPutComplete is used by PUT HTTP request once data has been put in Swift. -// -// Sets up inode, etc. -func (s *Server) RpcPutComplete(in *PutCompleteReq, reply *PutCompleteReply) (err error) { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - _, containerName, objectName, _, volumeHandle, err := parseVirtPath(in.VirtPath) - - // Call fs to complete the creation of the inode for the file and - // the directories. - mtime, ctime, ino, numWrites, err := volumeHandle.MiddlewarePutComplete(containerName, objectName, in.PhysPaths, in.PhysLengths, in.Metadata) - reply.ModificationTime = mtime - reply.AttrChangeTime = ctime - reply.InodeNumber = int64(uint64(ino)) - reply.NumWrites = numWrites - - return err -} - -// RpcPutLocation is used by PUT HTTP request to provision an object so that middleware -// can PUT the object in Swift. -// -// Later, a RpcPutComplete() will be called to setup inode, etc. -func (s *Server) RpcPutLocation(in *PutLocationReq, reply *PutLocationReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - - accountName, containerName, objectName, _, volumeHandle, err := parseVirtPath(in.VirtPath) - - // Validate the components of the objectName - err = fs.ValidateFullPath(containerName + "/" + objectName) - if err != nil { - return err - } - - _, baseName := splitPath(containerName + "/" + objectName) - err = fs.ValidateBaseName(baseName) - if err != nil { - return err - } - - // Via fs package, ask inode package to provision object - reply.PhysPath, err = volumeHandle.CallInodeToProvisionObject() - - if err != nil { - logger.DebugfIDWithError(internalDebug, err, "fs.CallInodeToProvisionObject() of acct: %v container: %v failed!", accountName, containerName) - } - return -} - -// RpcPutContainer creates or updates a container (top-level directory). -func (s *Server) RpcPutContainer(in *PutContainerReq, reply *PutContainerReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - _, containerName, _, _, volumeHandle, err := parseVirtPath(in.VirtPath) - if err != nil { - return err - } - - err = volumeHandle.MiddlewarePutContainer(containerName, in.OldMetadata, in.NewMetadata) - return err -} - -// Combine a bunch of files together into a big one. It's like "cat old1 old2 ... > new", but without the cat. Also -// removes the files old1 old2 ... -func (s *Server) RpcCoalesce(in *CoalesceReq, reply *CoalesceReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - _, destContainer, destObject, _, volumeHandle, err := parseVirtPath(in.VirtPath) - - var ino uint64 - ino, reply.NumWrites, reply.AttrChangeTime, reply.ModificationTime, err = - volumeHandle.MiddlewareCoalesce( - destContainer+"/"+destObject, in.NewMetaData, in.ElementAccountRelativePaths) - reply.InodeNumber = int64(ino) - return -} - -// Renew a lease, ensuring that the related file's log segments won't get deleted. This ensures that an HTTP client is -// able to complete an object GET request regardless of concurrent FS writes or HTTP PUTs to that file. -// -// Middleware calls this periodically while producing an object GET response. -func (s *Server) RpcRenewLease(in *RenewLeaseReq, reply *RenewLeaseReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - // This is currently a stub, as there's not yet any idea of a lease, so there's nothing to renew. - return -} - -// Release a lease, allowing a file's log segments to be deleted when necessary. -// -// Middleware calls this once an object GET response is complete. -func (s *Server) RpcReleaseLease(in *ReleaseLeaseReq, reply *ReleaseLeaseReply) (err error) { - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - defer func() { rpcEncodeError(&err) }() // Encode error for return by RPC - - // This is currently a stub, as there's not yet any idea of a lease, so there's nothing to release. - return -} - -// RpcIsAccountBimodal answers the question "is this account known by ProxyFS to be bimodal?". -// -// If bimodal, also indicates the PrivateIPAddr of the Peer ProxyFS instance serving the matching volume -func (s *Server) RpcIsAccountBimodal(in *IsAccountBimodalReq, reply *IsAccountBimodalReply) (err error) { - var ( - ok bool - volumeName string - ) - - enterGate() - defer leaveGate() - - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - - volumeName, reply.IsBimodal = fs.AccountNameToVolumeName(in.AccountName) - - if reply.IsBimodal { - reply.ActivePeerPrivateIPAddr, ok = fs.VolumeNameToActivePeerPrivateIPAddr(volumeName) - if !ok { - err = fmt.Errorf("%v indicated as Bimodal but no matching ActivePeer", volumeName) - logger.ErrorWithError(err) - return - } - } - - err = nil - return -} diff --git a/jrpcfs/middleware_test.go b/jrpcfs/middleware_test.go deleted file mode 100644 index f0242340..00000000 --- a/jrpcfs/middleware_test.go +++ /dev/null @@ -1,2079 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/stretchr/testify/assert" -) - -func middlewareCreateContainer(t *testing.T, server *Server, fullPathContainer string, expectedError blunder.FsError) { - assert := assert.New(t) - - // Create a container for testing - createRequest := CreateContainerRequest{ - VirtPath: fullPathContainer, - } - createResponse := CreateContainerReply{} - err := server.RpcCreateContainer(&createRequest, &createResponse) - assert.True(blunder.Is(err, expectedError)) -} - -func middlewareDeleteObject(server *Server, nameObject string) (err error) { - deleteRequest := DeleteReq{ - VirtPath: testVerAccountContainerName + "/" + nameObject, - } - deleteResponse := DeleteReply{} - err = server.RpcDelete(&deleteRequest, &deleteResponse) - return err -} - -func middlewarePost(server *Server, virtPath string, newMetaData []byte, oldMetaData []byte) (err error) { - PostRequest := MiddlewarePostReq{ - VirtPath: virtPath, - NewMetaData: newMetaData, - OldMetaData: oldMetaData, - } - PostResponse := MiddlewarePostReply{} - err = server.RpcPost(&PostRequest, &PostResponse) - return err -} - -func middlewarePutLocation(t *testing.T, server *Server, newPutPath string, expectedError blunder.FsError) (physPath string) { - assert := assert.New(t) - - putLocationReq := PutLocationReq{ - VirtPath: newPutPath, - } - putLocationReply := PutLocationReply{} - err := server.RpcPutLocation(&putLocationReq, &putLocationReply) - assert.True(blunder.Is(err, expectedError)) - - return putLocationReply.PhysPath -} - -func TestRpcHead(t *testing.T) { - s := &Server{} - - testRpcHeadExistingContainerWithMetadata(t, s) - testRpcHeadExistingContainerWithoutMetadata(t, s) - testRpcHeadAbsentContainer(t, s) - testRpcHeadObjectSymlink(t, s) - testRpcHeadObjectFile(t, s) - testRpcHeadUpdatedObjectFile(t, s) -} - -func testRpcHeadExistingContainerWithMetadata(t *testing.T, server *Server) { - assert := assert.New(t) - - request := HeadReq{ - VirtPath: testVerAccountName + "/" + "c", - } - response := HeadReply{} - err := server.RpcHead(&request, &response) - - assert.Nil(err) - assert.Equal([]byte("metadata for c"), response.Metadata) -} - -func testRpcHeadExistingContainerWithoutMetadata(t *testing.T, server *Server) { - assert := assert.New(t) - - request := HeadReq{ - VirtPath: testVerAccountName + "/" + "c-no-metadata", - } - response := HeadReply{} - err := server.RpcHead(&request, &response) - - assert.Nil(err) - assert.Equal([]byte(""), response.Metadata) -} - -func testRpcHeadAbsentContainer(t *testing.T, server *Server) { - assert := assert.New(t) - - request := HeadReq{ - VirtPath: testVerAccountName + "/" + "sir-not-appearing-in-this-test", - } - response := HeadReply{} - err := server.RpcHead(&request, &response) - - assert.NotNil(err) -} - -func testRpcHeadObjectSymlink(t *testing.T, server *Server) { - assert := assert.New(t) - - request := HeadReq{ - VirtPath: testVerAccountName + "/c/plants-symlink/eggplant.txt-symlink", - } - response := HeadReply{} - err := server.RpcHead(&request, &response) - - assert.Nil(err) - assert.Equal([]byte(""), response.Metadata) - assert.Equal(uint64(12), response.FileSize) - assert.Equal(false, response.IsDir) -} - -func testRpcHeadObjectFile(t *testing.T, server *Server) { - assert := assert.New(t) - - request := HeadReq{ - VirtPath: testVerAccountName + "/c/plants/eggplant.txt", - } - response := HeadReply{} - err := server.RpcHead(&request, &response) - - assert.Nil(err) - assert.Equal([]byte(""), response.Metadata) - assert.Equal(uint64(12), response.FileSize) - assert.Equal(false, response.IsDir) - - statResult := fsStatPath(testVerAccountName, "/c/plants/eggplant.txt") - - assert.Equal(statResult[fs.StatINum], uint64(response.InodeNumber)) - assert.Equal(statResult[fs.StatNumWrites], response.NumWrites) - assert.Equal(statResult[fs.StatMTime], response.ModificationTime) -} - -func testRpcHeadUpdatedObjectFile(t *testing.T, server *Server) { - assert := assert.New(t) - - request := HeadReq{ - VirtPath: testVerAccountName + "/c/README", - } - response := HeadReply{} - err := server.RpcHead(&request, &response) - - assert.Nil(err) - assert.Equal([]byte("metadata for c/README"), response.Metadata) - assert.Equal(uint64(37), response.FileSize) - assert.Equal(false, response.IsDir) - - statResult := fsStatPath(testVerAccountName, "/c/README") - - assert.Equal(statResult[fs.StatINum], uint64(response.InodeNumber)) - assert.Equal(statResult[fs.StatNumWrites], response.NumWrites) - // We've got different CTime and MTime, since we POSTed after writing - assert.True(statResult[fs.StatCTime] > statResult[fs.StatMTime], "Expected StatCTime (%v) > StatMTime (%v)", statResult[fs.StatCTime], statResult[fs.StatMTime]) - assert.Equal(statResult[fs.StatMTime], response.ModificationTime) - assert.Equal(statResult[fs.StatCTime], response.AttrChangeTime) -} - -func TestRpcGetContainerMetadata(t *testing.T) { - server := &Server{} - assert := assert.New(t) - - // Just get one entry; this test really only cares about the - // metadata - request := GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c", - Marker: "", - EndMarker: "", - MaxEntries: 1, - } - response := GetContainerReply{} - err := server.RpcGetContainer(&request, &response) - assert.Nil(err) - assert.Equal([]byte("metadata for c"), response.Metadata) - - statResult := fsStatPath(testVerAccountName, "c") - assert.Equal(statResult[fs.StatMTime], response.ModificationTime) -} - -func TestRpcGetContainerNested(t *testing.T) { - server := &Server{} - assert := assert.New(t) - - // Get a container listing with a limit of fewer than the total number - request := GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-nested", - Marker: "", - EndMarker: "", - MaxEntries: 10000, - } - response := GetContainerReply{} - err := server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(31, len(response.ContainerEntries)) - ents := response.ContainerEntries - assert.Equal(".DS_Store", ents[0].Basename) - assert.Equal(".git", ents[1].Basename) - assert.Equal(".git/.DS_Store", ents[2].Basename) - assert.Equal(".git/COMMIT_EDITMSG", ents[3].Basename) - assert.Equal(".git/FETCH_HEAD", ents[4].Basename) - assert.Equal(".git/HEAD", ents[5].Basename) - assert.Equal(".git/ORIG_HEAD", ents[6].Basename) - assert.Equal(".git/hooks", ents[7].Basename) - assert.Equal(".git/hooks/.DS_Store", ents[8].Basename) - assert.Equal(".git/hooks/applypatch-msg.sample", ents[9].Basename) - assert.Equal(".git/hooks/commit-msg.sample", ents[10].Basename) - assert.Equal(".git/index", ents[11].Basename) - assert.Equal(".git/logs", ents[12].Basename) - assert.Equal(".git/logs/.DS_Store", ents[13].Basename) - assert.Equal(".git/logs/HEAD", ents[14].Basename) - assert.Equal(".git/logs/refs", ents[15].Basename) - assert.Equal(".git/logs/refs/.DS_Store", ents[16].Basename) - assert.Equal(".git/logs/refs/heads", ents[17].Basename) - assert.Equal(".git/logs/refs/heads/.DS_Store", ents[18].Basename) - assert.Equal(".git/logs/refs/heads/development", ents[19].Basename) - assert.Equal(".git/logs/refs/heads/stable", ents[20].Basename) - assert.Equal(".git/logs/refs/stash", ents[21].Basename) - if testRequireSlashesInPathsToProperlySort { - assert.Equal("a", ents[22].Basename) - assert.Equal("a/b", ents[23].Basename) - assert.Equal("a/b-1", ents[24].Basename) - assert.Equal("a/b-2", ents[25].Basename) - assert.Equal("a/b/c", ents[26].Basename) - assert.Equal("a/b/c-1", ents[27].Basename) - assert.Equal("a/b/c-2", ents[28].Basename) - assert.Equal("a/b/c/d-1", ents[29].Basename) - assert.Equal("a/b/c/d-2", ents[30].Basename) - } else { - assert.Equal("a", ents[22].Basename) - assert.Equal("a/b", ents[23].Basename) - assert.Equal("a/b/c", ents[24].Basename) - assert.Equal("a/b/c/d-1", ents[25].Basename) - assert.Equal("a/b/c/d-2", ents[26].Basename) - assert.Equal("a/b/c-1", ents[27].Basename) - assert.Equal("a/b/c-2", ents[28].Basename) - assert.Equal("a/b-1", ents[29].Basename) - assert.Equal("a/b-2", ents[30].Basename) - } -} - -func TestRpcGetContainerPrefix(t *testing.T) { - server := &Server{} - assert := assert.New(t) - - request := GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-nested", - Marker: "", - EndMarker: "", - MaxEntries: 10000, - Prefix: ".git/logs/refs/", - } - response := GetContainerReply{} - err := server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(6, len(response.ContainerEntries)) - ents := response.ContainerEntries - assert.Equal(".git/logs/refs/.DS_Store", ents[0].Basename) - assert.Equal(".git/logs/refs/heads", ents[1].Basename) - assert.Equal(".git/logs/refs/heads/.DS_Store", ents[2].Basename) - assert.Equal(".git/logs/refs/heads/development", ents[3].Basename) - assert.Equal(".git/logs/refs/heads/stable", ents[4].Basename) - assert.Equal(".git/logs/refs/stash", ents[5].Basename) - - // Try with a prefix that starts mid-filename - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-nested", - Marker: "", - EndMarker: "", - MaxEntries: 10000, - Prefix: ".git/logs/re", - } - response = GetContainerReply{} - err = server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(7, len(response.ContainerEntries)) - ents = response.ContainerEntries - assert.Equal(".git/logs/refs", ents[0].Basename) - assert.Equal(".git/logs/refs/.DS_Store", ents[1].Basename) - assert.Equal(".git/logs/refs/heads", ents[2].Basename) - assert.Equal(".git/logs/refs/heads/.DS_Store", ents[3].Basename) - assert.Equal(".git/logs/refs/heads/development", ents[4].Basename) - assert.Equal(".git/logs/refs/heads/stable", ents[5].Basename) - assert.Equal(".git/logs/refs/stash", ents[6].Basename) - - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-nested", - Marker: "", - EndMarker: "", - MaxEntries: 10000, - Prefix: "a/b/", - } - response = GetContainerReply{} - err = server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(5, len(response.ContainerEntries)) - ents = response.ContainerEntries - if testRequireSlashesInPathsToProperlySort { - assert.Equal("a/b/c", ents[0].Basename) - assert.Equal("a/b/c-1", ents[1].Basename) - assert.Equal("a/b/c-2", ents[2].Basename) - assert.Equal("a/b/c/d-1", ents[3].Basename) - assert.Equal("a/b/c/d-2", ents[4].Basename) - } else { - assert.Equal("a/b/c", ents[0].Basename) - assert.Equal("a/b/c/d-1", ents[1].Basename) - assert.Equal("a/b/c/d-2", ents[2].Basename) - assert.Equal("a/b/c-1", ents[3].Basename) - assert.Equal("a/b/c-2", ents[4].Basename) - } -} - -func TestRpcGetContainerPrefixAndMarkers(t *testing.T) { - server := &Server{} - assert := assert.New(t) - - request := GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-nested", - Marker: ".git/logs/refs/heads", - EndMarker: "", - MaxEntries: 10000, - Prefix: ".git/logs/refs/", - } - response := GetContainerReply{} - err := server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(4, len(response.ContainerEntries)) - ents := response.ContainerEntries - assert.Equal(".git/logs/refs/heads/.DS_Store", ents[0].Basename) - assert.Equal(".git/logs/refs/heads/development", ents[1].Basename) - assert.Equal(".git/logs/refs/heads/stable", ents[2].Basename) - assert.Equal(".git/logs/refs/stash", ents[3].Basename) - - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-nested", - Marker: ".git/logs/refs/heads", - EndMarker: ".git/logs/refs/heads/stable", - MaxEntries: 10000, - Prefix: ".git/logs/refs/", - } - response = GetContainerReply{} - err = server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(2, len(response.ContainerEntries)) - ents = response.ContainerEntries - assert.Equal(".git/logs/refs/heads/.DS_Store", ents[0].Basename) - assert.Equal(".git/logs/refs/heads/development", ents[1].Basename) -} - -func TestRpcGetContainerPrefixAndDelimiter(t *testing.T) { - server := &Server{} - assert := assert.New(t) - - request := GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-nested", - Marker: "", - EndMarker: "", - MaxEntries: 10000, - Prefix: ".git/logs/refs/", - Delimiter: "/", - } - response := GetContainerReply{} - err := server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(3, len(response.ContainerEntries)) - ents := response.ContainerEntries - assert.Equal(".git/logs/refs/.DS_Store", ents[0].Basename) - assert.Equal(".git/logs/refs/heads", ents[1].Basename) - assert.Equal(".git/logs/refs/stash", ents[2].Basename) - - // Try with a prefix without a trailing slash - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-nested", - Marker: "", - EndMarker: "", - MaxEntries: 10000, - Prefix: ".git/logs/refs", - Delimiter: "/", - } - response = GetContainerReply{} - err = server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(1, len(response.ContainerEntries)) - ents = response.ContainerEntries - assert.Equal(".git/logs/refs", ents[0].Basename) -} - -func TestRpcGetContainerPaginated(t *testing.T) { - server := &Server{} - assert := assert.New(t) - - // Get a container listing with a limit of fewer than the total number - request := GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c", - Marker: "", - EndMarker: "", - MaxEntries: 5, - } - response := GetContainerReply{} - err := server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(5, len(response.ContainerEntries)) - ents := response.ContainerEntries - - // These values are set in the test setup. - assert.Equal("README", ents[0].Basename) - assert.Equal(uint64(37), ents[0].FileSize) - assert.Equal(false, ents[0].IsDir) - assert.Equal([]byte("metadata for c/README"), ents[0].Metadata) - - assert.Equal("animals", ents[1].Basename) - assert.Equal(uint64(0), ents[1].FileSize) - assert.Equal(true, ents[1].IsDir) - - assert.Equal("animals/bird.txt", ents[2].Basename) - assert.Equal(uint64(15), ents[2].FileSize) - assert.Equal(false, ents[2].IsDir) - - assert.Equal("animals/cat.txt", ents[3].Basename) - assert.Equal(uint64(13), ents[3].FileSize) - assert.Equal(false, ents[3].IsDir) - - assert.Equal("animals/cow.txt", ents[4].Basename) - assert.Equal(uint64(12), ents[4].FileSize) - assert.Equal(false, ents[4].IsDir) - - // We'll spot-check two files and one directory - statResult := fsStatPath(testVerAccountName, "c/README") - // We've got different CTime and MTime, since we POSTed after writing - assert.True(statResult[fs.StatCTime] > statResult[fs.StatMTime], "Expected StatCTime (%v) > StatMTime (%v)", statResult[fs.StatCTime], statResult[fs.StatMTime]) - assert.Equal(statResult[fs.StatMTime], ents[0].ModificationTime) - assert.Equal(statResult[fs.StatCTime], ents[0].AttrChangeTime) - assert.Equal(statResult[fs.StatNumWrites], ents[0].NumWrites) - assert.Equal(statResult[fs.StatINum], ents[0].InodeNumber) - - statResult = fsStatPath(testVerAccountName, "c/animals/cat.txt") - assert.Equal(statResult[fs.StatMTime], ents[3].ModificationTime) - assert.Equal(statResult[fs.StatNumWrites], ents[3].NumWrites) - assert.Equal(statResult[fs.StatINum], ents[3].InodeNumber) - - statResult = fsStatPath(testVerAccountName, "c/animals") - assert.Equal(statResult[fs.StatMTime], ents[1].ModificationTime) - assert.Equal(statResult[fs.StatNumWrites], ents[1].NumWrites) - assert.Equal(statResult[fs.StatINum], ents[1].InodeNumber) - - // Next page of results: - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c", - Marker: "animals/cow.txt", - EndMarker: "", - MaxEntries: 5, - } - response = GetContainerReply{} - err = server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(5, len(response.ContainerEntries)) - - ents = response.ContainerEntries - assert.Equal("animals/dog.txt", ents[0].Basename) - assert.Equal("animals/elephant.txt", ents[1].Basename) - assert.Equal("animals/frog.txt", ents[2].Basename) - assert.Equal("animals/mouse.txt", ents[3].Basename) - assert.Equal("empty-directory", ents[4].Basename) - - // Last page: it's shorter than 10 results, but that shouldn't - // break anything. - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c", - Marker: "empty-directory", - EndMarker: "", - MaxEntries: 10, - } - response = GetContainerReply{} - err = server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(8, len(response.ContainerEntries)) - - ents = response.ContainerEntries - if testRequireSlashesInPathsToProperlySort { - assert.Equal("plants", ents[0].Basename) - assert.Equal("plants-README", ents[1].Basename) - assert.Equal("plants-symlink", ents[2].Basename) - assert.Equal("plants/aloe.txt", ents[3].Basename) - assert.Equal("plants/banana.txt", ents[4].Basename) - assert.Equal("plants/cherry.txt", ents[5].Basename) - assert.Equal("plants/eggplant.txt", ents[6].Basename) - assert.Equal("plants/eggplant.txt-symlink", ents[7].Basename) - } else { - assert.Equal("plants", ents[0].Basename) - assert.Equal("plants/aloe.txt", ents[1].Basename) - assert.Equal("plants/banana.txt", ents[2].Basename) - assert.Equal("plants/cherry.txt", ents[3].Basename) - assert.Equal("plants/eggplant.txt", ents[4].Basename) - assert.Equal("plants/eggplant.txt-symlink", ents[5].Basename) - assert.Equal("plants-README", ents[6].Basename) - assert.Equal("plants-symlink", ents[7].Basename) - } - - // Some Swift clients keep asking for container listings until - // they see an empty page, which will result in RpcGetContainer - // being called with a marker equal to the last object. This - // should simply return 0 results. - if testRequireSlashesInPathsToProperlySort { - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c", - Marker: "plants/eggplant.txt-symlink", - EndMarker: "", - MaxEntries: 5, - } - } else { - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c", - Marker: "plants-symlink", - EndMarker: "", - MaxEntries: 5, - } - } - response = GetContainerReply{} - err = server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(0, len(response.ContainerEntries)) - - // If a client sends a marker that comes _after_ every object, - // that should also return zero results. - request = GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c", - Marker: "zzzzzzzzzzzzzz", - EndMarker: "", - MaxEntries: 5, - } - response = GetContainerReply{} - err = server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(0, len(response.ContainerEntries)) -} - -func TestRpcGetContainerZeroLimit(t *testing.T) { - server := &Server{} - assert := assert.New(t) - - request := GetContainerReq{ - VirtPath: "/v1/AN_account/c", - Marker: "", - EndMarker: "", - MaxEntries: 0, - } - response := GetContainerReply{} - err := server.RpcGetContainer(&request, &response) - - assert.Nil(err) - assert.Equal(len(response.ContainerEntries), 0) -} - -func TestRpcGetContainerSymlink(t *testing.T) { - server := &Server{} - assert := assert.New(t) - - request := GetContainerReq{ - VirtPath: testVerAccountName + "/" + "c-symlink", - Marker: "", - EndMarker: "", - MaxEntries: 3, // 1 - } - response := GetContainerReply{} - err := server.RpcGetContainer(&request, &response) - - // Note: We are not supporting GetContainer *thru* a symlink - // In other words, the Container itself cannot be a SymlinkInode - // This aligns with GetAccount which will only return DirInode dir_entry's - - assert.NotNil(err) - assert.Equal(fmt.Sprintf("errno: %d", blunder.NotFoundError), err.Error()) -} - -func TestRpcGetAccount(t *testing.T) { - assert := assert.New(t) - server := &Server{} - - request := GetAccountReq{ - VirtPath: "/v1/" + testAccountName2, - Marker: "", - EndMarker: "", - MaxEntries: 5, - } - response := GetAccountReply{} - err := server.RpcGetAccount(&request, &response) - - statResult := fsStatPath("/v1/"+testAccountName2, "/") - assert.Equal(statResult[fs.StatMTime], response.ModificationTime) - - assert.Nil(err) - assert.Equal(5, len(response.AccountEntries)) - assert.Equal("alpha", response.AccountEntries[0].Basename) - statResult = fsStatPath("/v1/"+testAccountName2, "/alpha") - assert.Equal(statResult[fs.StatMTime], response.AccountEntries[0].ModificationTime) - assert.Equal(statResult[fs.StatCTime], response.AccountEntries[0].AttrChangeTime) - - assert.Equal("bravo", response.AccountEntries[1].Basename) - assert.Equal("charlie", response.AccountEntries[2].Basename) - assert.Equal("delta", response.AccountEntries[3].Basename) - assert.Equal("echo", response.AccountEntries[4].Basename) - - // Marker query starts listing in the middle - request = GetAccountReq{ - VirtPath: "/v1/" + testAccountName2, - Marker: "lima", - EndMarker: "", - MaxEntries: 3, - } - response = GetAccountReply{} - err = server.RpcGetAccount(&request, &response) - - assert.Nil(err) - assert.Equal(3, len(response.AccountEntries)) - assert.Equal("mancy", response.AccountEntries[0].Basename) - assert.Equal("november", response.AccountEntries[1].Basename) - assert.Equal("oscar", response.AccountEntries[2].Basename) - - // EndMarker query can cap results ahead of MaxEntries - request = GetAccountReq{ - VirtPath: "/v1/" + testAccountName2, - Marker: "lima", - EndMarker: "oscar", - MaxEntries: 3, - } - response = GetAccountReply{} - err = server.RpcGetAccount(&request, &response) - - assert.Nil(err) - assert.Equal(2, len(response.AccountEntries)) - assert.Equal("mancy", response.AccountEntries[0].Basename) - assert.Equal("november", response.AccountEntries[1].Basename) - - // Asking past the end is not an error, just empty - request = GetAccountReq{ - VirtPath: "/v1/" + testAccountName2, - Marker: "zulu", - EndMarker: "", - MaxEntries: 3, - } - response = GetAccountReply{} - err = server.RpcGetAccount(&request, &response) - - assert.Nil(err) - assert.Equal(0, len(response.AccountEntries)) -} - -func TestRpcBasicApi(t *testing.T) { - s := &Server{} - - testRpcDelete(t, s) - testRpcPost(t, s) - testNameLength(t, s) -} - -func testRpcDelete(t *testing.T, server *Server) { - assert := assert.New(t) - - middlewareCreateContainer(t, server, testVerAccountContainerName, blunder.SuccessError) - - // Create an object which is a directory and see if we can delete it via bimodal. - _, _, _, _, volumeHandle, err := parseVirtPath(testVerAccountName) - assert.Nil(err) - - cInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testContainerName) - assert.Nil(err) - - var emptyDir string = "empty-directory" - _ = fsMkDir(volumeHandle, cInode, emptyDir) - - err = middlewareDeleteObject(server, emptyDir) - assert.Nil(err) - - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, emptyDir) - assert.NotNil(err) - - // Now create an object which is a file and see if we can delete it via bimodal. - var emptyFile string = "empty-file" - _ = fsCreateFile(volumeHandle, cInode, emptyFile) - - err = middlewareDeleteObject(server, emptyFile) - assert.Nil(err) - - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, emptyFile) - assert.NotNil(err) - - // Now create a directory with one file in it and prove we can remove file and - // then directory. - var aDir string = "dir1" - aDirInode := fsMkDir(volumeHandle, cInode, aDir) - _ = fsCreateFile(volumeHandle, aDirInode, emptyFile) - - err = middlewareDeleteObject(server, aDir+"/"+emptyFile) - assert.Nil(err) - - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, "/"+aDir+"/"+emptyFile) - assert.NotNil(err) - - err = middlewareDeleteObject(server, aDir) - assert.Nil(err) - - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, "/"+aDir) - assert.NotNil(err) - - // Now delete the container - deleteRequest := DeleteReq{ - VirtPath: testVerAccountContainerName, - } - deleteResponse := DeleteReply{} - err = server.RpcDelete(&deleteRequest, &deleteResponse) - assert.Nil(err) - - // Put it back to be nice to other test cases - middlewareCreateContainer(t, server, testVerAccountContainerName, blunder.SuccessError) -} - -func TestRpcDeleteSymlinks(t *testing.T) { - s := &Server{} - assert := assert.New(t) - - containerName := "unmaniac-imparticipable" - - // Test setup: - // Within our container, we've got the following: - // - // top-level.txt - // d1 - // d1/snap - // d1/snap-symlink -> snap - // d1/crackle - // d1/pop - // d1-symlink -> d1 - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - - tlInode := fsCreateFile(volumeHandle, containerInode, "top-level.txt") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, tlInode, 0, []byte("conusance-callboy"), nil) - - d1Inode := fsMkDir(volumeHandle, containerInode, "d1") - files := map[string]string{ - "snap": "contents of snap", - "crackle": "contents of crackle", - "pop": "contents of pop", - } - for fileName, fileContents := range files { - fileInode := fsCreateFile(volumeHandle, d1Inode, fileName) - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileContents), nil) - if err != nil { - panic(fmt.Sprintf("failed to write file %s: %v", fileName, err)) - } - } - - fsCreateSymlink(volumeHandle, containerInode, "d1-symlink", "d1") - fsCreateSymlink(volumeHandle, d1Inode, "snap-symlink", "snap") - fsCreateSymlink(volumeHandle, d1Inode, "pop-symlink", "./pop") - - fsCreateSymlink(volumeHandle, d1Inode, "dot-symlink", ".") - - // Symlinks in the directory portion of the name are followed - deleteRequest := DeleteReq{ - VirtPath: testVerAccountName + "/" + containerName + "/d1-symlink/crackle", - } - deleteResponse := DeleteReply{} - err = s.RpcDelete(&deleteRequest, &deleteResponse) - assert.Nil(err) - - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/d1/crackle") - assert.NotNil(err) - assert.True(blunder.Is(err, blunder.NotFoundError)) - - // Symlinks in the file portion of the name (i.e. the last - // segment) are not followed - deleteRequest = DeleteReq{ - VirtPath: testVerAccountName + "/" + containerName + "/d1-symlink/snap-symlink", - } - deleteResponse = DeleteReply{} - err = s.RpcDelete(&deleteRequest, &deleteResponse) - assert.Nil(err) - - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/d1/snap") - assert.Nil(err) - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/d1/snap-symlink") - assert.NotNil(err) - assert.True(blunder.Is(err, blunder.NotFoundError)) - - // Symlinks ending with "." don't cause problems - deleteRequest = DeleteReq{ - VirtPath: testVerAccountName + "/" + containerName + "/d1-symlink/dot-symlink/pop-symlink", - } - deleteResponse = DeleteReply{} - err = s.RpcDelete(&deleteRequest, &deleteResponse) - assert.Nil(err) - - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/d1/pop") - assert.Nil(err) - _, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/d1/pop-symlink") - assert.NotNil(err) - assert.True(blunder.Is(err, blunder.NotFoundError)) -} - -func testRpcPost(t *testing.T, server *Server) { - assert := assert.New(t) - - // We assume that the container already exists since currently we cannot - // delete the container. - - _, _, _, _, volumeHandle, err := parseVirtPath(testVerAccountName) - assert.Nil(err) - - // POST to account with empty string for account - var virtPath string = testVer - newContMetaData := []byte("account metadata") - oldContMetaData := []byte("") - err = middlewarePost(server, virtPath, newContMetaData, oldContMetaData) - assert.True(blunder.Is(err, blunder.AccountNotModifiable)) - - // POST to account - virtPath = testVerAccountName - newContMetaData = []byte("account metadata") - oldContMetaData = []byte("") - err = middlewarePost(server, virtPath, newContMetaData, oldContMetaData) - assert.True(blunder.Is(err, blunder.AccountNotModifiable)) - - // POST to account/container - virtPath = testVerAccountContainerName - newContMetaData = []byte("container metadata") - oldContMetaData = []byte("") - err = middlewarePost(server, virtPath, newContMetaData, oldContMetaData) - assert.Nil(err) - - // Try POST again with garbage metadata and make sure receive an error - virtPath = testVerAccountContainerName - newContMetaData = []byte("container metadata") - oldContMetaData = []byte("incorrect metadata") - err = middlewarePost(server, virtPath, newContMetaData, oldContMetaData) - assert.True(blunder.Is(err, blunder.OldMetaDataDifferent)) - - // Try POST one more time with valid version of old metadata and make sure no error. - virtPath = testVerAccountContainerName - newContMetaData = []byte("container metadata with more stuff") - oldContMetaData = []byte("container metadata") - err = middlewarePost(server, virtPath, newContMetaData, oldContMetaData) - assert.Nil(err) - - // Now POST to account/container/object after creating an object which - // is a directory and one which is a file. - cInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, testContainerName) - assert.Nil(err) - var emptyDir string = "empty-directory" - _ = fsMkDir(volumeHandle, cInode, emptyDir) - var emptyFile string = "empty-file" - _ = fsCreateFile(volumeHandle, cInode, emptyFile) - var emptyFileSymlink string = "empty-file-symlink" - fsCreateSymlink(volumeHandle, cInode, emptyFileSymlink, emptyFile) - - virtPath = testVerAccountContainerName + "/" + emptyDir - newContMetaData = []byte("object emptyDir metadata") - oldContMetaData = []byte("") - err = middlewarePost(server, virtPath, newContMetaData, oldContMetaData) - assert.Nil(err) - - virtPath = testVerAccountContainerName + "/" + emptyFile - newContMetaData = []byte("object emptyFile metadata") - oldContMetaData = []byte("") - err = middlewarePost(server, virtPath, newContMetaData, oldContMetaData) - assert.Nil(err) - - // POST to a symlink follows it - virtPath = testVerAccountContainerName + "/" + emptyFileSymlink - oldContMetaData = newContMetaData - newContMetaData = []byte("object emptyFile metadata take 2") - err = middlewarePost(server, virtPath, newContMetaData, oldContMetaData) - assert.Nil(err) - - headResponse, err := volumeHandle.MiddlewareHeadResponse(testContainerName + "/" + emptyFile) - assert.Nil(err) - assert.Equal(newContMetaData, headResponse.Metadata) - - // Cleanup objects - err = middlewareDeleteObject(server, emptyDir) - assert.Nil(err) - err = middlewareDeleteObject(server, emptyFile) - assert.Nil(err) -} - -func testNameLength(t *testing.T, server *Server) { - // Try to create a container with a name which is one larger than fs.FilePathMax - tooLongOfAString := make([]byte, (fs.FilePathMax + 1)) - for i := 0; i < (fs.FilePathMax + 1); i++ { - tooLongOfAString[i] = 'A' - } - tooLongOfAPathName := testVerAccountName + "/" + string(tooLongOfAString) - middlewareCreateContainer(t, server, tooLongOfAPathName, blunder.NameTooLongError) - - // Now try to create an objectName which is too long - tooLongOfAFileName := make([]byte, (fs.FileNameMax + 1)) - for i := 0; i < (fs.FileNameMax + 1); i++ { - tooLongOfAFileName[i] = 'A' - } - longFileName := testVerAccountContainerName + "/" + string(tooLongOfAFileName) - - _ = middlewarePutLocation(t, server, longFileName, blunder.NameTooLongError) -} - -// Tests for RpcPutLocation and RpcPutComplete together; an object PUT -// calls both -func testPutObjectSetup(t *testing.T) (*assert.Assertions, *Server, string, fs.VolumeHandle) { - // Just some common setup crud - - // We can't delete containers, so we grab a name and hope that it - // doesn't already exist. (We're using ramswift for tests, so it's - // almost certainly okay.) - containerName := fmt.Sprintf("mware-TestPutObject-%d", time.Now().UnixNano()) - - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - - assert := assert.New(t) - - server := &Server{} - - return assert, server, containerName, volumeHandle -} - -// Helper function to put a file into Swift using RpcPutLocation / RpcPutComplete plus an HTTP PUT request -func putFileInSwift(server *Server, virtPath string, objData []byte, objMetadata []byte) error { - - // Ask where to put it - putLocationReq := PutLocationReq{ - VirtPath: virtPath, - } - putLocationResp := PutLocationReply{} - - err := server.RpcPutLocation(&putLocationReq, &putLocationResp) - if err != nil { - return err - } - - // Put it there - pathParts := strings.SplitN(putLocationResp.PhysPath, "/", 5) - // pathParts[0] is empty, pathParts[1] is "v1" - pAccount, pContainer, pObject := pathParts[2], pathParts[3], pathParts[4] - - putContext, err := swiftclient.ObjectFetchChunkedPutContext(pAccount, pContainer, pObject, "") - if err != nil { - return err - } - - err = putContext.SendChunk(objData) - if err != nil { - return err - } - - err = putContext.Close() - if err != nil { - return err - } - - // Tell proxyfs about it - putCompleteReq := PutCompleteReq{ - VirtPath: virtPath, - PhysPaths: []string{putLocationResp.PhysPath}, - PhysLengths: []uint64{uint64(len(objData))}, - Metadata: objMetadata, - } - putCompleteResp := PutCompleteReply{} - - err = server.RpcPutComplete(&putCompleteReq, &putCompleteResp) - if err != nil { - return err - } - return nil -} - -func TestPutObjectSimple(t *testing.T) { - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - objName := "toplevel.bin" - objData := []byte("hello world\n") - objMetadata := []byte("{\"metadata for\": \"" + objName + "\"}") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - err := putFileInSwift(server, objVirtPath, objData, objMetadata) - assert.Nil(err) // sanity check - - // The file should exist now, so we can verify its attributes - theInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/"+objName) - assert.Nil(err) - contents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, theInode, 0, 99999, nil) - assert.Nil(err) - assert.Equal(objData, contents) - - headResponse, err := volumeHandle.MiddlewareHeadResponse(containerName + "/" + objName) - assert.Nil(err) - assert.Equal([]byte(objMetadata), headResponse.Metadata) -} - -func TestPutObjectInAllNewSubdirs(t *testing.T) { - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - objName := "d1/d2/d3/d4/nested.bin" - objData := []byte("hello nested world\n") - objMetadata := []byte("nested metadata") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - err := putFileInSwift(server, objVirtPath, objData, objMetadata) - assert.Nil(err) // sanity check - - // The file should exist now, so we can verify its attributes - theInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/"+objName) - assert.Nil(err) - contents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, theInode, 0, 99999, nil) - assert.Nil(err) - assert.Equal(objData, contents) -} - -func TestPutObjectInSomeNewSubdirs(t *testing.T) { - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - // make d1 and d1/d2, but leave creation of the rest to the RPC call - containerInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName) - if err != nil { - panic(err) - } - d1Inode := fsMkDir(volumeHandle, containerInode, "exists-d1") - _ = fsMkDir(volumeHandle, d1Inode, "exists-d2") - - objName := "exists-d1/exists-d2/d3/d4/nested.bin" - objData := []byte("hello nested world\n") - objMetadata := []byte("nested metadata") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - err = putFileInSwift(server, objVirtPath, objData, objMetadata) - assert.Nil(err) // sanity check - - // The file should exist now, so we can verify its attributes - theInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/"+objName) - assert.Nil(err) - contents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, theInode, 0, 99999, nil) - assert.Nil(err) - assert.Equal(objData, contents) -} - -func TestPutObjectOverwriteFile(t *testing.T) { - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - objName := "overwritten.bin" - objData1 := []byte("hello world 1\n") - objData2 := []byte("hello world 2\n") - objMetadata := []byte("{\"metadata for\": \"" + objName + "\"}") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - err := putFileInSwift(server, objVirtPath, objData1, objMetadata) - assert.Nil(err) // sanity check - err = putFileInSwift(server, objVirtPath, objData2, objMetadata) - assert.Nil(err) - - theInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/"+objName) - assert.Nil(err) - contents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, theInode, 0, 99999, nil) - assert.Nil(err) - assert.Equal(objData2, contents) -} - -func TestPutObjectOverwriteDirectory(t *testing.T) { - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - objName := "dir-with-stuff-in-it" - objData := []byte("irrelevant") - objMetadata := []byte("won't get written") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - containerInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName) - if err != nil { - panic(err) - } - dirInodeNumber := fsMkDir(volumeHandle, containerInode, "dir-with-stuff-in-it") - - fileInodeNumber := fsCreateFile(volumeHandle, dirInodeNumber, "stuff.txt") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInodeNumber, 0, []byte("churches, lead, small rocks, apples"), nil) - if err != nil { - panic(err) - } - - err = putFileInSwift(server, objVirtPath, objData, objMetadata) - assert.NotNil(err) - assert.Equal(fmt.Sprintf("errno: %d", blunder.NotEmptyError), err.Error()) - - // remove the file in the directory and the put should succeed, - // replacing the directory with a file - err = volumeHandle.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - dirInodeNumber, "stuff.txt") - assert.Nil(err) - - err = putFileInSwift(server, objVirtPath, objData, objMetadata) - assert.Nil(err) - - dirInodeNumber, err = volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, - containerName+"/"+objName) - assert.Nil(err) - - statResult, err := volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, dirInodeNumber) - assert.Nil(err) - assert.Equal(statResult[fs.StatFType], uint64(inode.FileType)) -} - -func TestPutObjectSymlinkedDir(t *testing.T) { - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - containerInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName) - if err != nil { - panic(err) - } - d1Inode := fsMkDir(volumeHandle, containerInode, "d1") - d2Inode := fsMkDir(volumeHandle, d1Inode, "d2") - fsCreateSymlink(volumeHandle, d1Inode, "d2-symlink", "./d2") - fsCreateSymlink(volumeHandle, d1Inode, "dot-symlink", ".") - fsCreateSymlink(volumeHandle, d2Inode, "abs-container-symlink", "/"+containerName) - - objName := "d1/d2-symlink/abs-container-symlink/d1/dot-symlink/dot-symlink/d2/d3/thing.dat" - objData := []byte("kamik-defensory") - objMetadata := []byte("{}") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - err = putFileInSwift(server, objVirtPath, objData, objMetadata) - assert.Nil(err) // sanity check - - // The file should exist now, so we can verify its attributes - theInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/"+"d1/d2/d3/thing.dat") - assert.Nil(err) - contents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, theInode, 0, 99999, nil) - assert.Nil(err) - assert.Equal(objData, contents) -} - -func TestPutObjectOverwriteSymlink(t *testing.T) { - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - containerInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName) - if err != nil { - panic(err) - } - fsCreateSymlink(volumeHandle, containerInode, "thing.dat", "somewhere-else") - - objName := "thing.dat" - objData := []byte("cottontop-aleuroscope") - objMetadata := []byte("{}") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - err = putFileInSwift(server, objVirtPath, objData, objMetadata) - assert.Nil(err) // sanity check - - // The file should exist now, so we can verify its attributes - theInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/"+"somewhere-else") - assert.Nil(err) - contents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, theInode, 0, 99999, nil) - assert.Nil(err) - assert.Equal(objData, contents) -} - -func TestPutObjectFileInDirPath(t *testing.T) { - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - objName := "d1/d2/actually-a-file/d3/d4/stuff.txt" - objData := []byte("irrelevant") - objMetadata := []byte("won't get written") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - containerInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName) - if err != nil { - panic(err) - } - d1InodeNumber := fsMkDir(volumeHandle, containerInode, "d1") - d2InodeNumber := fsMkDir(volumeHandle, d1InodeNumber, "d2") - - fileInodeNumber := fsCreateFile(volumeHandle, d2InodeNumber, "actually-a-file") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInodeNumber, 0, []byte("not a directory"), nil) - if err != nil { - panic(err) - } - - err = putFileInSwift(server, objVirtPath, objData, objMetadata) - assert.NotNil(err) - assert.Equal(fmt.Sprintf("errno: %d", blunder.PermDeniedError), err.Error()) -} - -func TestPutObjectCompound(t *testing.T) { - // In this test, we put data into two different log segments, but - // the data is for the same file - assert, server, containerName, volumeHandle := testPutObjectSetup(t) - - objName := "helloworld.txt" - objMetadata := []byte("{}") - objVirtPath := testVerAccountName + "/" + containerName + "/" + objName - - /// - var physPaths []string - var physLengths []uint64 - - // Put the first half - putLocationReq := PutLocationReq{ - VirtPath: objVirtPath, - } - putLocationResp := PutLocationReply{} - - err := server.RpcPutLocation(&putLocationReq, &putLocationResp) - if err != nil { - panic(err) - } - - physPaths = append(physPaths, putLocationResp.PhysPath) - physLengths = append(physLengths, uint64(6)) - pathParts := strings.SplitN(putLocationResp.PhysPath, "/", 5) - // pathParts[0] is empty, pathParts[1] is "v1" - pAccount, pContainer, pObject := pathParts[2], pathParts[3], pathParts[4] - - putContext, err := swiftclient.ObjectFetchChunkedPutContext(pAccount, pContainer, pObject, "") - if err != nil { - panic(err) - } - - err = putContext.SendChunk([]byte("hello ")) - if err != nil { - panic(err) - } - - err = putContext.Close() - if err != nil { - panic(err) - } - - // Put the second half - putLocationReq = PutLocationReq{ - VirtPath: objVirtPath, - } - putLocationResp = PutLocationReply{} - - err = server.RpcPutLocation(&putLocationReq, &putLocationResp) - if err != nil { - panic(err) - } - - physPaths = append(physPaths, putLocationResp.PhysPath) - physLengths = append(physLengths, uint64(6)) - pathParts = strings.SplitN(putLocationResp.PhysPath, "/", 5) - pAccount, pContainer, pObject = pathParts[2], pathParts[3], pathParts[4] - - putContext, err = swiftclient.ObjectFetchChunkedPutContext(pAccount, pContainer, pObject, "") - if err != nil { - panic(err) - } - - err = putContext.SendChunk([]byte("world!")) - if err != nil { - panic(err) - } - - err = putContext.Close() - if err != nil { - panic(err) - } - - // Tell proxyfs about it - putCompleteReq := PutCompleteReq{ - VirtPath: objVirtPath, - PhysPaths: physPaths, - PhysLengths: physLengths, - Metadata: objMetadata, - } - putCompleteResp := PutCompleteReply{} - - err = server.RpcPutComplete(&putCompleteReq, &putCompleteResp) - assert.Nil(err) - if err != nil { - panic(err) - } - - // The file should exist now, so we can verify its attributes - theInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/"+objName) - assert.Nil(err) - contents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, theInode, 0, 99999, nil) - assert.Nil(err) - assert.Equal([]byte("hello world!"), contents) - assert.Equal(uint64(theInode), uint64(putCompleteResp.InodeNumber)) - // 2 is the number of log segments we wrote - assert.Equal(uint64(2), putCompleteResp.NumWrites) - - headResponse, err := volumeHandle.MiddlewareHeadResponse(containerName + "/" + objName) - assert.Nil(err) - assert.Equal([]byte(objMetadata), headResponse.Metadata) - - statResult, err := volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, theInode) - assert.Nil(err) - assert.Equal(statResult[fs.StatMTime], putCompleteResp.ModificationTime) - assert.Equal(statResult[fs.StatCTime], putCompleteResp.AttrChangeTime) -} - -func TestIsAccountBimodal(t *testing.T) { - assert := assert.New(t) - server := Server{} - - request := IsAccountBimodalReq{ - AccountName: testAccountName, - } - response := IsAccountBimodalReply{} - - err := server.RpcIsAccountBimodal(&request, &response) - assert.Nil(err) - assert.True(response.IsBimodal) - - request = IsAccountBimodalReq{ - AccountName: testAccountName + "-adenoacanthoma-preperceptive", - } - response = IsAccountBimodalReply{} - - err = server.RpcIsAccountBimodal(&request, &response) - assert.Nil(err) - assert.False(response.IsBimodal) -} - -func TestRpcGetObjectMetadata(t *testing.T) { - // This tests the other, non-read-plan things returned by RpcGetObject. - // We're not actually going to test any read plans here; that is tested elsewhere. - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "covetingly-ahead" - - cInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - readmeInode := fsCreateFile(volumeHandle, cInode, "README") - - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, readmeInode, 0, []byte("unsurpassably-Rigelian"), nil) - if err != nil { - panic(err) - } - - statResult, err := volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, readmeInode) - if err != nil { - panic(err) - } - - req := GetObjectReq{VirtPath: "/v1/AN_account/" + containerName + "/README"} - reply := GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - assert.Equal(uint64(22), reply.FileSize) - assert.Equal(statResult[fs.StatMTime], reply.ModificationTime) - - // Also go check the modification time for objects that were POSTed to - req = GetObjectReq{VirtPath: testVerAccountName + "/c/README"} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - statResult = fsStatPath(testVerAccountName, "/c/README") - - assert.Equal(uint64(37), reply.FileSize) - assert.Equal(statResult[fs.StatMTime], reply.ModificationTime) - assert.Equal(statResult[fs.StatCTime], reply.AttrChangeTime) -} - -func TestRpcGetObjectSymlinkFollowing(t *testing.T) { - // This tests the symlink-following abilities of RpcGetObject. - // We're not actually going to test any read plans here; that is tested elsewhere. - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - // Our filesystem: - // - // / - // /c1 - // /c1/kitten.png - // /c1/symlink-1 -> kitten.png - // /c1/symlink-2 -> symlink-1 - // /c1/symlink-3 -> symlink-2 - // /c1/symlink-4 -> symlink-3 - // /c1/symlink-5 -> symlink-4 - // /c1/symlink-6 -> symlink-5 - // /c1/symlink-7 -> symlink-6 - // /c1/symlink-8 -> symlink-7 - // /c1/symlink-9 -> symlink-8 - // /c2/10-bytes - // /c2/symlink-10-bytes -> 10-bytes - // /c2/symlink-20-bytes -> /c3/20-bytes - // /c2/symlink-20-bytes-indirect -> symlink-20-bytes - // /c3/20-bytes - // /c3/symlink-20-bytes-double-indirect -> /c2/symlink-20-bytes-indirect - // /c3/cycle-a -> cycle-b - // /c3/cycle-b -> cycle-c - // /c3/cycle-c -> cycle-a - // /c3/symlink-c2 -> /c2 - // /c4 - // /c4/d1 - // /c4/symlink-d1 -> d1 - // /c4/d1/d2 - // /c4/d1/symlink-d2 -> d2 - // /c4/d1/d2/symlink-kitten.png -> /c1/kitten.png - - c1Inode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c1") - c2Inode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c2") - c3Inode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c3") - c4Inode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c4") - - fileInode := fsCreateFile(volumeHandle, c1Inode, "kitten.png") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte("if this were a real kitten, it would be cute"), nil) - if err != nil { - panic(err) - } - - fsCreateSymlink(volumeHandle, c1Inode, "symlink-1", "kitten.png") - fsCreateSymlink(volumeHandle, c1Inode, "symlink-2", "symlink-1") - fsCreateSymlink(volumeHandle, c1Inode, "symlink-3", "symlink-2") - fsCreateSymlink(volumeHandle, c1Inode, "symlink-4", "symlink-3") - fsCreateSymlink(volumeHandle, c1Inode, "symlink-5", "symlink-4") - fsCreateSymlink(volumeHandle, c1Inode, "symlink-6", "symlink-5") - fsCreateSymlink(volumeHandle, c1Inode, "symlink-7", "symlink-6") - fsCreateSymlink(volumeHandle, c1Inode, "symlink-8", "symlink-7") - fsCreateSymlink(volumeHandle, c1Inode, "symlink-9", "symlink-8") - - fileInode = fsCreateFile(volumeHandle, c2Inode, "10-bytes") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte("abcdefghij"), nil) - if err != nil { - panic(err) - } - - fsCreateSymlink(volumeHandle, c2Inode, "symlink-10-bytes", "10-bytes") - fsCreateSymlink(volumeHandle, c2Inode, "symlink-20-bytes", "/c3/20-bytes") - fsCreateSymlink(volumeHandle, c2Inode, "symlink-20-bytes-indirect", "symlink-20-bytes") - - fileInode = fsCreateFile(volumeHandle, c3Inode, "20-bytes") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte("abcdefghijklmnopqrst"), nil) - if err != nil { - panic(err) - } - - fsCreateSymlink(volumeHandle, c3Inode, "symlink-20-bytes-double-indirect", "/c2/symlink-20-bytes-indirect") - fsCreateSymlink(volumeHandle, c3Inode, "symlink-c2", "/c2") - fsCreateSymlink(volumeHandle, c3Inode, "cycle-a", "cycle-b") - fsCreateSymlink(volumeHandle, c3Inode, "cycle-b", "cycle-c") - fsCreateSymlink(volumeHandle, c3Inode, "cycle-c", "cycle-a") - - c4d1Inode := fsMkDir(volumeHandle, c4Inode, "d1") - c4d1d2Inode := fsMkDir(volumeHandle, c4d1Inode, "d2") - fsCreateSymlink(volumeHandle, c4Inode, "symlink-d1", "d1") - fsCreateSymlink(volumeHandle, c4d1Inode, "symlink-d2", "d2") - fsCreateSymlink(volumeHandle, c4d1d2Inode, "symlink-kitten.png", "/c1/kitten.png") - // Test setup complete - - // Test following a single symlink to a file in the same directory - req := GetObjectReq{VirtPath: "/v1/AN_account/c1/symlink-1"} - reply := GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - assert.Equal(uint64(44), reply.FileSize) // size of kitten.png - - // Test following a symlink with an absolute path in it (we treat - // "/" as the root of this filesystem, which is probably not - // helpful for symlinks created on a filesystem mounted somewhere - // like /mnt/smb-vol, but it's the best we've got) - req = GetObjectReq{VirtPath: "/v1/AN_account/c2/symlink-20-bytes"} - reply = GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - assert.Equal(uint64(20), reply.FileSize) - - // Test a chain with relative and absolute paths in it - req = GetObjectReq{VirtPath: "/v1/AN_account/c3/symlink-20-bytes-double-indirect"} - reply = GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - assert.Equal(uint64(20), reply.FileSize) - - // Test following a pair of symlinks to a file in the same directory - req = GetObjectReq{VirtPath: "/v1/AN_account/c1/symlink-2"} - reply = GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - assert.Equal(uint64(44), reply.FileSize) // size of kitten.png - - // Test following a max-length (8) chain of symlinks to a file - req = GetObjectReq{VirtPath: "/v1/AN_account/c1/symlink-8"} - reply = GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - assert.Equal(uint64(44), reply.FileSize) // size of kitten.png - - // Test following a too-long (9) chain of symlinks to a file - req = GetObjectReq{VirtPath: "/v1/AN_account/c1/symlink-9"} - reply = GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.NotNil(err) - assert.Equal(fmt.Sprintf("errno: %d", blunder.TooManySymlinksError.Value()), err.Error()) - - // Test following a cycle: it should look just like an over-length chain - req = GetObjectReq{VirtPath: "/v1/AN_account/c3/cycle-a"} - reply = GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.NotNil(err) - assert.Equal(fmt.Sprintf("errno: %d", blunder.TooManySymlinksError.Value()), err.Error()) - - // Test following a symlink to a directory - req = GetObjectReq{VirtPath: "/v1/AN_account/c3/symlink-c2"} - reply = GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - assert.True(reply.IsDir) - - // Test following a path where some directory components are symlinks - req = GetObjectReq{VirtPath: "/v1/AN_account/c4/symlink-d1/symlink-d2/symlink-kitten.png"} - reply = GetObjectReply{} - err = server.RpcGetObject(&req, &reply) - assert.Nil(err) - assert.Equal(uint64(44), reply.FileSize) // size of kitten.png -} - -func TestRpcPutContainer(t *testing.T) { - server := &Server{} - assert := assert.New(t) - _, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "rpc-put-container-Chartreux-sulphurea" - containerPath := testVerAccountName + "/" + containerName - containerMetadata := []byte("some metadata") - newMetadata := []byte("some new metadata") - req := PutContainerReq{ - VirtPath: containerPath, - OldMetadata: []byte{}, - NewMetadata: containerMetadata, - } - reply := PutContainerReply{} - - err = server.RpcPutContainer(&req, &reply) - assert.Nil(err) - - // Check metadata - headRequest := HeadReq{ - VirtPath: containerPath, - } - headReply := HeadReply{} - err = server.RpcHead(&headRequest, &headReply) - assert.Nil(err) - assert.Equal(containerMetadata, headReply.Metadata) - - // Can't update the metadata unless you know what's there - req = PutContainerReq{ - VirtPath: containerPath, - OldMetadata: []byte{}, // doesn't match what's there - NewMetadata: newMetadata, - } - reply = PutContainerReply{} - err = server.RpcPutContainer(&req, &reply) - assert.NotNil(err) - assert.Equal(fmt.Sprintf("errno: %d", blunder.TryAgainError), err.Error()) - - // Now update the metadata - req = PutContainerReq{ - VirtPath: containerPath, - OldMetadata: containerMetadata, - NewMetadata: newMetadata, - } - reply = PutContainerReply{} - - err = server.RpcPutContainer(&req, &reply) - assert.Nil(err) - - headRequest = HeadReq{ - VirtPath: containerPath, - } - headReply = HeadReply{} - err = server.RpcHead(&headRequest, &headReply) - assert.Nil(err) - assert.Equal(newMetadata, headReply.Metadata) -} - -func TestRpcPutContainerTooLong(t *testing.T) { - server := &Server{} - assert := assert.New(t) - _, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "rpc-put-container-adnascent-splint-" - containerName += strings.Repeat("A", 256-len(containerName)) - containerPath := testVerAccountName + "/" + containerName - req := PutContainerReq{ - VirtPath: containerPath, - OldMetadata: []byte{}, - NewMetadata: []byte{}, - } - reply := PutContainerReply{} - - err = server.RpcPutContainer(&req, &reply) - assert.NotNil(err) -} - -func TestRpcMiddlewareMkdir(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - containerName := "rpc-middleware-mkdir-container" - - fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - dirName := "rpc-middleware-mkdir-test" - dirPath := testVerAccountName + "/" + containerName + "/" + dirName - dirMetadata := []byte("some metadata b5fdbc4a0f1484225fcb7aa64b1e6b94") - req := MiddlewareMkdirReq{ - VirtPath: dirPath, - Metadata: dirMetadata, - } - reply := MiddlewareMkdirReply{} - - err = server.RpcMiddlewareMkdir(&req, &reply) - assert.Nil(err) - - // Check created dir - headRequest := HeadReq{ - VirtPath: dirPath, - } - headReply := HeadReply{} - err = server.RpcHead(&headRequest, &headReply) - assert.Nil(err) - assert.Equal(headReply.Metadata, dirMetadata) - assert.True(headReply.IsDir) - oldInodeNumber := headReply.InodeNumber - - // If the dir exists, we just reuse it, so verify this happened - req = MiddlewareMkdirReq{ - VirtPath: dirPath, - Metadata: dirMetadata, - } - reply = MiddlewareMkdirReply{} - err = server.RpcMiddlewareMkdir(&req, &reply) - assert.Nil(err) - assert.Equal(reply.InodeNumber, oldInodeNumber) -} - -func TestRpcMiddlewareMkdirNested(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - containerName := "rpc-middleware-mkdir-container-nested" - - fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - dirName := "some/deeply/nested/dir" - dirPath := testVerAccountName + "/" + containerName + "/" + dirName - dirMetadata := []byte("some metadata eeef146ba9e5875cb52b047ba4f03660") - req := MiddlewareMkdirReq{ - VirtPath: dirPath, - Metadata: dirMetadata, - } - reply := MiddlewareMkdirReply{} - - err = server.RpcMiddlewareMkdir(&req, &reply) - assert.Nil(err) - - // Check created dir - headRequest := HeadReq{ - VirtPath: dirPath, - } - headReply := HeadReply{} - err = server.RpcHead(&headRequest, &headReply) - assert.Nil(err) - assert.Equal(headReply.Metadata, dirMetadata) - assert.True(headReply.IsDir) -} - -func TestRpcCoalesce(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerAName := "rpc-coalesce-A-catagmatic-invincibly" - containerAPath := testVerAccountName + "/" + containerAName - containerBName := "rpc-coalesce-B-galeproof-palladium" - - destinationPath := containerAPath + "/" + "combined-file" - - containerAInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerAName) - containerBInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerBName) - - containerADir1Inode := fsMkDir(volumeHandle, containerAInode, "dir1") - containerADir1Dir2Inode := fsMkDir(volumeHandle, containerADir1Inode, "dir2") - - fileA1Path := "/" + containerAName + "/dir1/dir2/a1" - fileA1Inode := fsCreateFile(volumeHandle, containerADir1Dir2Inode, "a1") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileA1Inode, 0, []byte("red "), nil) - if err != nil { - panic(err) - } - - // Element paths are relative to the account, but the destination path is absolute. It's a little weird, but it - // means we don't have to worry about element paths pointing to different accounts. - fileA2Path := "/" + containerAName + "/dir1/dir2/a2" - fileA2Inode := fsCreateFile(volumeHandle, containerADir1Dir2Inode, "a2") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileA2Inode, 0, []byte("orange "), nil) - if err != nil { - panic(err) - } - - fileBPath := "/" + containerBName + "/b" - fileBInode := fsCreateFile(volumeHandle, containerBInode, "b") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileBInode, 0, []byte("yellow"), nil) - if err != nil { - panic(err) - } - - timeBeforeRequest := uint64(time.Now().UnixNano()) - - coalesceRequest := CoalesceReq{ - VirtPath: destinationPath, - ElementAccountRelativePaths: []string{ - fileA1Path, - fileA2Path, - fileBPath, - }, - } - coalesceReply := CoalesceReply{} - err = server.RpcCoalesce(&coalesceRequest, &coalesceReply) - assert.Nil(err) - - combinedInode, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerAName+"/combined-file") - assert.Nil(err) - assert.Equal(uint64(combinedInode), uint64(coalesceReply.InodeNumber)) - assert.True(coalesceReply.NumWrites > 0) - assert.True(coalesceReply.ModificationTime > 0) - assert.True(coalesceReply.ModificationTime > timeBeforeRequest) - assert.True(coalesceReply.ModificationTime == coalesceReply.AttrChangeTime) - - combinedContents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, combinedInode, 0, 99999, nil) - assert.Nil(err) - assert.Equal([]byte("red orange yellow"), combinedContents) -} - -func TestRpcCoalesceOverwrite(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "rpc-coalesce-Callynteria-sapor" - containerPath := testVerAccountName + "/" + containerName - containerInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - - destinationPath := containerPath + "/" + "combined" - - filesToWrite := []string{"red", "orange", "yellow", "green", "blue", "indigo", "violet"} - for _, fileName := range filesToWrite { - fileInode := fsCreateFile(volumeHandle, containerInode, fileName) - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileName+" "), nil) - if err != nil { - panic(err) - } - } - - // Create the file - coalesceRequest := CoalesceReq{ - VirtPath: destinationPath, - ElementAccountRelativePaths: []string{ - "/" + containerName + "/red", - "/" + containerName + "/orange", - "/" + containerName + "/yellow", - }, - } - coalesceReply := CoalesceReply{} - err = server.RpcCoalesce(&coalesceRequest, &coalesceReply) - assert.Nil(err) - combinedContents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(coalesceReply.InodeNumber), 0, 99999, nil) - assert.Nil(err) - assert.Equal([]byte("red orange yellow "), combinedContents) // sanity check - - // Now overwrite it - coalesceRequest = CoalesceReq{ - VirtPath: destinationPath, - ElementAccountRelativePaths: []string{ - "/" + containerName + "/green", - "/" + containerName + "/blue", - "/" + containerName + "/indigo", - "/" + containerName + "/violet", - }, - } - coalesceReply = CoalesceReply{} - err = server.RpcCoalesce(&coalesceRequest, &coalesceReply) - assert.Nil(err) - - combinedContents, err = volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.InodeNumber(coalesceReply.InodeNumber), 0, 99999, nil) - assert.Nil(err) - assert.Equal([]byte("green blue indigo violet "), combinedContents) - -} - -func TestRpcCoalesceOverwriteDir(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "rpc-coalesce-speller-spinally" - containerPath := testVerAccountName + "/" + containerName - containerInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - - destinationPath := containerPath + "/" + "combined" - - filesToWrite := []string{"red", "orange", "yellow", "green", "blue", "indigo", "violet"} - for _, fileName := range filesToWrite { - fileInode := fsCreateFile(volumeHandle, containerInode, fileName) - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileName+" "), nil) - if err != nil { - panic(err) - } - } - - combinedInode := fsMkDir(volumeHandle, containerInode, "combined") - - // Create the file - coalesceRequest := CoalesceReq{ - VirtPath: destinationPath, - ElementAccountRelativePaths: []string{ - "/" + containerName + "/red", - "/" + containerName + "/orange", - "/" + containerName + "/yellow", - }, - } - coalesceReply := CoalesceReply{} - err = server.RpcCoalesce(&coalesceRequest, &coalesceReply) - assert.NotNil(err) - - // The old dir is still there - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/combined") - assert.Equal(combinedInode, ino) -} - -func TestRpcCoalesceMakesDirs(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "rpc-coalesce-subsaturation-rowy" - containerPath := testVerAccountName + "/" + containerName - containerInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - - destinationPath := containerPath + "/a/b/c/d/e/f/combined" - - // The directory structure partially exists, but not totally - aInode := fsMkDir(volumeHandle, containerInode, "a") - bInode := fsMkDir(volumeHandle, aInode, "b") - fsMkDir(volumeHandle, bInode, "c") - - filesToWrite := []string{"red", "orange", "yellow", "green", "blue", "indigo", "violet"} - for _, fileName := range filesToWrite { - fileInode := fsCreateFile(volumeHandle, containerInode, fileName) - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileName+" "), nil) - if err != nil { - panic(err) - } - } - - // Create the file - coalesceRequest := CoalesceReq{ - VirtPath: destinationPath, - ElementAccountRelativePaths: []string{ - "/" + containerName + "/red", - "/" + containerName + "/orange", - "/" + containerName + "/yellow", - }, - } - coalesceReply := CoalesceReply{} - err = server.RpcCoalesce(&coalesceRequest, &coalesceReply) - assert.Nil(err) - - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/a/b/c/d/e/f/combined") - assert.Nil(err) - assert.Equal(inode.InodeNumber(coalesceReply.InodeNumber), ino) - - combinedContents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, 0, 99999, nil) - assert.Nil(err) - assert.Equal([]byte("red orange yellow "), combinedContents) // sanity check -} - -func TestRpcCoalesceSymlinks(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "rpc-coalesce-salpingian-utilizer" - containerPath := testVerAccountName + "/" + containerName - containerInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - - destinationPath := containerPath + "/a/b-sl/abs-a-sl/b-sl/c/combined" - - // The directory structure partially exists, but not totally - aInode := fsMkDir(volumeHandle, containerInode, "a") - fsCreateSymlink(volumeHandle, aInode, "b-sl", "b") - bInode := fsMkDir(volumeHandle, aInode, "b") - fsCreateSymlink(volumeHandle, bInode, "abs-a-sl", "/"+containerName+"/a") - fsMkDir(volumeHandle, bInode, "c") - - filesToWrite := []string{"red", "orange", "yellow", "green", "blue", "indigo", "violet"} - for _, fileName := range filesToWrite { - fileInode := fsCreateFile(volumeHandle, containerInode, fileName) - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileName+" "), nil) - if err != nil { - panic(err) - } - } - - // Create the file - coalesceRequest := CoalesceReq{ - VirtPath: destinationPath, - ElementAccountRelativePaths: []string{ - "/" + containerName + "/red", - "/" + containerName + "/orange", - "/" + containerName + "/yellow", - }, - } - coalesceReply := CoalesceReply{} - err = server.RpcCoalesce(&coalesceRequest, &coalesceReply) - assert.Nil(err) - - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/a/b/c/combined") - assert.Nil(err) - assert.Equal(inode.InodeNumber(coalesceReply.InodeNumber), ino) - - combinedContents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, 0, 99999, nil) - assert.Nil(err) - assert.Equal([]byte("red orange yellow "), combinedContents) // sanity check -} - -func TestRpcCoalesceBrokenSymlink(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "rpc-coalesce-Clathrus-playmonger" - containerPath := testVerAccountName + "/" + containerName - containerInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - - destinationPath := containerPath + "/a/busted/c/combined" - - // The directory structure partially exists, but not totally - aInode := fsMkDir(volumeHandle, containerInode, "a") - fsCreateSymlink(volumeHandle, aInode, "busted", "this-symlink-is-broken") - - filesToWrite := []string{"red", "orange", "yellow", "green", "blue", "indigo", "violet"} - for _, fileName := range filesToWrite { - fileInode := fsCreateFile(volumeHandle, containerInode, fileName) - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileName+" "), nil) - if err != nil { - panic(err) - } - } - - // Create the file - coalesceRequest := CoalesceReq{ - VirtPath: destinationPath, - ElementAccountRelativePaths: []string{ - "/" + containerName + "/red", - "/" + containerName + "/orange", - "/" + containerName + "/yellow", - }, - } - coalesceReply := CoalesceReply{} - err = server.RpcCoalesce(&coalesceRequest, &coalesceReply) - assert.Nil(err) - - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, containerName+"/a/this-symlink-is-broken/c/combined") - assert.Nil(err) - assert.Equal(inode.InodeNumber(coalesceReply.InodeNumber), ino) - - combinedContents, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, 0, 99999, nil) - assert.Nil(err) - assert.Equal([]byte("red orange yellow "), combinedContents) // sanity check -} - -func TestRpcCoalesceSubdirOfAFile(t *testing.T) { - server := &Server{} - assert := assert.New(t) - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - containerName := "rpc-coalesce-fanam-outswim" - containerPath := testVerAccountName + "/" + containerName - containerInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, containerName) - - destinationPath := containerPath + "/a/b-is-a-file/c/combined" - - // The directory structure partially exists, but not totally - aInode := fsMkDir(volumeHandle, containerInode, "a") - fsCreateFile(volumeHandle, aInode, "b-is-a-file") - - filesToWrite := []string{"red", "orange", "yellow"} - for _, fileName := range filesToWrite { - fileInode := fsCreateFile(volumeHandle, containerInode, fileName) - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileName+" "), nil) - if err != nil { - panic(err) - } - } - - // Create the file - coalesceRequest := CoalesceReq{ - VirtPath: destinationPath, - ElementAccountRelativePaths: []string{ - "/" + containerName + "/red", - "/" + containerName + "/orange", - "/" + containerName + "/yellow", - }, - } - coalesceReply := CoalesceReply{} - err = server.RpcCoalesce(&coalesceRequest, &coalesceReply) - assert.NotNil(err) - assert.Equal(fmt.Sprintf("errno: %d", blunder.PermDeniedError), err.Error()) -} diff --git a/jrpcfs/ping.go b/jrpcfs/ping.go deleted file mode 100644 index 9a6ce268..00000000 --- a/jrpcfs/ping.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -// Simple ping for testing the RPC layer -import ( - "fmt" - - "github.com/NVIDIA/proxyfs/logger" -) - -func (s *Server) RpcPing(in *PingReq, reply *PingReply) (err error) { - enterGate() - defer leaveGate() - - if globals.dataPathLogging { - flog := logger.TraceEnter("in.", in) - defer func() { flog.TraceExitErr("reply.", err, reply) }() - } - - reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) - return nil -} diff --git a/jrpcfs/retryrpc.go b/jrpcfs/retryrpc.go deleted file mode 100644 index 4021ea39..00000000 --- a/jrpcfs/retryrpc.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/retryrpc" -) - -func retryRPCServerUp(jserver *Server) { - var ( - err error - ) - - if globals.retryRPCPort == 0 { - return - } - - // Create a new RetryRPC Server. - retryConfig := &retryrpc.ServerConfig{ - LongTrim: globals.retryRPCTTLCompleted, - ShortTrim: globals.retryRPCAckTrim, - DNSOrIPAddr: globals.publicIPAddr, - Port: int(globals.retryRPCPort), - DeadlineIO: globals.retryRPCDeadlineIO, - KeepAlivePeriod: globals.retryRPCKeepAlivePeriod, - TLSCertificate: globals.retryRPCCertificate, - } - - rrSvr := retryrpc.NewServer(retryConfig) - - // Register jrpcsfs methods with the retryrpc server - err = rrSvr.Register(jserver) - if err != nil { - logger.ErrorfWithError(err, "failed to register Retry RPC handler") - return - } - - // Start the retryrpc server listener - startErr := rrSvr.Start() - if startErr != nil { - logger.ErrorfWithError(startErr, "retryrpc.Start() failed with err: %v", startErr) - return - } - - globals.connLock.Lock() - globals.retryrpcSvr = rrSvr - globals.connLock.Unlock() - - // Tell retryrpc server to start accepting requests - rrSvr.Run() -} - -func retryRPCServerDown() { - if globals.retryRPCPort == 0 { - return - } - - globals.connLock.Lock() - rrSvr := globals.retryrpcSvr - globals.connLock.Unlock() - rrSvr.Close() -} diff --git a/jrpcfs/setup_teardown_test.go b/jrpcfs/setup_teardown_test.go deleted file mode 100644 index 9d9a73b7..00000000 --- a/jrpcfs/setup_teardown_test.go +++ /dev/null @@ -1,463 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package jrpcfs - -import ( - "crypto/tls" - "crypto/x509/pkix" - "encoding/gob" - "fmt" - "io/ioutil" - "net" - "os" - "sync" - "testing" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/icert/icertpkg" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/ramswift" - "github.com/NVIDIA/proxyfs/transitions" -) - -// Shorthand for our testing debug log id; global to the package -var test_debug = logger.DbgTesting - -const testVer = "/v1/" -const testAccountName = "AN_account" -const testContainerName = "test_container" -const testVerAccountName = testVer + testAccountName -const testVerAccountContainerName = testVerAccountName + "/" + testContainerName -const testAccountName2 = "AN_account2" - -const testRequireSlashesInPathsToProperlySort = false - -type testTLSCertsStruct struct { - caCertPEMBlock []byte - caKeyPEMBlock []byte - endpointCertPEMBlock []byte - endpointKeyPEMBlock []byte - endpointTLSCert tls.Certificate - caCertFile string - caKeyFile string - endpointCertFile string - endpointKeyFile string -} - -var testTLSCerts *testTLSCertsStruct - -func TestMain(m *testing.M) { - //setup, run, teardown, exit - - cleanupFuncs := testSetup() - - makeSomeFilesAndSuch() - - verdict := m.Run() - - for _, cleanupFunc := range cleanupFuncs { - cleanupFunc() - } - - os.Exit(verdict) -} - -func testSetup() []func() { - var ( - cleanupFuncs []func() - cleanupTempDir func() - confStrings []string - doneChan chan bool - err error - signalHandlerIsArmedWG sync.WaitGroup - tempDir string - testConfMap conf.ConfMap - ) - - // register some RPC types to gob so it can encode/decode them efficiently - gob.Register(&LeaseRequest{}) - gob.Register(&LeaseReply{}) - - cleanupFuncs = make([]func(), 0) - - tempDir, err = ioutil.TempDir("", "jrpcfs_test") - if nil != err { - panic(fmt.Sprintf("failed in testSetup: %v", err)) - } - cleanupTempDir = func() { - _ = os.RemoveAll(tempDir) - } - cleanupFuncs = append(cleanupFuncs, cleanupTempDir) - - testTLSCerts = &testTLSCertsStruct{ - caCertFile: tempDir + "/caCertFile", - caKeyFile: tempDir + "/caKeyFile", - endpointCertFile: tempDir + "/endpoingCertFile", - endpointKeyFile: tempDir + "/endpointKeyFile", - } - - testTLSCerts.caCertPEMBlock, testTLSCerts.caKeyPEMBlock, err = icertpkg.GenCACert( - icertpkg.GenerateKeyAlgorithmEd25519, - pkix.Name{ - Organization: []string{"Test Organization CA"}, - Country: []string{}, - Province: []string{}, - Locality: []string{}, - StreetAddress: []string{}, - PostalCode: []string{}, - }, - time.Hour, - testTLSCerts.caCertFile, - testTLSCerts.caKeyFile) - if nil != err { - panic(fmt.Errorf("icertpkg.GenCACert() failed: %v", err)) - } - - testTLSCerts.endpointCertPEMBlock, testTLSCerts.endpointKeyPEMBlock, err = icertpkg.GenEndpointCert( - icertpkg.GenerateKeyAlgorithmEd25519, - pkix.Name{ - Organization: []string{"Test Organization Endpoint"}, - Country: []string{}, - Province: []string{}, - Locality: []string{}, - StreetAddress: []string{}, - PostalCode: []string{}, - }, - []string{}, - []net.IP{net.ParseIP("127.0.0.1")}, - time.Hour, - testTLSCerts.caCertPEMBlock, - testTLSCerts.caKeyPEMBlock, - testTLSCerts.endpointCertFile, - testTLSCerts.endpointKeyFile) - if nil != err { - panic(fmt.Errorf("icertpkg.genEndpointCert() failed: %v", err)) - } - - testTLSCerts.endpointTLSCert, err = tls.X509KeyPair(testTLSCerts.endpointCertPEMBlock, testTLSCerts.endpointKeyPEMBlock) - if nil != err { - panic(fmt.Errorf("tls.LoadX509KeyPair() failed: %v", err)) - } - - confStrings = []string{ - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - "FSGlobals.VolumeGroupList=JrpcfsTestVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=8", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.DirEntryCacheEvictLowLimit=10000", - "FSGlobals.DirEntryCacheEvictHighLimit=10010", - "FSGlobals.FileExtentMapEvictLowLimit=10000", - "FSGlobals.FileExtentMapEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=35262", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=3", - "SwiftClient.RetryLimitObject=3", - "SwiftClient.RetryDelay=10ms", - "SwiftClient.RetryDelayObject=10ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=64", - "SwiftClient.NonChunkedConnectionPoolSize=32", - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=256", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - "Peer:Peer0.PublicIPAddr=127.0.0.1", - "Peer:Peer0.PrivateIPAddr=127.0.0.1", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - "Cluster.Peers=Peer0", - "Cluster.WhoAmI=Peer0", - "Volume:SomeVolume.FSID=1", - "Volume:SomeVolume.AccountName=" + testAccountName, - "Volume:SomeVolume.AutoFormat=true", - "Volume:SomeVolume.CheckpointContainerName=.__checkpoint__", - "Volume:SomeVolume.CheckpointContainerStoragePolicy=gold", - "Volume:SomeVolume.CheckpointInterval=10s", - "Volume:SomeVolume.DefaultPhysicalContainerLayout=SomeContainerLayout", - "Volume:SomeVolume.MaxFlushSize=10027008", - "Volume:SomeVolume.MaxFlushTime=2s", - "Volume:SomeVolume.FileDefragmentChunkSize=10027008", - "Volume:SomeVolume.FileDefragmentChunkDelay=2ms", - "Volume:SomeVolume.NonceValuesToReserve=100", - "Volume:SomeVolume.MaxEntriesPerDirNode=32", - "Volume:SomeVolume.MaxExtentsPerFileNode=32", - "Volume:SomeVolume.MaxInodesPerMetadataNode=32", - "Volume:SomeVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:SomeVolume.MaxDirFileNodesPerMetadataNode=16", - "Volume:SomeVolume.MaxBytesInodeCache=100000", - "Volume:SomeVolume.InodeCacheEvictInterval=1s", - "Volume:SomeVolume.ActiveLeaseEvictLowLimit=2", - "Volume:SomeVolume.ActiveLeaseEvictHighLimit=4", - "Volume:SomeVolume2.FSID=2", - "Volume:SomeVolume2.AccountName=" + testAccountName2, - "Volume:SomeVolume2.AutoFormat=true", - "Volume:SomeVolume2.CheckpointContainerName=.__checkpoint__", - "Volume:SomeVolume2.CheckpointContainerStoragePolicy=gold", - "Volume:SomeVolume2.CheckpointInterval=10s", - "Volume:SomeVolume2.DefaultPhysicalContainerLayout=SomeContainerLayout2", - "Volume:SomeVolume2.MaxFlushSize=10027008", - "Volume:SomeVolume2.MaxFlushTime=2s", - "Volume:SomeVolume2.FileDefragmentChunkSize=10027008", - "Volume:SomeVolume2.FileDefragmentChunkDelay=2ms", - "Volume:SomeVolume2.NonceValuesToReserve=100", - "Volume:SomeVolume2.MaxEntriesPerDirNode=32", - "Volume:SomeVolume2.MaxExtentsPerFileNode=32", - "Volume:SomeVolume2.MaxInodesPerMetadataNode=32", - "Volume:SomeVolume2.MaxLogSegmentsPerMetadataNode=64", - "Volume:SomeVolume2.MaxDirFileNodesPerMetadataNode=16", - "Volume:SomeVolume2.MaxBytesInodeCache=100000", - "Volume:SomeVolume2.InodeCacheEvictInterval=1s", - "Volume:SomeVolume2.ActiveLeaseEvictLowLimit=5000", - "Volume:SomeVolume2.ActiveLeaseEvictHighLimit=5010", - "VolumeGroup:JrpcfsTestVolumeGroup.VolumeList=SomeVolume,SomeVolume2", - "VolumeGroup:JrpcfsTestVolumeGroup.VirtualIPAddr=", - "VolumeGroup:JrpcfsTestVolumeGroup.PrimaryPeer=Peer0", - "VolumeGroup:JrpcfsTestVolumeGroup.ReadCacheLineSize=1000000", - "VolumeGroup:JrpcfsTestVolumeGroup.ReadCacheWeight=100", - "PhysicalContainerLayout:SomeContainerLayout.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:SomeContainerLayout.ContainerNamePrefix=kittens", - "PhysicalContainerLayout:SomeContainerLayout.ContainersPerPeer=10", - "PhysicalContainerLayout:SomeContainerLayout.MaxObjectsPerContainer=1000000", - "PhysicalContainerLayout:SomeContainerLayout2.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:SomeContainerLayout2.ContainerNamePrefix=puppies", - "PhysicalContainerLayout:SomeContainerLayout2.ContainersPerPeer=10", - "PhysicalContainerLayout:SomeContainerLayout2.MaxObjectsPerContainer=1000000", - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - "JSONRPCServer.TCPPort=12346", // 12346 instead of 12345 so that test can run if proxyfsd is already running - "JSONRPCServer.FastTCPPort=32346", // ...and similarly here... - "JSONRPCServer.RetryRPCPort=32357", // ...and similarly here... - "JSONRPCServer.RetryRPCTTLCompleted=10s", - "JSONRPCServer.RetryRPCAckTrim=10ms", - "JSONRPCServer.RetryRPCCertFilePath=" + testTLSCerts.endpointCertFile, - "JSONRPCServer.RetryRPCKeyFilePath=" + testTLSCerts.endpointKeyFile, - "JSONRPCServer.DataPathLogging=false", - "JSONRPCServer.MinLeaseDuration=100ms", - "JSONRPCServer.LeaseInterruptInterval=100ms", - "JSONRPCServer.LeaseInterruptLimit=5", - } - - testConfMap, err = conf.MakeConfMapFromStrings(confStrings) - if nil != err { - panic(fmt.Sprintf("failed in testSetup: %v", err)) - } - - signalHandlerIsArmedWG.Add(1) - doneChan = make(chan bool) - go ramswift.Daemon("/dev/null", confStrings, &signalHandlerIsArmedWG, doneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - err = transitions.Up(testConfMap) - if nil != err { - panic(fmt.Sprintf("transitions.Up() failed: %v", err)) - } - - return cleanupFuncs -} - -func makeSomeFilesAndSuch() { - // we should have enough stuff up now that we can actually make - // some files and directories and such - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - cInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c") - cNestedInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c-nested") - fsCreateSymlink(volumeHandle, inode.RootDirInodeNumber, "c-symlink", "c") - - err = volumeHandle.MiddlewarePost("", "c", []byte("metadata for c"), []byte{}) - if err != nil { - panic(err) - } - _ = fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c-no-metadata") - _ = fsMkDir(volumeHandle, cInode, "empty-directory") - - readmeInode := fsCreateFile(volumeHandle, cInode, "README") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, readmeInode, 0, []byte("who am I kidding? nobody reads these."), nil) - err = volumeHandle.MiddlewarePost("", "c/README", []byte("metadata for c/README"), []byte{}) - if err != nil { - panic(err) - } - - animalsInode := fsMkDir(volumeHandle, cInode, "animals") - files := map[string]string{ - "dog.txt": "dog goes woof", - "cat.txt": "cat goes meow", - "bird.txt": "bird goes tweet", - "mouse.txt": "mouse goes squeak", - "cow.txt": "cow goes moo", - "frog.txt": "frog goes croak", - "elephant.txt": "elephant goes toot", - } - for fileName, fileContents := range files { - fileInode := fsCreateFile(volumeHandle, animalsInode, fileName) - - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileContents), nil) - if err != nil { - panic(fmt.Sprintf("failed to write file %s: %v", fileName, err)) - } - } - - plantsInode := fsMkDir(volumeHandle, cInode, "plants") - ino := fsCreateFile(volumeHandle, cInode, "plants-README") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, 0, []byte("nah"), nil) - if err != nil { - panic(fmt.Sprintf("failed to write file plants-README: %v", err)) - } - - files = map[string]string{ - // Random contents of varying lengths. - "aloe.txt": "skiameter-interlope", - "banana.txt": "ring ring ring ring ring ring ring bananaphone", - "cherry.txt": "archegonium-nonresidentiary", - "eggplant.txt": "bowk-unruled", - } - - for fileName, fileContents := range files { - fileInode := fsCreateFile(volumeHandle, plantsInode, fileName) - - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileContents), nil) - if err != nil { - panic(fmt.Sprintf("failed to write file %s: %v", fileName, err)) - } - } - - fsCreateSymlink(volumeHandle, cInode, "plants-symlink", "plants") - fsCreateSymlink(volumeHandle, plantsInode, "eggplant.txt-symlink", "eggplant.txt") - - // Put some deeply nested things in c-nested. This listing is a - // shortened version of a real directory tree that exposed a bug. - fsCreateFile(volumeHandle, cNestedInode, ".DS_Store") - dotGitInode := fsMkDir(volumeHandle, cNestedInode, ".git") - fsCreateFile(volumeHandle, dotGitInode, ".DS_Store") - fsCreateFile(volumeHandle, dotGitInode, "COMMIT_EDITMSG") - fsCreateFile(volumeHandle, dotGitInode, "FETCH_HEAD") - fsCreateFile(volumeHandle, dotGitInode, "HEAD") - fsCreateFile(volumeHandle, dotGitInode, "ORIG_HEAD") - fsCreateFile(volumeHandle, dotGitInode, "index") - dotGitHooks := fsMkDir(volumeHandle, dotGitInode, "hooks") - fsCreateFile(volumeHandle, dotGitHooks, ".DS_Store") - fsCreateFile(volumeHandle, dotGitHooks, "applypatch-msg.sample") - fsCreateFile(volumeHandle, dotGitHooks, "commit-msg.sample") - dotGitLogs := fsMkDir(volumeHandle, dotGitInode, "logs") - fsCreateFile(volumeHandle, dotGitLogs, ".DS_Store") - fsCreateFile(volumeHandle, dotGitLogs, "HEAD") - dotGitLogsRefs := fsMkDir(volumeHandle, dotGitLogs, "refs") - fsCreateFile(volumeHandle, dotGitLogsRefs, ".DS_Store") - fsCreateFile(volumeHandle, dotGitLogsRefs, "stash") - dotGitLogsRefsHeads := fsMkDir(volumeHandle, dotGitLogsRefs, "heads") - fsCreateFile(volumeHandle, dotGitLogsRefsHeads, ".DS_Store") - fsCreateFile(volumeHandle, dotGitLogsRefsHeads, "development") - fsCreateFile(volumeHandle, dotGitLogsRefsHeads, "stable") - - aInode := fsMkDir(volumeHandle, cNestedInode, "a") - fsCreateFile(volumeHandle, aInode, "b-1") - fsCreateFile(volumeHandle, aInode, "b-2") - abInode := fsMkDir(volumeHandle, aInode, "b") - fsCreateFile(volumeHandle, abInode, "c-1") - fsCreateFile(volumeHandle, abInode, "c-2") - abcInode := fsMkDir(volumeHandle, abInode, "c") - fsCreateFile(volumeHandle, abcInode, "d-1") - fsCreateFile(volumeHandle, abcInode, "d-2") - - // SomeVolume2 is set up for testing account listings - volumeHandle2, err := fs.FetchVolumeHandleByVolumeName("SomeVolume2") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "alpha") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "bravo") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "charlie") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "delta") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "echo") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "foxtrot") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "golf") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "hotel") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "india") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "juliet") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "kilo") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "lima") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "mancy") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "november") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "oscar") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "papa") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "quebec") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "romeo") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "sierra") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "tango") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "uniform") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "victor") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "whiskey") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "xray") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "yankee") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "zulu") - _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "alice.txt") - _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "bob.txt") - _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "carol.txt") -} - -func fsStatPath(accountName string, path string) fs.Stat { - _, _, _, _, volumeHandle, err := parseVirtPath(accountName) - if err != nil { - panic(err) - } - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, path) - if err != nil { - panic(err) - } - stats, err := volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino) - if err != nil { - panic(err) - } - return stats -} - -func fsMkDir(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, newDirName string) (createdInode inode.InodeNumber) { - createdInode, err := volumeHandle.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, newDirName, inode.PosixModePerm) - if err != nil { - panic(fmt.Sprintf("failed to create %v: %v", newDirName, err)) - } - return -} - -func fsCreateFile(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, newFileName string) (createdInode inode.InodeNumber) { - createdInode, err := volumeHandle.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, newFileName, inode.PosixModePerm) - if err != nil { - panic(fmt.Sprintf("failed to create file %v: %v", newFileName, err)) - } - return -} - -func fsCreateSymlink(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, symlinkName string, symlinkTarget string) { - _, err := volumeHandle.Symlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, symlinkName, symlinkTarget) - if err != nil { - panic(fmt.Sprintf("failed to create symlink %s -> %s: %v", symlinkName, symlinkTarget, err)) - } - return -} diff --git a/jrpcfs/test/Makefile b/jrpcfs/test/Makefile deleted file mode 100644 index 8c3d6a4a..00000000 --- a/jrpcfs/test/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -CC=gcc -CFLAGS=-I. -JSON_C_DIR=/Users/kmalone/code/gowork/src/github.com/NVIDIA/samba/json-c -CFLAGS += -I $(JSON_C_DIR)/include/json-c -LDFLAGS+= -L $(JSON_C_DIR)/lib -ljson-c -DEPS = - -%.o: %.c $(DEPS) - $(CC) $(CFLAGS) -c -o $@ $< - -client_test: client_test.o - $(CC) $(LDFLAGS) -o client_test client_test.o -I. diff --git a/jrpcfs/test/client_test.c b/jrpcfs/test/client_test.c deleted file mode 100644 index a644d5b7..00000000 --- a/jrpcfs/test/client_test.c +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -void error(const char *msg) -{ - perror(msg); - //exit(0); -} - -static int sockfd = 0; -static int portno = 12345; -static char* hostname = "localhost"; - -int sock_open() { - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - if (sockfd < 0) { - error("ERROR opening socket"); - return -1; - } - //printf("socket %s:%d opened successfully.\n",hostname,portno); - server = gethostbyname(hostname); - if (server == NULL) { - printf("ERROR, no such host\n"); - return -1; - } - //printf("got server for hostname %s.\n",hostname); - bzero((char *) &serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(portno); - - if (connect(sockfd,(struct sockaddr *) &serv_addr,sizeof(serv_addr)) < 0) { - //printf("ERROR connecting"); - error("ERROR connecting"); - return -1; - } - - return 0; -} - -int sock_close() { - close(sockfd); - - return 0; -} - -int sock_read(char* buf) { - int n = read(sockfd,buf,255); - if (n < 0) { - error("ERROR reading from socket"); - return -1; - } - return 0; -} - -int sock_write(const char* buf) { - int n = write(sockfd,buf,strlen(buf)); - if (n < 0) { - error("ERROR writing to socket"); - return -1; - } - return 0; -} - -/*printing the value corresponding to boolean, double, integer and strings*/ -void print_json_object(struct json_object *jobj, const char *msg) { - printf("\n%s: \n", msg); - printf("---\n%s\n---\n", json_object_to_json_string(jobj)); -} - -struct json_object * find_something(struct json_object *jobj, const char *key) { - struct json_object *tmp; - - json_object_object_get_ex(jobj, key, &tmp); - - return tmp; -} - -int get_jrpc_id(json_object* jobj) -{ - json_object* obj = find_something(jobj, "id"); - - enum json_type type = json_object_get_type(obj); - if (type != json_type_int) { - printf("Error, id field is not an int (type=%d)!\n",type); - return -1; - } - - return json_object_get_int(obj); -} - -int get_jrpc_error(json_object* jobj) -{ - json_object* obj = NULL; - if (json_object_object_get_ex(jobj, "error", &obj)) { - // key was found, as it should be - enum json_type type = json_object_get_type(obj); - if (type == json_type_string) { - // Found an error - printf("Error value: %s\n", json_object_get_string(obj)); - return -1; - } - } else { - printf("Error field not found in response!\n"); - } - - return 0; -} - -json_object* get_jrpc_result(json_object* jobj) -{ - return find_something(jobj, "result"); -} - -uint64_t get_jrpc_mount_id(json_object* jobj) -{ - json_object* obj = NULL; - // MountID is inside the result object - json_object* robj = get_jrpc_result(jobj); - if (!json_object_object_get_ex(robj, "MountID", &obj)) { - // key was not found - printf("MountID field not found in response!\n"); - return 0; - } - - return json_object_get_int64(obj); -} - -json_object* build_jrpc_request(int id, char* method) -{ - // Create a new JSON object to populate and return - json_object* jobj = json_object_new_object(); - - // Add the top-level key-value pairs - json_object_object_add(jobj, "id", json_object_new_int(id)); - json_object_object_add(jobj, "method", json_object_new_string(method)); - json_object_object_add(jobj, "jsonrpc", json_object_new_string("2.0")); - - // Create the params array, consisting of key-value pairs - json_object* pobj = json_object_new_array(); - json_object* tmp = json_object_new_object(); - json_object_object_add(tmp, "VolumeName", json_object_new_string("CommonVolume")); - json_object_object_add(tmp, "MountOptions", json_object_new_int(0)); - json_object_object_add(tmp, "AuthUser", json_object_new_string("balajirao")); - json_object_array_add(pobj,tmp); - - // Add the params array to the top-level object - json_object_object_add(jobj,"params",pobj); - - return jobj; -} - - -int main(int argc, char *argv[]) -{ - char buffer[256]; - int id = 1; - - // Open socket - //printf("Opening socket.\n"); - if (sock_open() < 0) { - goto done; - } - - // Build our request header - json_object* myobj = build_jrpc_request(id, "Server.RpcMount"); - - // Send something - const char* writeBuf = json_object_to_json_string_ext(myobj, JSON_C_TO_STRING_PLAIN); - printf("Sending data: %s\n",writeBuf); - if (sock_write(writeBuf) < 0) { - printf("Error writing to socket.\n"); - goto sock_close_and_done; - } - - // Read response - //printf("Reading from socket.\n"); - bzero(buffer,256); - if (sock_read(buffer) < 0) { - printf("Error reading from socket.\n"); - goto sock_close_and_done; - } - //printf("Read %s\n",buffer); - json_object* jobj = json_tokener_parse(buffer); - //printf("response:\n---\n%s\n---\n", json_object_to_json_string_ext(jobj, JSON_C_TO_STRING_SPACED | JSON_C_TO_STRING_PRETTY)); - - // Get id from response - int rsp_id = get_jrpc_id(jobj); - if (rsp_id != id) { - printf("Error, expected id=%d, received id=%d", id, rsp_id); - } else { - // printf("Response id = %d\n",rsp_id); - } - - // Was there an error? - if (get_jrpc_error(jobj) != 0) { - printf("Error was set in response.\n"); - } - - // Try to find the mount ID - printf("Returned MountID: %lld\n", get_jrpc_mount_id(jobj)); - -sock_close_and_done: - // Close socket - //printf("Closing socket.\n"); - sock_close(); - -done: - return 0; -} diff --git a/jrpcfs/test/client_test.py b/jrpcfs/test/client_test.py deleted file mode 100644 index baeed316..00000000 --- a/jrpcfs/test/client_test.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -import json -import requests -import socket - -# This version works as well, leaving here as a reference -#def main(): -# args = {'VolumeName' : "CommonVolume", 'MountOptions': 0, 'AuthUser': "balajirao"} -# -# payload = { -# "method": "Server.RpcMount", -# "params": [args], -# "id": 1, -# } -# -# s = socket.create_connection(("localhost", 12345)) -# s.sendall(json.dumps((payload))) -# rdata = s.recv(1024) -# -# print "received data:", rdata -# print "decoded received data:", json.loads(rdata) -# -# s.close - -def main(): - args = {'VolumeName' : "CommonVolume", 'MountOptions': 0, 'AuthUser': "balajirao"} - - id = 0 - payload = { - "method": "Server.RpcMount", - "params": [args], - "jsonrpc": "2.0", - "id": id, - } - - s = socket.create_connection(("localhost", 12345)) - data = json.dumps((payload)) - print "sending data:", data - s.sendall(data) - - # This will actually have to loop if resp is bigger - rdata = s.recv(1024) - - #print "received data:", rdata - resp = json.loads(rdata) - #print "decoded received data:", resp - - if resp["id"] != id: - raise Exception("expected id=%s, received id=%s: %s" %(id, resp["id"], resp["error"])) - - if resp["error"] is not None: - raise Exception(resp["error"]) - - - print "Returned MountID:", resp["result"]["MountID"] - - s.close - - -if __name__ == "__main__": - main() diff --git a/jrpcfs/test/example.go b/jrpcfs/test/example.go deleted file mode 100644 index fdb5f996..00000000 --- a/jrpcfs/test/example.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "errors" - "fmt" - "log" - "net" - "net/rpc" - "net/rpc/jsonrpc" -) - -type Args struct { - A, B int -} - -type Reply struct { - C int -} - -type Arith int - -//type ArithAddResp struct { -// Id interface{} `json:"id"` -// Result Reply `json:"result"` -// Error interface{} `json:"error"` -//} - -func (t *Arith) Add(args *Args, reply *Reply) error { - reply.C = args.A + args.B - return nil -} - -func (t *Arith) Mul(args *Args, reply *Reply) error { - reply.C = args.A * args.B - return nil -} - -func (t *Arith) Div(args *Args, reply *Reply) error { - if args.B == 0 { - return errors.New("divide by zero") - } - reply.C = args.A / args.B - return nil -} - -func (t *Arith) Error(args *Args, reply *Reply) error { - panic("ERROR") -} - -func startServer() { - arith := new(Arith) - - server := rpc.NewServer() - server.Register(arith) - - l, e := net.Listen("tcp", ":8222") - if e != nil { - log.Fatal("listen error:", e) - } - - for { - conn, err := l.Accept() - if err != nil { - log.Fatal(err) - } - - go server.ServeCodec(jsonrpc.NewServerCodec(conn)) - } -} - -func main() { - - // starting server in go routine (it ends on end - // of main function - go startServer() - - // now client part connecting to RPC service - // and calling methods - - conn, err := net.Dial("tcp", "localhost:8222") - - if err != nil { - panic(err) - } - defer conn.Close() - - c := jsonrpc.NewClient(conn) - - var reply Reply - var args *Args - for i := 0; i < 11; i++ { - // passing Args to RPC call - args = &Args{7, i} - - // calling "Arith.Mul" on RPC server - err = c.Call("Arith.Mul", args, &reply) - if err != nil { - log.Fatal("arith error:", err) - } - fmt.Printf("Arith: %d * %d = %v\n", args.A, args.B, reply.C) - - // calling "Arith.Add" on RPC server - err = c.Call("Arith.Add", args, &reply) - if err != nil { - log.Fatal("arith error:", err) - } - fmt.Printf("Arith: %d + %d = %v\n", args.A, args.B, reply.C) - - // NL - fmt.Printf("\033[33m%s\033[m\n", "---------------") - } -} \ No newline at end of file diff --git a/liveness/Makefile b/liveness/Makefile deleted file mode 100644 index b18e1302..00000000 --- a/liveness/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/liveness - -include ../GoMakefile diff --git a/liveness/api.go b/liveness/api.go deleted file mode 100644 index 357a5b8c..00000000 --- a/liveness/api.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package liveness - -import ( - "time" -) - -const ( - StateAlive = "alive" - StateDead = "dead" - StateUnknown = "unknown" -) - -type VolumeStruct struct { - Name string - State string // One of const State{Alive|Dead|Unknown} - LastCheckTime time.Time -} - -type VolumeGroupStruct struct { - Name string - State string // One of const State{Alive|Dead|Unknown} - LastCheckTime time.Time - Volume []*VolumeStruct -} - -type ServingPeerStruct struct { - Name string - State string // One of const State{Alive|Dead|Unknown} - LastCheckTime time.Time - VolumeGroup []*VolumeGroupStruct -} - -type ReconEndpointStruct struct { - IPAddrPort string - MaxDiskUsagePercentage uint8 -} - -type ObservingPeerStruct struct { - Name string - ServingPeer []*ServingPeerStruct - ReconEndpoint []*ReconEndpointStruct -} - -type LivenessReportStruct struct { - ObservingPeer []*ObservingPeerStruct -} - -func FetchLivenessReport() (livenessReport *LivenessReportStruct) { - livenessReport = fetchLivenessReport() - return -} diff --git a/liveness/api_internal.go b/liveness/api_internal.go deleted file mode 100644 index 23be90a0..00000000 --- a/liveness/api_internal.go +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package liveness - -import ( - "reflect" - "sync" -) - -func fetchLivenessReport() (livenessReport *LivenessReportStruct) { - var ( - err error - fetchLivenessReportRequest *FetchLivenessReportRequestStruct - fetchLivenessReportResponse *FetchLivenessReportResponseStruct - leaderPeer *peerStruct - leaderResponseDone sync.WaitGroup - ) - - globals.Lock() - if globals.active { - switch reflect.ValueOf(globals.nextState) { - case reflect.ValueOf(doCandidate): - // Return a local-only Liveness Report - livenessReport = &LivenessReportStruct{ - ObservingPeer: make([]*ObservingPeerStruct, 1), - } - livenessReport.ObservingPeer[0] = convertInternalToExternalObservingPeerReport(globals.myObservingPeerReport) - case reflect.ValueOf(doFollower): - leaderPeer = globals.currentLeader - if nil == leaderPeer { - // Special case where we just started in doFollower state but haven't yet received a HeartBeat - livenessReport = nil - } else { - // Need to go ask leaderPeer for Liveness Report - fetchLivenessReportRequest = &FetchLivenessReportRequestStruct{ - MsgType: MsgTypeFetchLivenessReportRequest, - MsgTag: fetchNonceWhileLocked(), - CurrentTerm: globals.currentTerm, - } - globals.Unlock() - leaderResponseDone.Add(1) - err = sendRequest( - leaderPeer, - fetchLivenessReportRequest.MsgTag, - nil, // Only one can be outstanding... so no need for non-nil requestContext - fetchLivenessReportRequest, - func(request *requestStruct) { - if request.expired { - fetchLivenessReportResponse = nil - } else { - fetchLivenessReportResponse = request.responseMsg.(*FetchLivenessReportResponseStruct) - } - leaderResponseDone.Done() - }, - ) - if nil != err { - panic(err) - } - leaderResponseDone.Wait() - livenessReport = fetchLivenessReportResponse.LivenessReport - globals.Lock() - } - case reflect.ValueOf(doLeader): - livenessReport = convertInternalToExternalLivenessReport(globals.livenessReport) - } - } else { - livenessReport = nil - } - globals.Unlock() - return -} - -func convertInternalToExternalLivenessReport(internalLivenessReport *internalLivenessReportStruct) (externalLivenessReport *LivenessReportStruct) { - var ( - internalObservingPeerReport *internalObservingPeerReportStruct - observingPeer *ObservingPeerStruct - ) - - if nil == internalLivenessReport { - externalLivenessReport = nil - return - } - - externalLivenessReport = &LivenessReportStruct{ - ObservingPeer: make([]*ObservingPeerStruct, 0, len(internalLivenessReport.observingPeer)), - } - - for _, internalObservingPeerReport = range internalLivenessReport.observingPeer { - observingPeer = convertInternalToExternalObservingPeerReport(internalObservingPeerReport) - externalLivenessReport.ObservingPeer = append(externalLivenessReport.ObservingPeer, observingPeer) - } - - return -} - -func convertExternalToInternalLivenessReport(externalLivenessReport *LivenessReportStruct) (internalLivenessReport *internalLivenessReportStruct) { - var ( - internalObservingPeerReport *internalObservingPeerReportStruct - observingPeer *ObservingPeerStruct - ) - - if nil == externalLivenessReport { - internalLivenessReport = nil - return - } - - internalLivenessReport = &internalLivenessReportStruct{ - observingPeer: make(map[string]*internalObservingPeerReportStruct), - } - - for _, observingPeer = range externalLivenessReport.ObservingPeer { - internalObservingPeerReport = convertExternalToInternalObservingPeerReport(observingPeer) - internalLivenessReport.observingPeer[internalObservingPeerReport.name] = internalObservingPeerReport - } - - return -} - -func convertInternalToExternalObservingPeerReport(internalObservingPeerReport *internalObservingPeerReportStruct) (externalObservingPeer *ObservingPeerStruct) { - var ( - internalReconEndpointReport *internalReconEndpointReportStruct - internalServingPeerReport *internalServingPeerReportStruct - internalVolumeGroupReport *internalVolumeGroupReportStruct - internalVolumeReport *internalVolumeReportStruct - reconEndpoint *ReconEndpointStruct - servingPeer *ServingPeerStruct - volume *VolumeStruct - volumeGroup *VolumeGroupStruct - ) - - if nil == internalObservingPeerReport { - externalObservingPeer = nil - return - } - - externalObservingPeer = &ObservingPeerStruct{ - Name: internalObservingPeerReport.name, - ServingPeer: make([]*ServingPeerStruct, 0, len(internalObservingPeerReport.servingPeer)), - ReconEndpoint: make([]*ReconEndpointStruct, 0, len(internalObservingPeerReport.reconEndpoint)), - } - - for _, internalServingPeerReport = range internalObservingPeerReport.servingPeer { - servingPeer = &ServingPeerStruct{ - Name: internalServingPeerReport.name, - State: internalServingPeerReport.state, - LastCheckTime: internalServingPeerReport.lastCheckTime, - VolumeGroup: make([]*VolumeGroupStruct, 0, len(internalServingPeerReport.volumeGroup)), - } - - for _, internalVolumeGroupReport = range internalServingPeerReport.volumeGroup { - volumeGroup = &VolumeGroupStruct{ - Name: internalVolumeGroupReport.name, - State: internalVolumeGroupReport.state, - LastCheckTime: internalVolumeGroupReport.lastCheckTime, - Volume: make([]*VolumeStruct, 0, len(internalVolumeGroupReport.volume)), - } - - for _, internalVolumeReport = range internalVolumeGroupReport.volume { - volume = &VolumeStruct{ - Name: internalVolumeReport.name, - State: internalVolumeReport.state, - LastCheckTime: internalVolumeReport.lastCheckTime, - } - - volumeGroup.Volume = append(volumeGroup.Volume, volume) - } - - servingPeer.VolumeGroup = append(servingPeer.VolumeGroup, volumeGroup) - } - - externalObservingPeer.ServingPeer = append(externalObservingPeer.ServingPeer, servingPeer) - } - - for _, internalReconEndpointReport = range internalObservingPeerReport.reconEndpoint { - reconEndpoint = &ReconEndpointStruct{ - IPAddrPort: internalReconEndpointReport.ipAddrPort, - MaxDiskUsagePercentage: internalReconEndpointReport.maxDiskUsagePercentage, - } - - externalObservingPeer.ReconEndpoint = append(externalObservingPeer.ReconEndpoint, reconEndpoint) - } - - return -} - -func convertExternalToInternalObservingPeerReport(externalObservingPeer *ObservingPeerStruct) (internalObservingPeerReport *internalObservingPeerReportStruct) { - var ( - internalReconEndpointReport *internalReconEndpointReportStruct - internalServingPeerReport *internalServingPeerReportStruct - internalVolumeGroupReport *internalVolumeGroupReportStruct - internalVolumeReport *internalVolumeReportStruct - reconEndpoint *ReconEndpointStruct - servingPeer *ServingPeerStruct - volume *VolumeStruct - volumeGroup *VolumeGroupStruct - ) - - if nil == externalObservingPeer { - internalObservingPeerReport = nil - return - } - - internalObservingPeerReport = &internalObservingPeerReportStruct{ - name: externalObservingPeer.Name, - servingPeer: make(map[string]*internalServingPeerReportStruct), - reconEndpoint: make(map[string]*internalReconEndpointReportStruct), - } - - for _, servingPeer = range externalObservingPeer.ServingPeer { - internalServingPeerReport = &internalServingPeerReportStruct{ - observingPeer: internalObservingPeerReport, - name: servingPeer.Name, - state: servingPeer.State, - lastCheckTime: servingPeer.LastCheckTime, - volumeGroup: make(map[string]*internalVolumeGroupReportStruct), - } - - for _, volumeGroup = range servingPeer.VolumeGroup { - internalVolumeGroupReport = &internalVolumeGroupReportStruct{ - servingPeer: internalServingPeerReport, - name: volumeGroup.Name, - state: volumeGroup.State, - lastCheckTime: volumeGroup.LastCheckTime, - volume: make(map[string]*internalVolumeReportStruct), - } - - for _, volume = range volumeGroup.Volume { - internalVolumeReport = &internalVolumeReportStruct{ - volumeGroup: internalVolumeGroupReport, - name: volume.Name, - state: volume.State, - lastCheckTime: volume.LastCheckTime, - } - - internalVolumeGroupReport.volume[internalVolumeReport.name] = internalVolumeReport - } - - internalServingPeerReport.volumeGroup[internalVolumeGroupReport.name] = internalVolumeGroupReport - } - - internalObservingPeerReport.servingPeer[internalServingPeerReport.name] = internalServingPeerReport - } - - for _, reconEndpoint = range externalObservingPeer.ReconEndpoint { - internalReconEndpointReport = &internalReconEndpointReportStruct{ - observingPeer: internalObservingPeerReport, - ipAddrPort: reconEndpoint.IPAddrPort, - maxDiskUsagePercentage: reconEndpoint.MaxDiskUsagePercentage, - } - - internalObservingPeerReport.reconEndpoint[internalReconEndpointReport.ipAddrPort] = internalReconEndpointReport - } - - return -} diff --git a/liveness/config.go b/liveness/config.go deleted file mode 100644 index f85d060b..00000000 --- a/liveness/config.go +++ /dev/null @@ -1,887 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package liveness - -import ( - "container/list" - "crypto/rand" - "fmt" - "hash/crc64" - "math" - "net" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -const ( - PrivateClusterUDPPortDefault = uint16(8123) - - UDPPacketSizeMin = uint64(1000) // Presumably >> udpPacketHeaderSize - UDPPacketSizeMax = uint64(8000) // Presumably >> udpPacketHeaderSize - - UDPPacketSendSizeDefault = uint64(1400) - UDPPacketRecvSizeDefault = uint64(1500) - - UDPPacketCapPerMessageDefault = uint8(math.MaxUint8) - - HeartBeatDurationDefault = "1s" - - HeartBeatMissLimitMin = uint64(2) - HeartBeatMissLimitDefault = uint64(3) - - MessageQueueDepthPerPeerMin = uint64(1) - MessageQueueDepthPerPeerDefault = uint64(4) - - MaxRequestDurationDefault = "1s" - - LivenessCheckRedundancyMin = uint64(1) - LivenessCheckRedundancyDefault = uint64(2) - - LogLevelNone = uint64(0) - LogLevelStateChanges = uint64(1) - LogLevelMessages = uint64(2) - LogLevelMessageDetails = uint64(3) - LogLevelMax = uint64(4) - - LogLevelDefault = LogLevelNone - - DefaultSwiftReconNoWriteThreshold = 80 - DefaultSwiftReconReadOnlyThreshold = 90 - DefaultSwiftConfDir = "/etc/swift" - DefaultSwiftReconChecksPerConfCheck = 10 -) - -type volumeStruct struct { - volumeGroup *volumeGroupStruct - name string - fuseMountPointName string - nfsExported bool - smbShareName string - accountName string -} - -type volumeGroupStruct struct { - peer *peerStruct // == nil if in globals.myVolumeGroupMap - name string - virtualIPAddr net.IP // Adopts the value of peer.publicIPAddr if not provided - volumeMap map[string]*volumeStruct // Key == volumeStruct.name -} - -type peerStruct struct { - name string - publicIPAddr net.IP - privateIPAddr net.IP - udpAddr *net.UDPAddr - curRecvMsgNonce uint64 - curRecvPacketCount uint8 - curRecvPacketSumSize uint64 - curRecvPacketMap map[uint8][]byte // Key is PacketIndex - prevRecvMsgQueueElement *recvMsgQueueElementStruct - incompleteRecvMsgMap map[uint64]*recvMsgQueueElementStruct // Key == recvMsgQueueElementStruct.msgNonce - incompleteRecvMsgQueue *list.List // LRU ordered - completeRecvMsgQueue *list.List // FIFO ordered - volumeGroupMap map[string]*volumeGroupStruct // Key == volumeGroupStruct.name -} - -type internalVolumeReportStruct struct { - volumeGroup *internalVolumeGroupReportStruct - name string - state string // One of const State{Alive|Dead|Unknown} - lastCheckTime time.Time -} - -type internalVolumeGroupReportStruct struct { - servingPeer *internalServingPeerReportStruct - name string - state string // One of const State{Alive|Dead|Unknown} - lastCheckTime time.Time - volume map[string]*internalVolumeReportStruct // Key = internalVolumeReportStruct.name -} - -type internalServingPeerReportStruct struct { - observingPeer *internalObservingPeerReportStruct - name string - state string // One of const State{Alive|Dead|Unknown} - lastCheckTime time.Time - volumeGroup map[string]*internalVolumeGroupReportStruct // Key = internalVolumeGroupReportStruct.name -} - -type internalReconEndpointReportStruct struct { - observingPeer *internalObservingPeerReportStruct - ipAddrPort string - maxDiskUsagePercentage uint8 -} - -type internalObservingPeerReportStruct struct { - name string - servingPeer map[string]*internalServingPeerReportStruct // Key = internalServingPeerReportStruct.name - reconEndpoint map[string]*internalReconEndpointReportStruct // Key = internalReconEndpointReportStruct.ipAddrPort -} - -type internalLivenessReportStruct struct { - observingPeer map[string]*internalObservingPeerReportStruct // Key = internalObservingPeerReportStruct.name -} - -type globalsStruct struct { - trackedlock.Mutex - active bool - enabled bool - whoAmI string - myPublicIPAddr net.IP - myPrivateIPAddr net.IP - myUDPAddr *net.UDPAddr - myUDPConn *net.UDPConn - myVolumeGroupMap map[string]*volumeGroupStruct // Key == volumeGroupStruct.name - peersByName map[string]*peerStruct // Key == peerStruct.name - peersByTuple map[string]*peerStruct // Key == peerStruct.udpAddr.String() (~= peerStruct.tuple) - udpPacketSendSize uint64 - udpPacketSendPayloadSize uint64 - udpPacketRecvSize uint64 - udpPacketRecvPayloadSize uint64 - udpPacketCapPerMessage uint8 - sendMsgMessageSizeMax uint64 - heartbeatDuration time.Duration - heartbeatMissLimit uint64 - heartbeatMissDuration time.Duration - messageQueueDepthPerPeer uint64 - maxRequestDuration time.Duration - livenessCheckRedundancy uint64 - logLevel uint64 - jsonRPCServerPort uint16 - swiftReconNoWriteThreshold uint8 - swiftReconReadOnlyThreshold uint8 - swiftConfDir string - swiftReconChecksPerConfCheck uint64 - swiftReconChecksUntilConfCheck uint64 - swiftConfFileMap map[string]time.Time // Key == os.FileInfo.Name(); Value == os.FileInfo.ModTime() - swiftReconEndpointSet map[string]struct{} // Key == IPAddrPort of ReconEndpoint - crc64ECMATable *crc64.Table - nextNonce uint64 // Randomly initialized... skips 0 - recvMsgsDoneChan chan struct{} - recvMsgQueue *list.List // FIFO ordered - recvMsgChan chan struct{} - requestsByExpirationTime *list.List // FIFO ordered - requestsByMsgTag map[uint64]*requestStruct // Key == requestStruct.msgTag - requestExpirerStartChan chan struct{} // Signaled when inserting the first element of requestsByExpirationTime - requestExpirerStopChan chan struct{} // Signaled when asking requestExpirer() to halt - requestExpirerDone sync.WaitGroup // Signaled when requestExpirer() has exited - currentLeader *peerStruct - currentVote *peerStruct - currentTerm uint64 - nextState func() - stateMachineStopChan chan struct{} - stateMachineDone sync.WaitGroup - livenessCheckerControlChan chan bool // Send true to trigger livenessChecker() to recompute polling schedule - // Send false to trigger livenessChecker() to exit - livenessCheckerWG sync.WaitGroup - volumeToCheckList []*volumeStruct - emptyVolumeGroupToCheckSet map[string]string // List (in "set" form) of VolumeGroups (by name) with no Volumes (Value == ServingPeer) - emptyServingPeerToCheckSet map[string]struct{} // List (in "set" form) of ServingPeers (by name) with no VolumeGroups - myObservingPeerReport *internalObservingPeerReportStruct - livenessReport *internalLivenessReportStruct - curRWMode inode.RWModeType -} - -var globals globalsStruct - -func init() { - transitions.Register("liveness", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - var ( - u64RandBuf []byte - ) - - // Ensure API behavior is disabled at startup - - globals.active = false - - // Do one-time initialization - - globals.crc64ECMATable = crc64.MakeTable(crc64.ECMA) - - u64RandBuf = make([]byte, 8) - _, err = rand.Read(u64RandBuf) - if nil != err { - err = fmt.Errorf("read.Rand() failed: %v", err) - return - } - globals.nextNonce = deserializeU64LittleEndian(u64RandBuf) - if 0 == globals.nextNonce { - globals.nextNonce = 1 - } - - globals.requestsByExpirationTime = list.New() - globals.requestsByMsgTag = make(map[uint64]*requestStruct) - globals.requestExpirerStartChan = make(chan struct{}, 1) - globals.requestExpirerStopChan = make(chan struct{}, 1) - - globals.curRWMode = inode.RWModeNormal - inode.SetRWMode(globals.curRWMode) - - globals.requestExpirerDone.Add(1) - go requestExpirer() - - globals.livenessCheckerControlChan = make(chan bool, 1) - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} - -// SignaledStart will be used to halt the cluster leadership process. This is to support -// SIGHUP handling incorporates all confMap changes are incorporated... not just during a restart. -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - var ( - stillDeactivating bool - ) - - // If the liveness checker is not enabled, stopping it will hang - - if !globals.enabled { - return - } - - // Disable API behavior as we enter the SIGHUP-handling state - - globals.active = false - - // Stop livenessChecker() - - globals.livenessCheckerControlChan <- false - globals.livenessCheckerWG.Wait() - - // Stop state machine - - globals.stateMachineStopChan <- struct{}{} - globals.stateMachineDone.Wait() - - // Shut off recvMsgs() - - err = globals.myUDPConn.Close() - if nil != err { - logger.Errorf("liveness.globals.myUDPConn.Close() failed: %v", err) - } - - stillDeactivating = true - - for stillDeactivating { - select { - case <-globals.recvMsgChan: - // Just discard it - case <-globals.recvMsgsDoneChan: - // Since recvMsgs() exited, we are done deactivating - stillDeactivating = false - } - } - - // Clear out Swift recon settings and computed details - - globals.swiftReconNoWriteThreshold = 101 // Never enforce NoWrite Mode - globals.swiftReconReadOnlyThreshold = 101 // Never enforce ReadOnly Mode - globals.swiftConfDir = "" - globals.swiftReconChecksPerConfCheck = 0 // Disabled - - // Free up remaining allocated resources - - globals.myVolumeGroupMap = nil - - globals.peersByName = nil - globals.peersByTuple = nil - - globals.recvMsgQueue = list.New() - - globals.myObservingPeerReport = nil - globals.livenessReport = nil - - globals.volumeToCheckList = nil - globals.emptyVolumeGroupToCheckSet = nil - globals.emptyServingPeerToCheckSet = nil - - globals.myObservingPeerReport = nil - globals.livenessReport = nil - - err = nil - return -} - -// SignaledFinish will be used to kick off the cluster leadership process. This is to support -// SIGHUP handling incorporates all confMap changes are incorporated... not just during a restart. -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - var ( - myTuple string - ok bool - enabled bool - peer *peerStruct - peerName string - peerList []string - peerTuple string - privateClusterUDPPortAsString string - privateClusterUDPPortAsUint16 uint16 - privateIPAddr string - publicIPAddr string - virtualIPAddr string - volume *volumeStruct - volumeGroup *volumeGroupStruct - volumeGroupList []string - volumeGroupName string - volumeList []string - volumeName string - ) - - // see if liveness checker is enabled - globals.enabled = false - enabled, err = confMap.FetchOptionValueBool("Cluster", "LivenessCheckerEnabled") - if nil != err { - logger.InfoWithError(err, "Unable to find and/or parse [Cluster]LivenessCheckerEnabled;"+ - " defaulting to disabled") - err = nil - return - } - if !enabled { - logger.Infof("Liveness checker disabled") - return - } - - // don't set globals.enabled = true until it's actually started - logger.Infof("Liveness checker will be enabled") - - // Fetch cluster parameters - - privateClusterUDPPortAsUint16, err = confMap.FetchOptionValueUint16("Cluster", "PrivateClusterUDPPort") - if nil != err { - privateClusterUDPPortAsUint16 = PrivateClusterUDPPortDefault // TODO: Eventually just return - } - privateClusterUDPPortAsString = fmt.Sprintf("%d", privateClusterUDPPortAsUint16) - - globals.whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - return - } - - publicIPAddr, err = confMap.FetchOptionValueString("Peer:"+globals.whoAmI, "PublicIPAddr") - if nil != err { - return - } - - globals.myPublicIPAddr = net.ParseIP(publicIPAddr) - if nil == globals.myPublicIPAddr { - err = fmt.Errorf("Unable to parse myPublicIPAddr") - return - } - - privateIPAddr, err = confMap.FetchOptionValueString("Peer:"+globals.whoAmI, "PrivateIPAddr") - if nil != err { - return - } - - globals.myPrivateIPAddr = net.ParseIP(privateIPAddr) - if nil == globals.myPrivateIPAddr { - err = fmt.Errorf("Unable to parse myPrivateIPAddr") - return - } - - myTuple = net.JoinHostPort(privateIPAddr, privateClusterUDPPortAsString) - - globals.myUDPAddr, err = net.ResolveUDPAddr("udp", myTuple) - if nil != err { - err = fmt.Errorf("Cannot parse myTuple (%s): %v", myTuple, err) - return - } - - globals.myUDPConn, err = net.ListenUDP("udp", globals.myUDPAddr) - if nil != err { - err = fmt.Errorf("Cannot bind to myTuple (%v): %v", globals.myUDPAddr, err) - return - } - - globals.myVolumeGroupMap = make(map[string]*volumeGroupStruct) - - peerList, err = confMap.FetchOptionValueStringSlice("Cluster", "Peers") - if nil != err { - return - } - - globals.peersByName = make(map[string]*peerStruct) - globals.peersByTuple = make(map[string]*peerStruct) - - // Initialize emptyServingPeerToCheckSet with all ServingPeers (including self) - // This set will be pruned later as VolumeGroups are assigned to a ServingPeer - - globals.emptyServingPeerToCheckSet = make(map[string]struct{}) - - for _, peerName = range peerList { - globals.emptyServingPeerToCheckSet[peerName] = struct{}{} - - if peerName != globals.whoAmI { - peer = &peerStruct{ - name: peerName, - curRecvMsgNonce: 0, - curRecvPacketCount: 0, - curRecvPacketSumSize: 0, - curRecvPacketMap: nil, - prevRecvMsgQueueElement: nil, - incompleteRecvMsgMap: make(map[uint64]*recvMsgQueueElementStruct), - incompleteRecvMsgQueue: list.New(), - completeRecvMsgQueue: list.New(), - volumeGroupMap: make(map[string]*volumeGroupStruct), - } - - publicIPAddr, err = confMap.FetchOptionValueString("Peer:"+peerName, "PublicIPAddr") - if nil != err { - return - } - - peer.publicIPAddr = net.ParseIP(publicIPAddr) - if nil == peer.publicIPAddr { - err = fmt.Errorf("Cannot parse [Peer:%v]PublicIPAddr", peerName) - return - } - - privateIPAddr, err = confMap.FetchOptionValueString("Peer:"+peerName, "PrivateIPAddr") - if nil != err { - return - } - - peer.privateIPAddr = net.ParseIP(privateIPAddr) - if nil == peer.privateIPAddr { - err = fmt.Errorf("Cannot parse [Peer:%v]PrivateIPAddr", peerName) - return - } - - peerTuple = net.JoinHostPort(privateIPAddr, privateClusterUDPPortAsString) - - peer.udpAddr, err = net.ResolveUDPAddr("udp", peerTuple) - if nil != err { - err = fmt.Errorf("Cannot parse peerTuple (%s): %v", peerTuple, err) - return - } - - if globals.myUDPAddr.String() == peer.udpAddr.String() { - err = fmt.Errorf("peerTuple cannot match myTuple (%v)", globals.myUDPAddr) - return - } - _, ok = globals.peersByName[peer.name] - if ok { - err = fmt.Errorf("peerName must not match multiple peers (%v)", peer.name) - return - } - _, ok = globals.peersByTuple[peer.udpAddr.String()] - if ok { - err = fmt.Errorf("peerTuple must not match multiple peers (%v)", peer.udpAddr) - return - } - - globals.peersByName[peer.name] = peer - globals.peersByTuple[peer.udpAddr.String()] = peer - } - } - - globals.udpPacketSendSize, err = confMap.FetchOptionValueUint64("Cluster", "UDPPacketSendSize") - if nil != err { - globals.udpPacketSendSize = UDPPacketSendSizeDefault // TODO: Eventually just return - } - if (globals.udpPacketSendSize < UDPPacketSizeMin) || (globals.udpPacketSendSize > UDPPacketSizeMax) { - err = fmt.Errorf("udpPacketSendSize (%v) must be between %v and %v (inclusive)", globals.udpPacketSendSize, UDPPacketSizeMin, UDPPacketSizeMax) - return - } - - globals.udpPacketSendPayloadSize = globals.udpPacketSendSize - udpPacketHeaderSize - - globals.udpPacketRecvSize, err = confMap.FetchOptionValueUint64("Cluster", "UDPPacketRecvSize") - if nil != err { - globals.udpPacketRecvSize = UDPPacketRecvSizeDefault // TODO: Eventually just return - } - if (globals.udpPacketRecvSize < UDPPacketSizeMin) || (globals.udpPacketRecvSize > UDPPacketSizeMax) { - err = fmt.Errorf("udpPacketRecvSize (%v) must be between %v and %v (inclusive)", globals.udpPacketRecvSize, UDPPacketSizeMin, UDPPacketSizeMax) - return - } - - globals.udpPacketRecvPayloadSize = globals.udpPacketRecvSize - udpPacketHeaderSize - - globals.udpPacketCapPerMessage, err = confMap.FetchOptionValueUint8("Cluster", "UDPPacketCapPerMessage") - if nil != err { - globals.udpPacketCapPerMessage = UDPPacketCapPerMessageDefault // TODO: Eventually just return - } - if 0 == globals.udpPacketCapPerMessage { - err = fmt.Errorf("udpPacketCapPerMessage must be non-zero") - return - } - - globals.sendMsgMessageSizeMax = uint64(globals.udpPacketCapPerMessage) * globals.udpPacketSendPayloadSize - - globals.heartbeatDuration, err = confMap.FetchOptionValueDuration("Cluster", "HeartBeatDuration") - if nil != err { - // TODO: Eventually just return - globals.heartbeatDuration, err = time.ParseDuration(HeartBeatDurationDefault) - if nil != err { - return - } - } - if time.Duration(0) == globals.heartbeatDuration { - err = fmt.Errorf("heartbeatDuration must be non-zero") - return - } - - globals.heartbeatMissLimit, err = confMap.FetchOptionValueUint64("Cluster", "HeartBeatMissLimit") - if nil != err { - globals.heartbeatMissLimit = HeartBeatMissLimitDefault // TODO: Eventually just return - } - if globals.heartbeatMissLimit < HeartBeatMissLimitMin { - err = fmt.Errorf("heartbeatMissLimit (%v) must be at least %v", globals.heartbeatMissLimit, HeartBeatMissLimitMin) - return - } - - globals.heartbeatMissDuration = time.Duration(globals.heartbeatMissLimit) * globals.heartbeatDuration - - globals.messageQueueDepthPerPeer, err = confMap.FetchOptionValueUint64("Cluster", "MessageQueueDepthPerPeer") - if nil != err { - globals.messageQueueDepthPerPeer = MessageQueueDepthPerPeerDefault // TODO: Eventually just return - } - if globals.messageQueueDepthPerPeer < MessageQueueDepthPerPeerMin { - err = fmt.Errorf("messageQueueDepthPerPeer (%v) must be at least %v", globals.messageQueueDepthPerPeer, MessageQueueDepthPerPeerMin) - return - } - - globals.maxRequestDuration, err = confMap.FetchOptionValueDuration("Cluster", "MaxRequestDuration") - if nil != err { - // TODO: Eventually just return - globals.maxRequestDuration, err = time.ParseDuration(MaxRequestDurationDefault) - if nil != err { - return - } - } - if time.Duration(0) == globals.maxRequestDuration { - err = fmt.Errorf("maxRequestDuration must be non-zero") - return - } - - globals.livenessCheckRedundancy, err = confMap.FetchOptionValueUint64("Cluster", "LivenessCheckRedundancy") - if nil != err { - globals.livenessCheckRedundancy = LivenessCheckRedundancyDefault // TODO: Eventually just return - } - if globals.livenessCheckRedundancy < LivenessCheckRedundancyMin { - err = fmt.Errorf("livenessCheckRedundancy (%v) must be at least %v", globals.livenessCheckRedundancy, LivenessCheckRedundancyMin) - return - } - - // Set LogLevel as specified or use default - - globals.logLevel, err = confMap.FetchOptionValueUint64("Cluster", "LogLevel") - if nil != err { - globals.logLevel = LogLevelDefault - } - if globals.logLevel > LogLevelMax { - err = fmt.Errorf("logLevel (%v) must be between 0 and %v (inclusive)", globals.logLevel, LogLevelMax) - return - } - - // Record current Peer->VolumeGroup->Volume mapping - - globals.volumeToCheckList = make([]*volumeStruct, 0) - - volumeGroupList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - return - } - - globals.emptyVolumeGroupToCheckSet = make(map[string]string) - - for _, volumeGroupName = range volumeGroupList { - virtualIPAddr, err = confMap.FetchOptionValueString("VolumeGroup:"+volumeGroupName, "VirtualIPAddr") - if nil != err { - virtualIPAddr = "" - } - - peerList, err = confMap.FetchOptionValueStringSlice("VolumeGroup:"+volumeGroupName, "PrimaryPeer") - if nil != err { - return - } - - switch len(peerList) { - case 0: - // Since VolumeGroup has no PrimaryPeer, just skip it - case 1: - // Include this VolumeGroup - - peerName = peerList[0] - - delete(globals.emptyServingPeerToCheckSet, peerName) - - if peerName == globals.whoAmI { - _, ok = globals.myVolumeGroupMap[volumeGroupName] - if ok { - err = fmt.Errorf("Duplicate VolumeGroup (%v) not allowed", volumeGroupName) - return - } - - volumeGroup = &volumeGroupStruct{ - peer: nil, - name: volumeGroupName, - volumeMap: make(map[string]*volumeStruct), - } - - if "" == virtualIPAddr { - volumeGroup.virtualIPAddr = globals.myPublicIPAddr - } else { - - // virtualIPAddr must be a valid IP address or valid - // IP address in CIDR notation - volumeGroup.virtualIPAddr = net.ParseIP(virtualIPAddr) - if nil == volumeGroup.virtualIPAddr { - - volumeGroup.virtualIPAddr, _, err = net.ParseCIDR(virtualIPAddr) - if err != nil { - err = fmt.Errorf("Cannot parse [VolumeGroup:%v]VirtualIPAddr: '%s' "+ - " as IP address or CIDR IP address: %v", - volumeGroupName, virtualIPAddr, err) - return - } - } - } - - globals.myVolumeGroupMap[volumeGroupName] = volumeGroup - } else { - peer, ok = globals.peersByName[peerName] - if !ok { - err = fmt.Errorf("[VolumeGroup:%v]PrimaryPeer (%v) not found in [Cluster]Peers", volumeGroupName, peerName) - return - } - - _, ok = peer.volumeGroupMap[volumeGroupName] - if ok { - err = fmt.Errorf("Duplicate VolumeGroup (%v) not allowed", volumeGroupName) - return - } - - volumeGroup = &volumeGroupStruct{ - peer: peer, - name: volumeGroupName, - volumeMap: make(map[string]*volumeStruct), - } - - if "" == virtualIPAddr { - volumeGroup.virtualIPAddr = peer.publicIPAddr - } else { - - // virtualIPAddr must be a valid IP address or valid - // IP address in CIDR notation - volumeGroup.virtualIPAddr = net.ParseIP(virtualIPAddr) - if nil == volumeGroup.virtualIPAddr { - - volumeGroup.virtualIPAddr, _, err = net.ParseCIDR(virtualIPAddr) - if err != nil { - err = fmt.Errorf("Cannot parse [VolumeGroup:%v]VirtualIPAddr: '%s' "+ - " as IP address or CIDR IP address: %v", - volumeGroupName, virtualIPAddr, err) - return - } - } - } - - peer.volumeGroupMap[volumeGroupName] = volumeGroup - } - - volumeList, err = confMap.FetchOptionValueStringSlice("VolumeGroup:"+volumeGroupName, "VolumeList") - if nil != err { - return - } - - if 0 == len(volumeList) { - globals.emptyVolumeGroupToCheckSet[volumeGroupName] = peerName - } else { - for _, volumeName = range volumeList { - _, ok = volumeGroup.volumeMap[volumeName] - if ok { - err = fmt.Errorf("[VolumeGroup:%v]VolumeList contains Volume %v more than once", volumeGroupName, volumeName) - return - } - - volume = &volumeStruct{ - volumeGroup: volumeGroup, - name: volumeName, - } - - volume.fuseMountPointName, err = confMap.FetchOptionValueString("Volume:"+volumeName, "FUSEMountPointName") - if nil != err { - return - } - - volume.nfsExported, err = confMap.FetchOptionValueBool("Volume:"+volumeName, "NFSExported") - if nil != err { - // Default to no NFS Export - volume.nfsExported = false - } - - volume.smbShareName, err = confMap.FetchOptionValueString("Volume:"+volumeName, "SMBShareName") - if nil != err { - // Default to no SMB Share - volume.smbShareName = "" - } - - volume.accountName, err = confMap.FetchOptionValueString("Volume:"+volumeName, "AccountName") - if nil != err { - return - } - - volumeGroup.volumeMap[volumeName] = volume - - globals.volumeToCheckList = append(globals.volumeToCheckList, volume) - } - } - default: - err = fmt.Errorf("[VolumeGroup:%s]PrimaryPeer must be empty or single-valued", volumeGroupName) - return - } - } - - // Fetch JSON RPC Port to be used when polling Peers - - globals.jsonRPCServerPort, err = confMap.FetchOptionValueUint16("JSONRPCServer", "TCPPort") - if nil != err { - return - } - - // Fetch Swift recon settings - - err = confMap.VerifyOptionIsMissing("SwiftClient", "SwiftReconChecksPerConfCheck") - if nil == err { - logger.WarnfWithError(err, "Unable to fetch [SwiftClient]SwiftReconChecksPerConfCheck... defaulting to %d", DefaultSwiftReconChecksPerConfCheck) - globals.swiftReconChecksPerConfCheck = DefaultSwiftReconChecksPerConfCheck - } else { - globals.swiftReconChecksPerConfCheck, err = confMap.FetchOptionValueUint64("SwiftClient", "SwiftReconChecksPerConfCheck") - if nil != err { - logger.ErrorfWithError(err, "Unable to parse [SwiftClient]SwiftReconChecksPerConfCheck") - return - } - } - - if 0 == globals.swiftReconChecksPerConfCheck { - logger.Warnf("[SwiftClient]SwiftReconChecksPerConfCheck == 0... disabling recon checks") - } else { - globals.swiftReconNoWriteThreshold, err = confMap.FetchOptionValueUint8("SwiftClient", "SwiftReconNoWriteThreshold") - if nil == err { - if 100 < globals.swiftReconNoWriteThreshold { - err = fmt.Errorf("[SwiftClient]SwiftReconNoWriteThreshold cannot be greater than 100") - return - } - } else { - logger.WarnfWithError(err, "Unable to fetch [SwiftClient]SwiftReconNoWriteThreshold... defaulting to %d", DefaultSwiftReconNoWriteThreshold) - globals.swiftReconNoWriteThreshold = DefaultSwiftReconNoWriteThreshold - } - - globals.swiftReconReadOnlyThreshold, err = confMap.FetchOptionValueUint8("SwiftClient", "SwiftReconReadOnlyThreshold") - if nil == err { - if 100 < globals.swiftReconReadOnlyThreshold { - err = fmt.Errorf("[SwiftClient]SwiftReconReadOnlyThreshold cannot be greater than 100") - return - } - if globals.swiftReconReadOnlyThreshold < globals.swiftReconNoWriteThreshold { - err = fmt.Errorf("[SwiftClient]SwiftReconReadOnlyThreshold cannot be less than [SwiftClient]SwiftReconNoWriteThreshold") - return - } - } else { - if globals.swiftReconNoWriteThreshold > DefaultSwiftReconReadOnlyThreshold { - logger.WarnfWithError(err, "Unable to fetch [SwiftClient]SwiftReconReadOnlyThreadhold... defaulting to %d", globals.swiftReconNoWriteThreshold) - globals.swiftReconReadOnlyThreshold = globals.swiftReconNoWriteThreshold - } else { - logger.WarnfWithError(err, "Unable to fetch [SwiftClient]SwiftReconReadOnlyThreadhold... defaulting to %d", DefaultSwiftReconReadOnlyThreshold) - globals.swiftReconReadOnlyThreshold = DefaultSwiftReconReadOnlyThreshold - } - } - - globals.swiftConfDir, err = confMap.FetchOptionValueString("SwiftClient", "SwiftConfDir") - if nil != err { - logger.WarnfWithError(err, "Unable to fetch [SwiftClient]SwiftConfDir... defaulting to %s", DefaultSwiftConfDir) - globals.swiftConfDir = DefaultSwiftConfDir - } - } - - // the liveness checker will be enabled (no more error out cases) - - globals.enabled = true - - // Initialize remaining globals - - globals.swiftReconChecksUntilConfCheck = 0 // First ReconCheck will trigger a ConfCheck - globals.swiftConfFileMap = make(map[string]time.Time) - - globals.recvMsgQueue = list.New() - - globals.recvMsgChan = make(chan struct{}, 1) - - globals.recvMsgsDoneChan = make(chan struct{}, 1) - go recvMsgs() - - globals.currentLeader = nil - globals.currentVote = nil - globals.currentTerm = 0 - - globals.nextState = doFollower - - globals.stateMachineStopChan = make(chan struct{}, 1) - - // Initialize internal Liveness Report data as being empty - - globals.myObservingPeerReport = nil - globals.livenessReport = nil - - // Start up livenessChecker() - - globals.livenessCheckerWG.Add(1) - go livenessChecker() - - // Become an active participant in the cluster - - globals.stateMachineDone.Add(1) - go stateMachine() - - // Enable API behavior as we leave the SIGHUP-handling state - - globals.active = true - - err = nil - return -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - globals.requestExpirerStopChan <- struct{}{} - globals.requestExpirerDone.Wait() - - return nil -} diff --git a/liveness/messages.go b/liveness/messages.go deleted file mode 100644 index 3337b2c0..00000000 --- a/liveness/messages.go +++ /dev/null @@ -1,597 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package liveness - -import ( - "container/list" - "encoding/json" - "fmt" - "hash/crc64" - "net" - "reflect" - "time" - - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" -) - -// Each UDP Packet Header is made up of (also in LittleEndian byte order): -// CRC64 uint64 -// MsgNonce uint64 -// PacketIndex uint8 -// PacketCount uint8 - -const udpPacketHeaderSize uint64 = 8 + 8 + 1 + 1 // sizeof(CRC64) + sizeof(MsgNonce) + sizeof(PacketIndex) + sizeof(PacketCount) - -// Every msg is JSON-encoded - -type MsgType uint8 - -const ( - MsgTypeHeartBeatRequest MsgType = iota + 1 // Skip zero to avoid msg's missing the MsgType field - MsgTypeHeartBeatResponse - MsgTypeRequestVoteRequest - MsgTypeRequestVoteResponse - MsgTypeFetchLivenessReportRequest - MsgTypeFetchLivenessReportResponse -) - -type MsgTypeStruct struct { - MsgType MsgType - MsgTag uint64 // Used for matching a *RequestStruct to a subsequent *ResponseStruct -} - -type HeartBeatRequestStruct struct { - MsgType MsgType // == MsgTypeHeartBeatRequest - MsgTag uint64 // Used for matching this HeartBeatRequestStruct to a subsequent HeartBeatResponseStruct - LeaderTerm uint64 - NewRWMode inode.RWModeType // One of inode.RWModeNormal, inode.RWModeNoWrite, or inode.RWModeReadOnly - ToObserve *ObservingPeerStruct // VolumeStruct.State & VolumeStruct.LastCheckTime are ignored -} - -type HeartBeatResponseStruct struct { - MsgType MsgType // == MsgTypeHeartBeatResponse - MsgTag uint64 // Used for matching this HeartBeatResponseStruct to a previous HeartBeatRequestStruct - CurrentTerm uint64 - Success bool - Observed *ObservingPeerStruct -} - -type RequestVoteRequestStruct struct { - MsgType MsgType // == MsgTypeRequestVoteRequest - MsgTag uint64 // Used for matching this RequestVoteRequestStruct to a subsequent RequestVoteResponseStruct - CandidateTerm uint64 -} - -type RequestVoteResponseStruct struct { - MsgType MsgType // == MsgTypeRequestVoteResponse - MsgTag uint64 // Used for matching this RequestVoteResponseStruct to a previous RequestVoteRequestStruct - CurrentTerm uint64 - VoteGranted bool -} - -type FetchLivenessReportRequestStruct struct { - MsgType MsgType // == MsgTypeFetchLivenessReportRequest - // Used to request Liveness Report from who we think is the Leader - MsgTag uint64 // Used for matching this FetchLivenessReportRequestStruct to a subsequent FetchLivenessReportResponseStruct - CurrentTerm uint64 -} - -type FetchLivenessReportResponseStruct struct { - MsgType MsgType // == MsgTypeFetchLivenessReportResponse - MsgTag uint64 // Used for matching this FetchLivenessReportResponseStruct to a previous FetchLivenessReportRequestStruct - CurrentTerm uint64 // == LeaderTerm if Success === true (by definition) - CurrentLeader string // If Success == false, this is who should actually be contacted for this (if known) - Success bool // == true if Leader is responding; == false if we are not the Leader - LivenessReport *LivenessReportStruct // Liveness Report as collected by Leader -} - -type recvMsgQueueElementStruct struct { - peer *peerStruct - msgNonce uint64 - packetCount uint8 - packetSumSize uint64 - packetMap map[uint8][]byte // Key == PacketIndex - peerRecvMsgQueueElement *list.Element - globalRecvMsgQueueElement *list.Element - msgType MsgType // Even though it's inside msg, make it easier to decode - msgTag uint64 // Even though it's inside msg, make it easier to decode - msg interface{} // Must be a pointer to one of the above Msg structs (other than CommonMsgHeaderStruct) -} - -type requestStruct struct { - element *list.Element - msgTag uint64 - expirationTime time.Time - expired bool - requestContext interface{} - requestMsg interface{} - responseMsg interface{} - callback func(request *requestStruct) // If expired == true, responseMsg will be nil -} - -func fetchNonceWhileLocked() (nonce uint64) { - nonce = globals.nextNonce - globals.nextNonce++ - if 0 == globals.nextNonce { - globals.nextNonce = 1 - } - return -} - -func fetchNonce() (nonce uint64) { - globals.Lock() - nonce = fetchNonceWhileLocked() - globals.Unlock() - return -} - -func serializeU64LittleEndian(u64 uint64) (u64Buf []byte) { - u64Buf = make([]byte, 8) - u64Buf[0] = byte(u64 & 0xFF) - u64Buf[1] = byte((u64 >> 8) & 0xFF) - u64Buf[2] = byte((u64 >> 16) & 0xFF) - u64Buf[3] = byte((u64 >> 24) & 0xFF) - u64Buf[4] = byte((u64 >> 32) & 0xFF) - u64Buf[5] = byte((u64 >> 40) & 0xFF) - u64Buf[6] = byte((u64 >> 48) & 0xFF) - u64Buf[7] = byte((u64 >> 56) & 0xFF) - return -} - -func deserializeU64LittleEndian(u64Buf []byte) (u64 uint64) { - u64 = uint64(u64Buf[7]) - u64 = (u64 << 8) | uint64(u64Buf[6]) - u64 = (u64 << 8) | uint64(u64Buf[5]) - u64 = (u64 << 8) | uint64(u64Buf[4]) - u64 = (u64 << 8) | uint64(u64Buf[3]) - u64 = (u64 << 8) | uint64(u64Buf[2]) - u64 = (u64 << 8) | uint64(u64Buf[1]) - u64 = (u64 << 8) | uint64(u64Buf[0]) - return -} - -func appendGlobalRecvMsgQueueElementWhileLocked(recvMsgQueueElement *recvMsgQueueElementStruct) { - recvMsgQueueElement.globalRecvMsgQueueElement = globals.recvMsgQueue.PushBack(recvMsgQueueElement) -} -func appendGlobalRecvMsgQueueElement(recvMsgQueueElement *recvMsgQueueElementStruct) { - globals.Lock() - appendGlobalRecvMsgQueueElementWhileLocked(recvMsgQueueElement) - globals.Unlock() -} - -func removeGlobalRecvMsgQueueElementWhileLocked(recvMsgQueueElement *recvMsgQueueElementStruct) { - _ = globals.recvMsgQueue.Remove(recvMsgQueueElement.globalRecvMsgQueueElement) -} -func removeGlobalRecvMsgQueueElement(recvMsgQueueElement *recvMsgQueueElementStruct) { - globals.Lock() - removeGlobalRecvMsgQueueElementWhileLocked(recvMsgQueueElement) - globals.Unlock() -} - -func popGlobalMsgWhileLocked() (recvMsgQueueElement *recvMsgQueueElementStruct) { - if 0 == globals.recvMsgQueue.Len() { - recvMsgQueueElement = nil - } else { - recvMsgQueueElement = globals.recvMsgQueue.Front().Value.(*recvMsgQueueElementStruct) - recvMsgQueueElement.peer.completeRecvMsgQueue.Remove(recvMsgQueueElement.peerRecvMsgQueueElement) - removeGlobalRecvMsgQueueElementWhileLocked(recvMsgQueueElement) - } - return -} -func popGlobalMsg() (recvMsgQueueElement *recvMsgQueueElementStruct) { - globals.Lock() - recvMsgQueueElement = popGlobalMsgWhileLocked() - globals.Unlock() - return -} - -func recvMsgs() { - var ( - computedCRC64 uint64 - err error - msgBuf []byte - msgNonce uint64 - msgTypeStruct MsgTypeStruct - ok bool - packetBuf []byte - packetCount uint8 - packetIndex uint8 - packetSize int - peer *peerStruct - receivedCRC64 uint64 - recvMsgQueueElement *recvMsgQueueElementStruct - udpAddr *net.UDPAddr - ) - - for { - // Read next packet - - packetBuf = make([]byte, globals.udpPacketRecvSize) - - packetSize, udpAddr, err = globals.myUDPConn.ReadFromUDP(packetBuf) - - if nil != err { - globals.recvMsgsDoneChan <- struct{}{} - return - } - - // Decode packet header - - if uint64(packetSize) < udpPacketHeaderSize { - continue // Ignore it - } - - packetBuf = packetBuf[:packetSize] - - receivedCRC64 = deserializeU64LittleEndian(packetBuf[:8]) - msgNonce = deserializeU64LittleEndian(packetBuf[8:16]) - packetIndex = packetBuf[16] - packetCount = packetBuf[17] - - // Validate packet - - computedCRC64 = crc64.Checksum(packetBuf[8:], globals.crc64ECMATable) - - if receivedCRC64 != computedCRC64 { - continue // Ignore it - } - - if 0 == msgNonce { - continue // Ignore it - } - - if packetIndex >= packetCount { - continue // Ignore it - } - - if packetCount > globals.udpPacketCapPerMessage { - continue // Ignore it - } - - // Locate peer - - globals.Lock() - peer, ok = globals.peersByTuple[udpAddr.String()] - globals.Unlock() - - if !ok { - continue // Ignore it - } - - // Check if packet is part of a new msg - - recvMsgQueueElement, ok = peer.incompleteRecvMsgMap[msgNonce] - if ok { - // Packet is part of an existing incomplete msg - - peer.incompleteRecvMsgQueue.MoveToBack(recvMsgQueueElement.peerRecvMsgQueueElement) - - if packetCount != recvMsgQueueElement.packetCount { - // Forget prior msg packets and start receiving a new msg with this packet - - recvMsgQueueElement.packetCount = packetCount - recvMsgQueueElement.packetSumSize = uint64(len(packetBuf[18:])) - recvMsgQueueElement.packetMap = make(map[uint8][]byte) - - recvMsgQueueElement.packetMap[packetIndex] = packetBuf[18:] - } else { - // Update existing incomplete msg with this packet - - _, ok = recvMsgQueueElement.packetMap[packetIndex] - if ok { - continue // Ignore it - } - - recvMsgQueueElement.packetSumSize += uint64(len(packetBuf[18:])) - recvMsgQueueElement.packetMap[packetIndex] = packetBuf[18:] - } - } else { - // Packet is part of a new msg - - if uint64(peer.incompleteRecvMsgQueue.Len()) >= globals.messageQueueDepthPerPeer { - // Make room for this new msg in .incompleteRecvMsgQueue - - recvMsgQueueElement = peer.incompleteRecvMsgQueue.Front().Value.(*recvMsgQueueElementStruct) - delete(peer.incompleteRecvMsgMap, recvMsgQueueElement.msgNonce) - _ = peer.incompleteRecvMsgQueue.Remove(recvMsgQueueElement.peerRecvMsgQueueElement) - } - - // Construct a new recvMsgQueueElement - - recvMsgQueueElement = &recvMsgQueueElementStruct{ - peer: peer, - msgNonce: msgNonce, - packetCount: packetCount, - packetSumSize: uint64(len(packetBuf[18:])), - packetMap: make(map[uint8][]byte), - } - - recvMsgQueueElement.packetMap[packetIndex] = packetBuf[18:] - - peer.incompleteRecvMsgMap[recvMsgQueueElement.msgNonce] = recvMsgQueueElement - recvMsgQueueElement.peerRecvMsgQueueElement = peer.incompleteRecvMsgQueue.PushBack(recvMsgQueueElement) - } - - // Have all packets of msg been received? - - if len(recvMsgQueueElement.packetMap) == int(recvMsgQueueElement.packetCount) { - // All packets received... assemble completed msg - - delete(peer.incompleteRecvMsgMap, recvMsgQueueElement.msgNonce) - _ = peer.incompleteRecvMsgQueue.Remove(recvMsgQueueElement.peerRecvMsgQueueElement) - - msgBuf = make([]byte, 0, recvMsgQueueElement.packetSumSize) - - for packetIndex = 0; packetIndex < recvMsgQueueElement.packetCount; packetIndex++ { - msgBuf = append(msgBuf, recvMsgQueueElement.packetMap[packetIndex]...) - } - - // Decode the msg - - msgTypeStruct = MsgTypeStruct{} - - err = json.Unmarshal(msgBuf, &msgTypeStruct) - if nil != err { - continue // Ignore it - } - - recvMsgQueueElement.msgType = msgTypeStruct.MsgType - recvMsgQueueElement.msgTag = msgTypeStruct.MsgTag - - switch recvMsgQueueElement.msgType { - case MsgTypeHeartBeatRequest: - recvMsgQueueElement.msg = &HeartBeatRequestStruct{} - case MsgTypeHeartBeatResponse: - recvMsgQueueElement.msg = &HeartBeatResponseStruct{} - case MsgTypeRequestVoteRequest: - recvMsgQueueElement.msg = &RequestVoteRequestStruct{} - case MsgTypeRequestVoteResponse: - recvMsgQueueElement.msg = &RequestVoteResponseStruct{} - case MsgTypeFetchLivenessReportRequest: - recvMsgQueueElement.msg = &FetchLivenessReportRequestStruct{} - case MsgTypeFetchLivenessReportResponse: - recvMsgQueueElement.msg = &FetchLivenessReportResponseStruct{} - default: - continue // Ignore it - } - - err = json.Unmarshal(msgBuf, recvMsgQueueElement.msg) - if nil != err { - continue // Ignore it - } - - // Deliver msg - - globals.Lock() - - recvMsgQueueElement.peerRecvMsgQueueElement = peer.completeRecvMsgQueue.PushBack(recvMsgQueueElement) - - appendGlobalRecvMsgQueueElementWhileLocked(recvMsgQueueElement) - - globals.Unlock() - - globals.recvMsgChan <- struct{}{} - - // Log delivery if requested - - if LogLevelMessages <= globals.logLevel { - if LogLevelMessageDetails > globals.logLevel { - logger.Infof("%s rec'd %s from %s", globals.myUDPAddr, reflect.TypeOf(recvMsgQueueElement.msg), udpAddr) - } else { - logger.Infof("%s rec'd %s from %s [%#v]", globals.myUDPAddr, reflect.TypeOf(recvMsgQueueElement.msg), udpAddr, recvMsgQueueElement.msg) - } - } - } - } -} - -// sendMsg JSON-encodes msg and sends it to peer (all peers if nil == peer) -func sendMsg(peer *peerStruct, msg interface{}) (peers []*peerStruct, err error) { - var ( - computedCRC64 uint64 - computedCRC64Buf []byte - loggerInfofBuf string - msgBuf []byte - msgBufOffset uint64 - msgBufSize uint64 - msgNonce uint64 - msgNonceBuf []byte - packetBuf []byte - packetBufIndex uint64 - packetCount uint8 - packetIndex uint8 - ) - - if nil == peer { - globals.Lock() - peers = make([]*peerStruct, 0, len(globals.peersByTuple)) - for _, peer = range globals.peersByTuple { - peers = append(peers, peer) - } - globals.Unlock() - } else { - peers = make([]*peerStruct, 1) - peers[0] = peer - } - - msgNonce = fetchNonce() - - msgBuf, err = json.Marshal(msg) - if nil != err { - return - } - - msgBufSize = uint64(len(msgBuf)) - - if msgBufSize > globals.sendMsgMessageSizeMax { - err = fmt.Errorf("sendMsg() called for excessive len(msgBuf) == %v (must be <= %v)", msgBufSize, globals.sendMsgMessageSizeMax) - return - } - - packetCount = uint8((msgBufSize + globals.udpPacketSendPayloadSize - 1) / globals.udpPacketSendPayloadSize) - - msgNonceBuf = serializeU64LittleEndian(msgNonce) - - msgBufOffset = 0 - - for packetIndex = 0; packetIndex < packetCount; packetIndex++ { - if packetIndex < (packetCount - 1) { - packetBuf = make([]byte, udpPacketHeaderSize, globals.udpPacketSendSize) - packetBuf = append(packetBuf, msgBuf[msgBufOffset:msgBufOffset+globals.udpPacketSendPayloadSize]...) - msgBufOffset += globals.udpPacketSendPayloadSize - } else { // packetIndex == (packetCount - 1) - packetBuf = make([]byte, udpPacketHeaderSize, udpPacketHeaderSize+msgBufSize-msgBufOffset) - packetBuf = append(packetBuf, msgBuf[msgBufOffset:]...) - } - - for packetBufIndex = uint64(8); packetBufIndex < uint64(16); packetBufIndex++ { - packetBuf[packetBufIndex] = msgNonceBuf[packetBufIndex-8] - } - - packetBuf[16] = packetIndex - packetBuf[17] = packetCount - - computedCRC64 = crc64.Checksum(packetBuf[8:], globals.crc64ECMATable) - computedCRC64Buf = serializeU64LittleEndian(computedCRC64) - - for packetBufIndex = uint64(0); packetBufIndex < uint64(8); packetBufIndex++ { - packetBuf[packetBufIndex] = computedCRC64Buf[packetBufIndex] - } - - for _, peer = range peers { - _, err = globals.myUDPConn.WriteToUDP(packetBuf, peer.udpAddr) - if nil != err { - err = fmt.Errorf("sendMsg() failed writing to %v: %v", peer.udpAddr, err) - return - } - } - } - - if LogLevelMessages <= globals.logLevel { - loggerInfofBuf = fmt.Sprintf("%s sent %s to", globals.myUDPAddr, reflect.TypeOf(msg)) - for _, peer = range peers { - loggerInfofBuf = loggerInfofBuf + fmt.Sprintf(" %s", peer.udpAddr) - } - if LogLevelMessageDetails <= globals.logLevel { - loggerInfofBuf = loggerInfofBuf + fmt.Sprintf(" [%#v]", msg) - } - logger.Info(loggerInfofBuf) - } - - err = nil - return -} - -func requestExpirer() { - var ( - expirationDuration time.Duration - frontRequest *requestStruct - frontRequestElement *list.Element - ok bool - ) - - for { - globals.Lock() - frontRequestElement = globals.requestsByExpirationTime.Front() - globals.Unlock() - - if nil == frontRequestElement { - select { - case <-globals.requestExpirerStartChan: - // Go look again... there is likely something in globals.requestsByExpirationTime now - case <-globals.requestExpirerStopChan: - globals.requestExpirerDone.Done() - return - } - } else { - frontRequest = frontRequestElement.Value.(*requestStruct) - expirationDuration = frontRequest.expirationTime.Sub(time.Now()) - - select { - case <-time.After(expirationDuration): - // Expire this request if it hasn't already been responded to - globals.Lock() - _, ok = globals.requestsByMsgTag[frontRequest.msgTag] - if ok { - _ = globals.requestsByExpirationTime.Remove(frontRequest.element) - delete(globals.requestsByMsgTag, frontRequest.msgTag) - } - globals.Unlock() - if ok { - frontRequest.expired = true - frontRequest.responseMsg = nil - frontRequest.callback(frontRequest) - } - case <-globals.requestExpirerStopChan: - globals.requestExpirerDone.Done() - return - } - } - } -} - -func sendRequest(peer *peerStruct, msgTag uint64, requestContext interface{}, requestMsg interface{}, callback func(request *requestStruct)) (err error) { - var ( - request *requestStruct - ) - - if nil == peer { - err = fmt.Errorf("sendRequest() requires non-nil peer") - return - } - - request = &requestStruct{ - msgTag: msgTag, - expirationTime: time.Now().Add(globals.maxRequestDuration), - expired: false, - requestContext: requestContext, - requestMsg: requestMsg, - responseMsg: nil, - callback: callback, - } - - globals.Lock() - if nil == globals.requestsByExpirationTime.Front() { - globals.requestExpirerStartChan <- struct{}{} - } - request.element = globals.requestsByExpirationTime.PushBack(request) - globals.requestsByMsgTag[request.msgTag] = request - globals.Unlock() - - _, err = sendMsg(peer, request.requestMsg) - - if nil != err { - globals.Lock() - if !request.expired { - _ = globals.requestsByExpirationTime.Remove(request.element) - delete(globals.requestsByMsgTag, request.msgTag) - } - globals.Unlock() - } - - return // err return from sendMsg() is sufficient status -} - -func deliverResponse(msgTag uint64, responseMsg interface{}) { - var ( - ok bool - request *requestStruct - ) - - globals.Lock() - request, ok = globals.requestsByMsgTag[msgTag] - if ok { - _ = globals.requestsByExpirationTime.Remove(request.element) - delete(globals.requestsByMsgTag, request.msgTag) - } - globals.Unlock() - - if ok { - request.responseMsg = responseMsg - request.callback(request) - } else { - // Already marked expired... so ignore it - } -} diff --git a/liveness/polling.go b/liveness/polling.go deleted file mode 100644 index 4ce6e85a..00000000 --- a/liveness/polling.go +++ /dev/null @@ -1,1014 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package liveness - -import ( - "bytes" - "compress/gzip" - "container/list" - "encoding/binary" - "encoding/json" - "fmt" - "io/ioutil" - "math/big" - "net" - "net/http" - "os" - "reflect" - "regexp" - "time" - - "github.com/NVIDIA/proxyfs/jrpcfs" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/utils" -) - -type pingReqStruct struct { - JSONrpc string `json:"jsonrpc"` - Method string `json:"method"` - Params [1]jrpcfs.PingReq `json:"params"` - ID uint64 `json:"id"` -} - -type pingReplyStruct struct { - ID uint64 `json:"id"` - Result jrpcfs.PingReply `json:"result"` - Error string `json:"error"` -} - -type ringFilePayloadJSONDevStruct struct { - IP string `json:"ip"` - Port uint16 `json:"port"` -} - -type ringFilePayloadJSONStruct struct { - Devs []*ringFilePayloadJSONDevStruct `json:"devs"` -} - -type reconDevReportStruct struct { - Size int64 `json:"size"` - Used int64 `json:"used"` -} - -const maxRPCReplySize = 4096 - -func livenessChecker() { - var ( - checkEntityList *list.List - durationPerCheck time.Duration - entitiesToCheck uint64 - entityToCheck *list.Element - err error - livenessCheckerControlChanValue bool - reconEndpoint *internalReconEndpointReportStruct - servingPeer *internalServingPeerReportStruct - volume *internalVolumeReportStruct - volumeGroup *internalVolumeGroupReportStruct - ) - - for { - globals.Lock() - - if nil == globals.myObservingPeerReport { - // Just wait to be signaled to check again... or exit - globals.Unlock() - livenessCheckerControlChanValue = <-globals.livenessCheckerControlChan - if livenessCheckerControlChanValue { - // Just loop back and re-check globals.myObservingPeerReport - } else { - // Exit - globals.livenessCheckerWG.Done() - return - } - } else { // nil != globals.myObservingPeerReport - // Check to see if we are supposed to exit - - select { - case livenessCheckerControlChanValue = <-globals.livenessCheckerControlChan: - if livenessCheckerControlChanValue { - // Just fall into checkEntityList processing - } else { - // Exit - globals.Unlock() - globals.livenessCheckerWG.Done() - return - } - default: - // Just fall into checkEntityList processing - } - - // Compute randomly-ordered list of entities to check - - checkEntityList = list.New() - - for _, servingPeer = range globals.myObservingPeerReport.servingPeer { - _ = checkEntityList.PushBack(servingPeer) - - for _, volumeGroup = range servingPeer.volumeGroup { - _ = checkEntityList.PushBack(volumeGroup) - - for _, volume = range volumeGroup.volume { - _ = checkEntityList.PushBack(volume) - } - } - } - - for _, reconEndpoint = range globals.myObservingPeerReport.reconEndpoint { - _ = checkEntityList.PushBack(reconEndpoint) - } - - globals.Unlock() - - utils.RandomizeList(checkEntityList) - - // Compute number of entities to check & time between each check - // Allow for one extra time slice to hopefully get all entities checked - - entitiesToCheck = uint64(checkEntityList.Len()) - - durationPerCheck = globals.heartbeatDuration / time.Duration(entitiesToCheck+1) - - // Perform each check spaced out by durationPerCheck - - entityToCheck = checkEntityList.Front() - - for nil != entityToCheck { - switch reflect.TypeOf(entityToCheck.Value) { - case reflect.TypeOf(servingPeer): - livenessCheckServingPeer(entityToCheck.Value.(*internalServingPeerReportStruct)) - case reflect.TypeOf(volumeGroup): - livenessCheckVolumeGroup(entityToCheck.Value.(*internalVolumeGroupReportStruct)) - case reflect.TypeOf(volume): - livenessCheckVolume(entityToCheck.Value.(*internalVolumeReportStruct)) - case reflect.TypeOf(reconEndpoint): - livenessCheckReconEndpoint(entityToCheck.Value.(*internalReconEndpointReportStruct)) - default: - err = fmt.Errorf("Unrecognized reflect.TypeOf(entityToCheck.Value): %v", reflect.TypeOf(entityToCheck.Value)) - panic(err) - } - - // Delay before next entityToCheck - - select { - case livenessCheckerControlChanValue = <-globals.livenessCheckerControlChan: - if livenessCheckerControlChanValue { - // Just finish current checks before re-checking globals.myObservingPeerReport - } else { - // Exit - globals.livenessCheckerWG.Done() - return - } - case <-time.After(durationPerCheck): - // Proceed to next check - } - - // Loop back for next entityToCheck - - entityToCheck = entityToCheck.Next() - } - } - } -} - -func livenessCheckServingPeer(servingPeer *internalServingPeerReportStruct) { - var ( - err error - nextID uint64 - pingReply pingReplyStruct - pingReplyBuf []byte - pingReplyLen int - pingReq pingReqStruct - pingReqBuf []byte - servingPeerState string - tcpAddr *net.TCPAddr - tcpAddrToResolve string - tcpConn *net.TCPConn - timeNow time.Time - ) - - // Setup exit path to atomically update servingPeer (initially servingPeer.state == StateUnknown) - - timeNow = time.Now() - servingPeerState = StateUnknown - - defer func() { - globals.Lock() - servingPeer.state = servingPeerState - servingPeer.lastCheckTime = timeNow - globals.Unlock() - }() - - // Form RpcPing message to poll servingPeer's JSONRPCServer.TCPPort with - - pingReq.JSONrpc = "2.0" - pingReq.Method = "Server.RpcPing" - pingReq.Params[0].Message = "Ping at " + timeNow.Format(time.RFC3339) - pingReq.ID = nextID - - pingReqBuf, err = json.Marshal(pingReq) - if nil != err { - err = fmt.Errorf("json.Marshal(pingReq) failed: %v", err) - logger.Error(err) - return - } - - // Compute tcpAddr of servingPeer's JSONRPCServer.TCPPort - - if servingPeer.name == globals.whoAmI { - tcpAddrToResolve = net.JoinHostPort(globals.myPrivateIPAddr.String(), fmt.Sprintf("%d", globals.jsonRPCServerPort)) - } else { - tcpAddrToResolve = net.JoinHostPort(globals.peersByName[servingPeer.name].privateIPAddr.String(), fmt.Sprintf("%d", globals.jsonRPCServerPort)) - } - - tcpAddr, err = net.ResolveTCPAddr("tcp", tcpAddrToResolve) - - if nil != err { - err = fmt.Errorf("net.ResolveTCPAddr(\"tcp\", \"%v\") failed: %v", tcpAddrToResolve, err) - logger.Error(err) - return - } - - // Perform RpcPing... from here on, default servingPeer.state == StateDead - - servingPeerState = StateDead - - tcpConn, err = net.DialTCP("tcp", nil, tcpAddr) - if nil != err { - return - } - - err = tcpConn.SetDeadline(time.Now().Add(globals.maxRequestDuration)) - if nil != err { - return - } - - _, err = tcpConn.Write(pingReqBuf) - if nil != err { - return - } - - pingReplyBuf = make([]byte, maxRPCReplySize) - - pingReplyLen, err = tcpConn.Read(pingReplyBuf) - if nil != err { - return - } - - err = tcpConn.Close() - if nil != err { - return - } - - pingReplyBuf = pingReplyBuf[:pingReplyLen] - - err = json.Unmarshal(pingReplyBuf, &pingReply) - if nil != err { - return - } - - // RpcPing worked... so ensure servingPeer.state == StateAlive - - servingPeerState = StateAlive -} - -func livenessCheckVolumeGroup(volumeGroup *internalVolumeGroupReportStruct) { - var ( - volumeGroupState string - timeNow time.Time - ) - - // Setup exit path to atomically update servingPeer (initially servingPeer.state == StateUnknown) - - timeNow = time.Now() - volumeGroupState = StateUnknown - - defer func() { - globals.Lock() - volumeGroup.state = volumeGroupState - volumeGroup.lastCheckTime = timeNow - globals.Unlock() - }() - - // TODO: Implement livenessCheckVolumeGroup() -} - -func livenessCheckVolume(volume *internalVolumeReportStruct) { - var ( - volumeState string - timeNow time.Time - ) - - // Setup exit path to atomically update servingPeer (initially servingPeer.state == StateUnknown) - - timeNow = time.Now() - volumeState = StateUnknown - - defer func() { - globals.Lock() - volume.state = volumeState - volume.lastCheckTime = timeNow - globals.Unlock() - }() - - // TODO: Implement livenessCheckVolume() -} - -func livenessCheckReconEndpoint(reconEndpoint *internalReconEndpointReportStruct) { - var ( - bigDividend *big.Int - bigDivisor *big.Int - bigQuotient *big.Int - bigRemainder *big.Int - devUtilization uint8 - err error - quotient int64 - reconDevReport *reconDevReportStruct - reconDevReportSlice []*reconDevReportStruct - reconResp *http.Response - reconRespBody []byte - remainder int64 - url string - ) - - reconEndpoint.maxDiskUsagePercentage = 0 - - url = fmt.Sprintf("http://%s/recon/diskusage", reconEndpoint.ipAddrPort) - - reconResp, err = http.Get(url) - if nil == err { - reconRespBody, err = ioutil.ReadAll(reconResp.Body) - if nil == err { - if http.StatusOK == reconResp.StatusCode { - reconDevReportSlice = make([]*reconDevReportStruct, 0) - err = json.Unmarshal(reconRespBody, &reconDevReportSlice) - if nil == err { - for _, reconDevReport = range reconDevReportSlice { - if (reconDevReport.Used > 0) && (reconDevReport.Size > 0) && (reconDevReport.Used <= reconDevReport.Size) { - bigDividend = new(big.Int).Mul(big.NewInt(100), big.NewInt(reconDevReport.Used)) - bigDivisor = big.NewInt(reconDevReport.Size) - bigQuotient = new(big.Int).Quo(bigDividend, bigDivisor) - bigRemainder = new(big.Int).Rem(bigDividend, bigDivisor) - quotient = bigQuotient.Int64() - remainder = bigRemainder.Int64() - if 0 == remainder { - devUtilization = uint8(quotient) - } else { - devUtilization = uint8(quotient) + 1 - } - if devUtilization > reconEndpoint.maxDiskUsagePercentage { - reconEndpoint.maxDiskUsagePercentage = devUtilization - } - } else { - logger.Warnf("livenessCheckReconEndpoint() GET to %s got responseBody with unreasonable used and size values", url) - } - } - } else { - logger.WarnfWithError(err, "livenessCheckReconEndpoint() GET to %s got response.Body with invalid JSON", url) - } - } else { - logger.WarnfWithError(err, "livenessCheckReconEndpoint() GET to %s got bad status: %s", url, reconResp.Status) - } - } else { - logger.WarnfWithError(err, "livenessCheckReconEndpoint() GET to %s response.Body() read failed", url) - } - err = reconResp.Body.Close() - if nil != err { - logger.WarnfWithError(err, "livenessCheckReconEndpoint() GET to %s response.Body.Close() failed", url) - } - } else { - logger.WarnfWithError(err, "livenessCheckReconEndpoint() failed to issue GET to %s", url) - } -} - -// computeLivenessCheckAssignments takes a list of ObservingPeer and produces a -// template internalLivenessReport that is to be filled in by this collection of peers. -// While the elements of the resultant internalLivenessReport have State, LastCheckTime, -// and MaxDiskUsagePercentage fields, these are ignored as they will ultimately be filled -// in by each ObservingPeer. The livenessCheckRedundancy is used to ensure that each -// ServingPeer, VolumeGroup, Volume, and ReconEndpoint is adequately covered. As every -// Volume is part of a VolumeGroup and every VolumeGroup is assigned to a single ServingPeer, -// this amounts to just dolling out the Volumes to ObervingPeers with the required -// livenessCheckRedundancy. Similarly, the ReconEndpoints are dolled out with this -// same livenessCheckRedundancy. -// -// It is a bit misleading for an ObservingPeer to report that a VolumeGroup is "alive" -// when not all of that VolumeGroup's Volumes have been checked. Similarly, it is a -// bit misleading for an ObservingPeer to report that a ServingPeer is "alive" when -// not all of that ServingPeer's VolumeGroups have been checked. Therefore, to get an -// accurate picture of that state of a VolumeGroup or ServingPeer, all results from -// all ObservingPeers should be consulted as a set when making any availability -// decision. As there is no way to check an empty VolumeGroup, there state will not -// be in the resultant internalLivenessReport. However, ServingPeers that have no -// VolumeGroups assigned will still be in the resultant internalLivenessReport. -func computeLivenessCheckAssignments(observingPeerNameList []string) (internalLivenessReport *internalLivenessReportStruct) { - var ( - alreadyInSwiftReconEndpointIAddrSet bool - curSwiftConfFileMap map[string]time.Time - effectiveLivenessCheckRedundancy uint64 - effectiveLivenessCheckRedundancyIndex uint64 - err error - fileInfo os.FileInfo - fileInfoSlice []os.FileInfo - fileInfoModTime time.Time - fileInfoName string - inSwiftConfFileMap bool - internalObservingPeerReport *internalObservingPeerReportStruct - internalReconEndpointReport *internalReconEndpointReportStruct - internalServingPeerReport *internalServingPeerReportStruct - internalVolumeGroupReport *internalVolumeGroupReportStruct - internalVolumeReport *internalVolumeReportStruct - matchedRingFilename bool - needToUpdateSwiftConfFileMap bool - notYetAdded bool - observingPeerIndex uint64 - observingPeerName string - ok bool - prevFileInfoModTime time.Time - ringFileData []byte - ringFileName string - ringFileMagic []byte - ringFilePayload []byte - ringFilePayloadJSON *ringFilePayloadJSONStruct - ringFilePayloadJSONDev *ringFilePayloadJSONDevStruct - ringFilePayloadLen int32 - ringFileReader *gzip.Reader - ringFileReadLen int - ringFileVersion uint16 - servingPeer *peerStruct - servingPeerName string - swiftReconEndpoint string - swiftReconEndpointIPAddrSet map[string]struct{} - volumeGroup *volumeGroupStruct - volumeGroupName string - volumeName string - volumeToCheck *volumeStruct - ) - - if 0 == len(observingPeerNameList) { - err = fmt.Errorf("computeLivenessCheckAssignments(): len(observingPeerNameList) cannot be zero") - panic(err) - } - - // Determine reconEndpoints - - if 0 == globals.swiftReconChecksPerConfCheck { - globals.swiftReconEndpointSet = make(map[string]struct{}) - } else { - if 0 == globals.swiftReconChecksUntilConfCheck { - // Time to potentially refresh globals.swiftConfFileMap & globals.swiftReconEndpointSet - - globals.swiftReconChecksUntilConfCheck = globals.swiftReconChecksPerConfCheck - - fileInfoSlice, err = ioutil.ReadDir(globals.swiftConfDir) - if nil != err { - logger.FatalfWithError(err, "Unable to read [SwiftClient]SwiftConfDir (%s)", globals.swiftConfDir) - } - - curSwiftConfFileMap = make(map[string]time.Time) - - for _, fileInfo = range fileInfoSlice { - fileInfoName = fileInfo.Name() - switch fileInfoName { - case "account.ring.gz": - matchedRingFilename = true - case "container.ring.gz": - matchedRingFilename = true - default: - matchedRingFilename, err = regexp.MatchString("^object.*\\.ring\\.gz$", fileInfoName) - if nil != err { - logger.FatalfWithError(err, "Unexpected failure calling regexp.MatchString()") - } - } - - if matchedRingFilename { - curSwiftConfFileMap[fileInfoName] = fileInfo.ModTime() - } - } - - if len(globals.swiftConfFileMap) != len(curSwiftConfFileMap) { - needToUpdateSwiftConfFileMap = true - } else { - needToUpdateSwiftConfFileMap = false - for fileInfoName, fileInfoModTime = range curSwiftConfFileMap { - prevFileInfoModTime, inSwiftConfFileMap = globals.swiftConfFileMap[fileInfoName] - if !inSwiftConfFileMap || (fileInfoModTime != prevFileInfoModTime) { - needToUpdateSwiftConfFileMap = true - } - } - } - - if needToUpdateSwiftConfFileMap { - // We must refresh globals.swiftConfFileMap & globals.swiftReconEndpointSet - - globals.swiftConfFileMap = curSwiftConfFileMap - - swiftReconEndpointIPAddrSet = make(map[string]struct{}) - globals.swiftReconEndpointSet = make(map[string]struct{}) - - for ringFileName = range globals.swiftConfFileMap { - ringFileData, err = ioutil.ReadFile(globals.swiftConfDir + "/" + ringFileName) - if nil == err { - ringFileReader, err = gzip.NewReader(bytes.NewReader(ringFileData)) - if nil == err { - ringFileMagic = make([]byte, 4) - ringFileReadLen, err = ringFileReader.Read(ringFileMagic) - if nil == err { - if ringFileReadLen == len(ringFileMagic) { - if bytes.Equal([]byte("R1NG"), ringFileMagic) { - err = binary.Read(ringFileReader, binary.BigEndian, &ringFileVersion) - if nil == err { - if 1 == ringFileVersion { - err = binary.Read(ringFileReader, binary.BigEndian, &ringFilePayloadLen) - if nil == err { - ringFilePayload = make([]byte, ringFilePayloadLen) - ringFileReadLen, err = ringFileReader.Read(ringFilePayload) - if nil == err { - if ringFileReadLen == len(ringFilePayload) { - ringFilePayloadJSON = &ringFilePayloadJSONStruct{} - err = json.Unmarshal(ringFilePayload, ringFilePayloadJSON) - if nil == err { - for _, ringFilePayloadJSONDev = range ringFilePayloadJSON.Devs { - if nil != ringFilePayloadJSONDev { - _, alreadyInSwiftReconEndpointIAddrSet = swiftReconEndpointIPAddrSet[ringFilePayloadJSONDev.IP] - if !alreadyInSwiftReconEndpointIAddrSet { - swiftReconEndpointIPAddrSet[ringFilePayloadJSONDev.IP] = struct{}{} - swiftReconEndpoint = fmt.Sprintf("%s:%d", ringFilePayloadJSONDev.IP, ringFilePayloadJSONDev.Port) - globals.swiftReconEndpointSet[swiftReconEndpoint] = struct{}{} - } - } - } - } else { - logger.WarnfWithError(err, "Unable to json.Unmarshal ringFilePayload from ring file %s", fileInfoName) - } - } else { - logger.Warnf("Misread of ringFilePayload from ring file %s", fileInfoName) - } - } else { - logger.WarnfWithError(err, "Unable to read ringFilePayload from ring file %s", fileInfoName) - } - } else { - logger.WarnfWithError(err, "Unable to read ringFilePayloadLen from ring file %s", fileInfoName) - } - } else { - logger.Warnf("Value of ringFileVersion unexpected from ring file %s", fileInfoName) - } - } else { - logger.WarnfWithError(err, "Unable to read ringFileVersion from ring file %s", fileInfoName) - } - } else { - logger.Warnf("Value of ringFileMagic unexpected from ring file %s", fileInfoName) - } - } else { - logger.Warnf("Misread of ringFileMagic from ring file %s", fileInfoName) - } - } else { - logger.WarnfWithError(err, "Unable to read ringFileMagic from ring file %s", fileInfoName) - } - err = ringFileReader.Close() - if nil != err { - logger.WarnfWithError(err, "Unable to close gzip.Reader from ring file %s", fileInfoName) - } - } else { - logger.WarnfWithError(err, "Unable to create gzip.Reader from ring file %s", fileInfoName) - } - } else { - logger.WarnfWithError(err, "Unable to read ring file %s", fileInfoName) - } - } - } - } else { - globals.swiftReconChecksUntilConfCheck-- - } - } - - // Prepare fresh internalLivenessReport - - internalLivenessReport = &internalLivenessReportStruct{ - observingPeer: make(map[string]*internalObservingPeerReportStruct), - } - - // Adjust effectiveLivenessCheckRedundancy to be no more than len(observingPeerNameList) - - if uint64(len(observingPeerNameList)) < globals.livenessCheckRedundancy { - effectiveLivenessCheckRedundancy = uint64(len(observingPeerNameList)) - } else { - effectiveLivenessCheckRedundancy = globals.livenessCheckRedundancy - } - - // Iterate through observingPeerNameList effectiveLivenessCheckRedundancy times scheduling Volumes - - observingPeerIndex = 0 - - for effectiveLivenessCheckRedundancyIndex = 0; effectiveLivenessCheckRedundancyIndex < effectiveLivenessCheckRedundancy; effectiveLivenessCheckRedundancyIndex++ { - for _, volumeToCheck = range globals.volumeToCheckList { - // Add volumeToCheck to currently indexed ObservingPeer - - volumeName = volumeToCheck.name - volumeGroup = volumeToCheck.volumeGroup - volumeGroupName = volumeGroup.name - servingPeer = volumeGroup.peer - if nil == servingPeer { - servingPeerName = globals.whoAmI - } else { - servingPeerName = servingPeer.name - } - - notYetAdded = true // Avoid duplicate assignments - - for notYetAdded { - observingPeerName = observingPeerNameList[observingPeerIndex] - - internalObservingPeerReport, ok = internalLivenessReport.observingPeer[observingPeerName] - if !ok { - internalObservingPeerReport = &internalObservingPeerReportStruct{ - name: observingPeerName, - servingPeer: make(map[string]*internalServingPeerReportStruct), - reconEndpoint: make(map[string]*internalReconEndpointReportStruct), - } - internalLivenessReport.observingPeer[observingPeerName] = internalObservingPeerReport - } - - internalServingPeerReport, ok = internalObservingPeerReport.servingPeer[servingPeerName] - if !ok { - internalServingPeerReport = &internalServingPeerReportStruct{ - observingPeer: internalObservingPeerReport, - name: servingPeerName, - state: StateUnknown, - lastCheckTime: time.Time{}, - volumeGroup: make(map[string]*internalVolumeGroupReportStruct), - } - internalObservingPeerReport.servingPeer[servingPeerName] = internalServingPeerReport - } - - internalVolumeGroupReport, ok = internalServingPeerReport.volumeGroup[volumeGroupName] - if !ok { - internalVolumeGroupReport = &internalVolumeGroupReportStruct{ - servingPeer: internalServingPeerReport, - name: volumeGroupName, - state: StateUnknown, - lastCheckTime: time.Time{}, - volume: make(map[string]*internalVolumeReportStruct), - } - internalServingPeerReport.volumeGroup[volumeGroupName] = internalVolumeGroupReport - } - - _, ok = internalVolumeGroupReport.volume[volumeName] - - if ok { - // Need to step to the next ObservingPeer because this one is already watching this Volume - } else { - // New Volume for this ObservingPeer... so add it - - internalVolumeReport = &internalVolumeReportStruct{ - volumeGroup: internalVolumeGroupReport, - name: volumeName, - state: StateUnknown, - lastCheckTime: time.Time{}, - } - - internalVolumeGroupReport.volume[volumeName] = internalVolumeReport - - notYetAdded = false - } - - // Cycle to next ObservingPeer - - observingPeerIndex++ - if observingPeerIndex == uint64(len(observingPeerNameList)) { - observingPeerIndex = 0 - } - } - } - } - - // Iterate through observingPeerNameList effectiveLivenessCheckRedundancy times scheduling "empty" VolumeGroups - - for effectiveLivenessCheckRedundancyIndex = 0; effectiveLivenessCheckRedundancyIndex < effectiveLivenessCheckRedundancy; effectiveLivenessCheckRedundancyIndex++ { - for volumeGroupName, servingPeerName = range globals.emptyVolumeGroupToCheckSet { - // Add "empty" VolumeGroup to currently indexed ObservingPeer - - notYetAdded = true // Avoid duplicate assignments - - for notYetAdded { - observingPeerName = observingPeerNameList[observingPeerIndex] - - internalObservingPeerReport, ok = internalLivenessReport.observingPeer[observingPeerName] - if !ok { - internalObservingPeerReport = &internalObservingPeerReportStruct{ - name: observingPeerName, - servingPeer: make(map[string]*internalServingPeerReportStruct), - reconEndpoint: make(map[string]*internalReconEndpointReportStruct), - } - internalLivenessReport.observingPeer[observingPeerName] = internalObservingPeerReport - } - - internalServingPeerReport, ok = internalObservingPeerReport.servingPeer[servingPeerName] - if !ok { - internalServingPeerReport = &internalServingPeerReportStruct{ - observingPeer: internalObservingPeerReport, - name: servingPeerName, - state: StateUnknown, - lastCheckTime: time.Time{}, - volumeGroup: make(map[string]*internalVolumeGroupReportStruct), - } - } - internalObservingPeerReport.servingPeer[servingPeerName] = internalServingPeerReport - - _, ok = internalServingPeerReport.volumeGroup[volumeGroupName] - - if ok { - // Need to step to the next ObservingPeer because this one is already watching this VolumeGroup - } else { - // New VolumeGroup for this ObservingPeer->ServingPeer... so add it - - internalVolumeGroupReport = &internalVolumeGroupReportStruct{ - servingPeer: internalServingPeerReport, - name: volumeGroupName, - state: StateUnknown, - lastCheckTime: time.Time{}, - volume: make(map[string]*internalVolumeReportStruct), - } - - internalServingPeerReport.volumeGroup[volumeGroupName] = internalVolumeGroupReport - - notYetAdded = false - } - - // Cycle to next ObservingPeer - - observingPeerIndex++ - if observingPeerIndex == uint64(len(observingPeerNameList)) { - observingPeerIndex = 0 - } - } - } - } - - // Iterate through observingPeerNameList effectiveLivenessCheckRedundancy times scheduling "empty" ServingPeers - - for effectiveLivenessCheckRedundancyIndex = 0; effectiveLivenessCheckRedundancyIndex < effectiveLivenessCheckRedundancy; effectiveLivenessCheckRedundancyIndex++ { - for servingPeerName = range globals.emptyServingPeerToCheckSet { - // Add "empty" ServingPeer to currently indexed ObservingPeer - - notYetAdded = true // Avoid duplicate assignments - - for notYetAdded { - observingPeerName = observingPeerNameList[observingPeerIndex] - - internalObservingPeerReport, ok = internalLivenessReport.observingPeer[observingPeerName] - if !ok { - internalObservingPeerReport = &internalObservingPeerReportStruct{ - name: observingPeerName, - servingPeer: make(map[string]*internalServingPeerReportStruct), - reconEndpoint: make(map[string]*internalReconEndpointReportStruct), - } - internalLivenessReport.observingPeer[observingPeerName] = internalObservingPeerReport - } - - _, ok = internalObservingPeerReport.servingPeer[servingPeerName] - - if ok { - // Need to step to the next ObservingPeer because this one is already watching this ServingPeer - } else { - // New ServingPeer for this ObservingPeer... so add it - - internalServingPeerReport = &internalServingPeerReportStruct{ - observingPeer: internalObservingPeerReport, - name: servingPeerName, - state: StateUnknown, - lastCheckTime: time.Time{}, - volumeGroup: make(map[string]*internalVolumeGroupReportStruct), - } - - internalObservingPeerReport.servingPeer[servingPeerName] = internalServingPeerReport - - notYetAdded = false - } - - // Cycle to next ObservingPeer - - observingPeerIndex++ - if observingPeerIndex == uint64(len(observingPeerNameList)) { - observingPeerIndex = 0 - } - } - } - } - - // Iterate through observingPeerNameList effectiveLivenessCheckRedundancy times scheduling ReconEndpoints - - for effectiveLivenessCheckRedundancyIndex = 0; effectiveLivenessCheckRedundancyIndex < effectiveLivenessCheckRedundancy; effectiveLivenessCheckRedundancyIndex++ { - for swiftReconEndpoint = range globals.swiftReconEndpointSet { - // Add volumeToCheck to currently indexed ObservingPeer - - notYetAdded = true // Avoid duplicate assignments - - for notYetAdded { - observingPeerName = observingPeerNameList[observingPeerIndex] - - internalObservingPeerReport, ok = internalLivenessReport.observingPeer[observingPeerName] - if !ok { - internalObservingPeerReport = &internalObservingPeerReportStruct{ - name: observingPeerName, - servingPeer: make(map[string]*internalServingPeerReportStruct), - reconEndpoint: make(map[string]*internalReconEndpointReportStruct), - } - internalLivenessReport.observingPeer[observingPeerName] = internalObservingPeerReport - } - - _, ok = internalObservingPeerReport.reconEndpoint[swiftReconEndpoint] - - if ok { - // Need to step to the next ObservingPeer because this one is already watching this ReconEndpoint - } else { - // New ReconEndpoint for this ObservingPeer... so add it - - internalReconEndpointReport = &internalReconEndpointReportStruct{ - observingPeer: internalObservingPeerReport, - ipAddrPort: swiftReconEndpoint, - maxDiskUsagePercentage: 0, - } - - internalObservingPeerReport.reconEndpoint[swiftReconEndpoint] = internalReconEndpointReport - - notYetAdded = false - } - - // Cycle to next ObservingPeer - - observingPeerIndex++ - if observingPeerIndex == uint64(len(observingPeerNameList)) { - observingPeerIndex = 0 - } - } - } - } - - return -} - -func mergeObservingPeerReportIntoLivenessReport(internalObservingPeerReport *internalObservingPeerReportStruct, internalLivenessReport *internalLivenessReportStruct) { - var ( - ok bool - ) - - _, ok = internalLivenessReport.observingPeer[internalObservingPeerReport.name] - if ok { - delete(internalLivenessReport.observingPeer, internalObservingPeerReport.name) - } - - internalLivenessReport.observingPeer[internalObservingPeerReport.name] = internalObservingPeerReport -} - -func updateMyObservingPeerReportWhileLocked(internalObservingPeerReport *internalObservingPeerReportStruct) { - var ( - ok bool - reconEndpointIPAddrPort string - reconEndpointIPAddrPortSet map[string]struct{} - servingPeerName string - servingPeerNameSet map[string]struct{} - servingPeerNew *internalServingPeerReportStruct - servingPeerOld *internalServingPeerReportStruct - volumeGroupName string - volumeGroupNameSet map[string]struct{} - volumeGroupNew *internalVolumeGroupReportStruct - volumeGroupOld *internalVolumeGroupReportStruct - volumeName string - volumeNameSet map[string]struct{} - ) - - if (nil == globals.myObservingPeerReport) || (nil == internalObservingPeerReport) { - globals.myObservingPeerReport = internalObservingPeerReport - return - } - - // Remove any ServingPeers from globals.myObservingPeerReport missing from internalObservingPeerReport - - servingPeerNameSet = make(map[string]struct{}) - - for servingPeerName = range globals.myObservingPeerReport.servingPeer { - _, ok = internalObservingPeerReport.servingPeer[servingPeerName] - if !ok { - servingPeerNameSet[servingPeerName] = struct{}{} - } - } - - for servingPeerName = range servingPeerNameSet { - delete(globals.myObservingPeerReport.servingPeer, servingPeerName) - } - - // Add any ServingPeers from internalObservingPeerReport missing from globals.myObservingPeerReport - - for servingPeerName = range internalObservingPeerReport.servingPeer { - _, ok = globals.myObservingPeerReport.servingPeer[servingPeerName] - if !ok { - globals.myObservingPeerReport.servingPeer[servingPeerName] = &internalServingPeerReportStruct{ - observingPeer: globals.myObservingPeerReport, - name: servingPeerName, - state: StateUnknown, - lastCheckTime: time.Time{}, - volumeGroup: make(map[string]*internalVolumeGroupReportStruct), - } - } - } - - // Now loop inside each ServingPeer (must now exist in both globals.myObservingPeerReport & internalObservingPeerReport) - - for servingPeerName, servingPeerOld = range globals.myObservingPeerReport.servingPeer { - servingPeerNew = internalObservingPeerReport.servingPeer[servingPeerName] - - // Remove any VolumeGroups from servingPeerOld missing from servingPeerNew - - volumeGroupNameSet = make(map[string]struct{}) - - for volumeGroupName = range servingPeerOld.volumeGroup { - _, ok = servingPeerNew.volumeGroup[volumeGroupName] - if !ok { - volumeGroupNameSet[volumeGroupName] = struct{}{} - } - } - - for volumeGroupName = range volumeGroupNameSet { - delete(servingPeerOld.volumeGroup, volumeGroupName) - } - - // Add any VolumeGroups from servingPeerNew missing from servingPeerOld - - for volumeGroupName = range servingPeerNew.volumeGroup { - _, ok = servingPeerOld.volumeGroup[volumeGroupName] - if !ok { - servingPeerOld.volumeGroup[volumeGroupName] = &internalVolumeGroupReportStruct{ - servingPeer: servingPeerOld, - name: volumeGroupName, - state: StateUnknown, - lastCheckTime: time.Time{}, - volume: make(map[string]*internalVolumeReportStruct), - } - } - } - - // Now loop inside each VolumeGroup (must now exist in both servingPeerOld & servingPeerNew) - - for volumeGroupName, volumeGroupOld = range servingPeerOld.volumeGroup { - volumeGroupNew = servingPeerNew.volumeGroup[volumeGroupName] - - // Remove any Volumes from volumeGroupOld missing from volumeGroupNew - - volumeNameSet = make(map[string]struct{}) - - for volumeName = range volumeGroupOld.volume { - _, ok = volumeGroupNew.volume[volumeName] - if !ok { - volumeNameSet[volumeName] = struct{}{} - } - } - - for volumeName = range volumeNameSet { - delete(volumeGroupOld.volume, volumeName) - } - - // Add any Volumes from volumeGroupNew missing from VolumeGroupOld - - for volumeName = range volumeGroupNew.volume { - _, ok = volumeGroupOld.volume[volumeName] - if !ok { - volumeGroupOld.volume[volumeName] = &internalVolumeReportStruct{ - volumeGroup: volumeGroupOld, - name: volumeName, - state: StateUnknown, - lastCheckTime: time.Time{}, - } - } - } - } - } - - // Remove any ReconEndpoints from globals.myObservingPeerReport missing from internalObservingPeerReport - - reconEndpointIPAddrPortSet = make(map[string]struct{}) - - for reconEndpointIPAddrPort = range globals.myObservingPeerReport.reconEndpoint { - _, ok = internalObservingPeerReport.reconEndpoint[reconEndpointIPAddrPort] - if !ok { - reconEndpointIPAddrPortSet[reconEndpointIPAddrPort] = struct{}{} - } - } - - for reconEndpointIPAddrPort = range reconEndpointIPAddrPortSet { - delete(globals.myObservingPeerReport.reconEndpoint, reconEndpointIPAddrPort) - } - - // Add any ReconEndpoints from internalObservingPeerReport missing from globals.myObservingPeerReport - - for reconEndpointIPAddrPort = range internalObservingPeerReport.reconEndpoint { - _, ok = globals.myObservingPeerReport.reconEndpoint[reconEndpointIPAddrPort] - if !ok { - globals.myObservingPeerReport.reconEndpoint[reconEndpointIPAddrPort] = &internalReconEndpointReportStruct{ - observingPeer: globals.myObservingPeerReport, - ipAddrPort: reconEndpointIPAddrPort, - maxDiskUsagePercentage: 0, - } - } - } -} diff --git a/liveness/polling_test.go b/liveness/polling_test.go deleted file mode 100644 index cd504568..00000000 --- a/liveness/polling_test.go +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package liveness - -import ( - "encoding/json" - "fmt" - "testing" -) - -// TestComputeLivenessCheckAssignments tests the named func against the following config: -// -// PeerA - hosting VolumeGroupAA & VolumeGroupAB -// PeerB - hosting VolumeGroupBA, VolumeGroupBB, & VolumeGroupBC -// PeerC - hosting VolumeGroupCA, VolumeGroupCB, & VolumeGroupCC -// PeerD - not hosting any VolumeGroups -// -// VolumeGroupAA - hosting VolumeAAA & VolumeAAB -// VolumeGroupAB - hosting VolumeABA, VolumeABB, and VolumeABC -// -// VolumeGroupBA - hosting VolumeBAA -// VolumeGroupBB - hosting VolumeBBA & VolumeBBB -// VolumeGroupBC - hosting VolumeBCA -// -// VolumeGroupCA - hosting VolumeCAA, VolumeCAB, & VolumeCAC -// VolumeGroupCB - hosting VolumeCBA & VolumeCBB -// VolumeGroupCC - hosting no volumes -// -// We will always use livenessCheckRedundancy == 3 -// We will always use whoAmI == "PeerA" -// We will vary observingPeerNameList == []{"PeerA"} and []{"PeerA", "PeerB", "PeerC", "PeerD"} -// -func TestComputeLivenessCheckAssignments(t *testing.T) { - var ( - // Peers (other than whoAmI) - peerB *peerStruct - peerC *peerStruct - peerD *peerStruct - // VolumeGroups - volumeGroupAA *volumeGroupStruct - volumeGroupAB *volumeGroupStruct - volumeGroupBA *volumeGroupStruct - volumeGroupBB *volumeGroupStruct - volumeGroupBC *volumeGroupStruct - volumeGroupCA *volumeGroupStruct - volumeGroupCB *volumeGroupStruct - volumeGroupCC *volumeGroupStruct - // Volumes - volumeAAA *volumeStruct - volumeAAB *volumeStruct - volumeABA *volumeStruct - volumeABB *volumeStruct - volumeABC *volumeStruct - volumeBAA *volumeStruct - volumeBBA *volumeStruct - volumeBBB *volumeStruct - volumeBCA *volumeStruct - volumeCAA *volumeStruct - volumeCAB *volumeStruct - volumeCAC *volumeStruct - volumeCBA *volumeStruct - volumeCBB *volumeStruct - // Other locals - err error - externalLivenessReportForAllPeers *LivenessReportStruct - externalLivenessReportForAllPeersAsJSON []byte - externalLivenessReportForJustPeerA *LivenessReportStruct - externalLivenessReportForJustPeerAAsJSON []byte - internalLivenessReportForAllPeers *internalLivenessReportStruct - internalLivenessReportForJustPeerA *internalLivenessReportStruct - observingPeernameListWithAllPeers []string - observingPeernameListWithJustPeerA []string - ) - - // Setup baseline globals - - globals.whoAmI = "PeerA" - globals.myVolumeGroupMap = make(map[string]*volumeGroupStruct) - - volumeGroupAA = &volumeGroupStruct{ - peer: nil, - name: "VolumeGroupAA", - volumeMap: make(map[string]*volumeStruct), - } - globals.myVolumeGroupMap[volumeGroupAA.name] = volumeGroupAA - - volumeAAA = &volumeStruct{ - volumeGroup: volumeGroupAA, - name: "VolumeAAA", - } - volumeGroupAA.volumeMap[volumeAAA.name] = volumeAAA - - volumeAAB = &volumeStruct{ - volumeGroup: volumeGroupAA, - name: "VolumeAAB", - } - volumeGroupAA.volumeMap[volumeAAB.name] = volumeAAB - - volumeGroupAB = &volumeGroupStruct{ - peer: nil, - name: "VolumeGroupAB", - volumeMap: make(map[string]*volumeStruct), - } - globals.myVolumeGroupMap[volumeGroupAB.name] = volumeGroupAB - - volumeABA = &volumeStruct{ - volumeGroup: volumeGroupAB, - name: "VolumeABA", - } - volumeGroupAB.volumeMap[volumeAAA.name] = volumeABA - - volumeABB = &volumeStruct{ - volumeGroup: volumeGroupAB, - name: "VolumeABB", - } - volumeGroupAB.volumeMap[volumeAAB.name] = volumeABB - - volumeABC = &volumeStruct{ - volumeGroup: volumeGroupAB, - name: "VolumeABC", - } - volumeGroupAB.volumeMap[volumeAAB.name] = volumeABC - - peerB = &peerStruct{ - name: "PeerB", - volumeGroupMap: make(map[string]*volumeGroupStruct), - } - - volumeGroupBA = &volumeGroupStruct{ - peer: peerB, - name: "VolumeGroupBA", - volumeMap: make(map[string]*volumeStruct), - } - peerB.volumeGroupMap[volumeGroupBA.name] = volumeGroupBA - - volumeBAA = &volumeStruct{ - volumeGroup: volumeGroupBA, - name: "VolumeBAA", - } - volumeGroupBA.volumeMap[volumeBAA.name] = volumeBAA - - volumeGroupBB = &volumeGroupStruct{ - peer: peerB, - name: "VolumeGroupBB", - volumeMap: make(map[string]*volumeStruct), - } - peerB.volumeGroupMap[volumeGroupBB.name] = volumeGroupBB - - volumeBBA = &volumeStruct{ - volumeGroup: volumeGroupBA, - name: "VolumeBBA", - } - volumeGroupBB.volumeMap[volumeBBA.name] = volumeBBA - - volumeBBB = &volumeStruct{ - volumeGroup: volumeGroupBA, - name: "VolumeBBB", - } - volumeGroupBB.volumeMap[volumeBBB.name] = volumeBBB - - volumeGroupBC = &volumeGroupStruct{ - peer: peerB, - name: "VolumeGroupBC", - volumeMap: make(map[string]*volumeStruct), - } - peerB.volumeGroupMap[volumeGroupBC.name] = volumeGroupBC - - volumeBCA = &volumeStruct{ - volumeGroup: volumeGroupBC, - name: "VolumeBCA", - } - volumeGroupBC.volumeMap[volumeBCA.name] = volumeBCA - - peerC = &peerStruct{ - name: "PeerC", - volumeGroupMap: make(map[string]*volumeGroupStruct), - } - - volumeGroupCA = &volumeGroupStruct{ - peer: peerC, - name: "VolumeGroupCA", - volumeMap: make(map[string]*volumeStruct), - } - peerC.volumeGroupMap[volumeGroupCA.name] = volumeGroupCA - - volumeCAA = &volumeStruct{ - volumeGroup: volumeGroupCA, - name: "VolumeCAA", - } - volumeGroupCA.volumeMap[volumeCAA.name] = volumeCAA - - volumeCAB = &volumeStruct{ - volumeGroup: volumeGroupCA, - name: "VolumeCAB", - } - volumeGroupCA.volumeMap[volumeCAB.name] = volumeCAB - - volumeCAC = &volumeStruct{ - volumeGroup: volumeGroupCA, - name: "VolumeCAC", - } - volumeGroupCA.volumeMap[volumeCAC.name] = volumeCAC - - volumeGroupCB = &volumeGroupStruct{ - peer: peerC, - name: "VolumeGroupCB", - volumeMap: make(map[string]*volumeStruct), - } - peerC.volumeGroupMap[volumeGroupCB.name] = volumeGroupCB - - volumeCBA = &volumeStruct{ - volumeGroup: volumeGroupCB, - name: "VolumeCBA", - } - volumeGroupCB.volumeMap[volumeCBA.name] = volumeCBA - - volumeCBB = &volumeStruct{ - volumeGroup: volumeGroupCB, - name: "VolumeCBB", - } - volumeGroupCB.volumeMap[volumeCBB.name] = volumeCBB - - volumeGroupCC = &volumeGroupStruct{ - peer: peerC, - name: "VolumeGroupCC", - volumeMap: make(map[string]*volumeStruct), - } - peerC.volumeGroupMap[volumeGroupCC.name] = volumeGroupCC - - peerD = &peerStruct{ - name: "PeerB", - volumeGroupMap: make(map[string]*volumeGroupStruct), - } - - globals.peersByName = make(map[string]*peerStruct) - - globals.peersByName["PeerB"] = peerB - globals.peersByName["PeerC"] = peerC - globals.peersByName["PeerD"] = peerD - - globals.volumeToCheckList = make([]*volumeStruct, 0) - - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeAAA) - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeAAB) - - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeABA) - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeABB) - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeABC) - - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeBAA) - - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeBBA) - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeBBB) - - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeBCA) - - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeCAA) - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeCAB) - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeCAC) - - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeCBA) - globals.volumeToCheckList = append(globals.volumeToCheckList, volumeCBB) - - globals.emptyServingPeerToCheckSet = make(map[string]struct{}) - - globals.emptyServingPeerToCheckSet["PeerD"] = struct{}{} - - globals.livenessCheckRedundancy = 3 - - // Validate for observingPeerNameList == []{"PeerA"} - - observingPeernameListWithJustPeerA = []string{"PeerA"} - - internalLivenessReportForJustPeerA = computeLivenessCheckAssignments(observingPeernameListWithJustPeerA) - - externalLivenessReportForJustPeerA = convertInternalToExternalLivenessReport(internalLivenessReportForJustPeerA) - - fmt.Println("externalLivenessReportForJustPeerA:") - externalLivenessReportForJustPeerAAsJSON, err = json.MarshalIndent(externalLivenessReportForJustPeerA, "", " ") - if nil != err { - t.Fatal(err) - } - fmt.Println(string(externalLivenessReportForJustPeerAAsJSON)) - - // TODO: Actually validate it programmatically - - // Validate for observingPeerNameList == []{"PeerA", "PeerB", "PeerC", "PeerD"} - - observingPeernameListWithAllPeers = []string{"PeerA", "PeerB", "PeerC", "PeerD"} - - internalLivenessReportForAllPeers = computeLivenessCheckAssignments(observingPeernameListWithAllPeers) - - externalLivenessReportForAllPeers = convertInternalToExternalLivenessReport(internalLivenessReportForAllPeers) - - fmt.Println("externalLivenessReportForAllPeers:") - externalLivenessReportForAllPeersAsJSON, err = json.MarshalIndent(externalLivenessReportForAllPeers, "", " ") - if nil != err { - t.Fatal(err) - } - fmt.Println(string(externalLivenessReportForAllPeersAsJSON)) - - // TODO: Actually validate it programmatically - - // All done -} diff --git a/liveness/states.go b/liveness/states.go deleted file mode 100644 index 3997388c..00000000 --- a/liveness/states.go +++ /dev/null @@ -1,855 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package liveness - -import ( - "crypto/rand" - "fmt" - "reflect" - "runtime" - "time" - - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" -) - -func stateMachine() { - for { - globals.nextState() - } -} - -func doCandidate() { - var ( - awaitingResponses map[*peerStruct]struct{} - durationDelta time.Duration - err error - livenessReportWhileCandidate *internalLivenessReportStruct - msgAsFetchLivenessReportRequest *FetchLivenessReportRequestStruct - msgAsFetchLivenessReportResponse *FetchLivenessReportResponseStruct - msgAsHeartBeatRequest *HeartBeatRequestStruct - msgAsHeartBeatResponse *HeartBeatResponseStruct - msgAsRequestVoteRequest *RequestVoteRequestStruct - msgAsRequestVoteResponse *RequestVoteResponseStruct - ok bool - recvMsgQueueElement *recvMsgQueueElementStruct - peer *peerStruct - peers []*peerStruct - randByteBuf []byte - requestVoteSuccessfulResponses uint64 - requestVoteSuccessfulResponsesRequiredForQuorum uint64 - requestVoteExpirationTime time.Time - requestVoteExpirationDurationRemaining time.Duration - requestVoteMsgTag uint64 - timeNow time.Time - ) - - if LogLevelStateChanges <= globals.logLevel { - logger.Infof("%s entered Candidate state", globals.myUDPAddr) - } - - // Point all LivenessCheckAssignments at globals.whoAmI - - globals.Lock() - livenessReportWhileCandidate = computeLivenessCheckAssignments([]string{globals.whoAmI}) - updateMyObservingPeerReportWhileLocked(livenessReportWhileCandidate.observingPeer[globals.whoAmI]) - globals.Unlock() - globals.livenessCheckerControlChan <- true - - // Attempt to start a new term - - globals.currentTerm++ - - if 0 == len(globals.peersByTuple) { - // Special one peer cluster case... there will be no RequestForVote Responses, so just convert to Leader - globals.nextState = doLeader - return - } - - // Issue RequestVoteRequest to all other Peers - - requestVoteMsgTag = fetchNonce() - - msgAsRequestVoteRequest = &RequestVoteRequestStruct{ - MsgType: MsgTypeRequestVoteRequest, - MsgTag: requestVoteMsgTag, - CandidateTerm: globals.currentTerm, - } - - peers, err = sendMsg(nil, msgAsRequestVoteRequest) - if nil != err { - panic(err) - } - - awaitingResponses = make(map[*peerStruct]struct{}) - - for _, peer = range peers { - awaitingResponses[peer] = struct{}{} - } - - // Minimize split votes by picking a requestVoteExpirationTime at some random - // point between globals.heartbeatDuration and globals.heartbeatMissDuration - - randByteBuf = make([]byte, 1) - _, err = rand.Read(randByteBuf) - if nil != err { - err = fmt.Errorf("rand.Read(randByteBuf) failed: %v", err) - panic(err) - } - - durationDelta = globals.heartbeatMissDuration - globals.heartbeatDuration - durationDelta *= time.Duration(randByteBuf[0]) - durationDelta /= time.Duration(0x100) - durationDelta += globals.heartbeatDuration - - requestVoteExpirationTime = time.Now().Add(durationDelta) - - requestVoteSuccessfulResponsesRequiredForQuorum = (uint64(len(awaitingResponses)) + 1) / 2 - requestVoteSuccessfulResponses = 0 - - for { - timeNow = time.Now() - - if timeNow.After(requestVoteExpirationTime) || timeNow.Equal(requestVoteExpirationTime) { - // Simply return to try again - return - } - - requestVoteExpirationDurationRemaining = requestVoteExpirationTime.Sub(timeNow) - - select { - case <-globals.stateMachineStopChan: - globals.stateMachineDone.Done() - runtime.Goexit() - case <-globals.recvMsgChan: - recvMsgQueueElement = popGlobalMsg() - if nil != recvMsgQueueElement { - peer = recvMsgQueueElement.peer - switch recvMsgQueueElement.msgType { - case MsgTypeHeartBeatRequest: - msgAsHeartBeatRequest = recvMsgQueueElement.msg.(*HeartBeatRequestStruct) - if msgAsHeartBeatRequest.LeaderTerm < globals.currentTerm { - // Ignore it - } else if msgAsHeartBeatRequest.LeaderTerm == globals.currentTerm { - // Somebody else must have won the election... so convert to Follower - globals.currentLeader = peer - msgAsHeartBeatResponse = &HeartBeatResponseStruct{ - MsgType: MsgTypeHeartBeatResponse, - MsgTag: msgAsHeartBeatRequest.MsgTag, - CurrentTerm: globals.currentTerm, - Success: true, - } - _, err = sendMsg(peer, msgAsHeartBeatResponse) - if nil != err { - panic(err) - } - globals.nextState = doFollower - return - } else { // msgAsHeartBeatRequest.LeaderTerm > globals.currentTerm - globals.currentTerm = msgAsHeartBeatRequest.LeaderTerm - // We missed a subsequent election, so convert to Follower state - globals.currentLeader = peer - msgAsHeartBeatResponse = &HeartBeatResponseStruct{ - MsgType: MsgTypeHeartBeatResponse, - MsgTag: msgAsHeartBeatRequest.MsgTag, - CurrentTerm: globals.currentTerm, - Success: true, - } - _, err = sendMsg(peer, msgAsHeartBeatResponse) - if nil != err { - panic(err) - } - globals.nextState = doFollower - return - } - case MsgTypeHeartBeatResponse: - msgAsHeartBeatResponse = recvMsgQueueElement.msg.(*HeartBeatResponseStruct) - if msgAsHeartBeatResponse.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsHeartBeatResponse.CurrentTerm == globals.currentTerm { - // Unexpected... so convert to Follower state - globals.nextState = doFollower - return - } else { // msgAsHeartBeatResponse.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsHeartBeatResponse.CurrentTerm - // Convert to Follower state - globals.nextState = doFollower - return - } - case MsgTypeRequestVoteRequest: - msgAsRequestVoteRequest = recvMsgQueueElement.msg.(*RequestVoteRequestStruct) - if msgAsRequestVoteRequest.CandidateTerm < globals.currentTerm { - // Ignore it - } else if msgAsRequestVoteRequest.CandidateTerm == globals.currentTerm { - // We voted for ourself, so vote no - msgAsRequestVoteResponse = &RequestVoteResponseStruct{ - MsgType: MsgTypeRequestVoteResponse, - MsgTag: msgAsRequestVoteRequest.MsgTag, - CurrentTerm: globals.currentTerm, - VoteGranted: false, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - } else { // msgAsRequestVoteRequest.CandidateTerm > globals.currentTerm - globals.currentTerm = msgAsRequestVoteRequest.CandidateTerm - // Abandon our election, vote yes, and convert to Follower - globals.currentLeader = nil - globals.currentVote = peer - msgAsRequestVoteResponse = &RequestVoteResponseStruct{ - MsgType: MsgTypeRequestVoteResponse, - MsgTag: msgAsRequestVoteRequest.MsgTag, - CurrentTerm: globals.currentTerm, - VoteGranted: true, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - globals.nextState = doFollower - return - } - case MsgTypeRequestVoteResponse: - msgAsRequestVoteResponse = recvMsgQueueElement.msg.(*RequestVoteResponseStruct) - if msgAsRequestVoteResponse.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsRequestVoteResponse.CurrentTerm == globals.currentTerm { - if requestVoteMsgTag == msgAsRequestVoteResponse.MsgTag { - // If this is an unduplicated VoteGranted==true response, check if we are now Leader - _, ok = awaitingResponses[peer] - if ok { - delete(awaitingResponses, peer) - if msgAsRequestVoteResponse.VoteGranted { - requestVoteSuccessfulResponses++ - if requestVoteSuccessfulResponses >= requestVoteSuccessfulResponsesRequiredForQuorum { - // Convert to Leader - globals.nextState = doLeader - return - } - } - } - } else { - // Unexpected... but ignore it - } - } else { // msgAsRequestVoteResponse.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsRequestVoteResponse.CurrentTerm - // Unexpected... so convert to Follower state - globals.nextState = doFollower - return - } - case MsgTypeFetchLivenessReportRequest: - msgAsFetchLivenessReportRequest = recvMsgQueueElement.msg.(*FetchLivenessReportRequestStruct) - if msgAsFetchLivenessReportRequest.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsFetchLivenessReportRequest.CurrentTerm == globals.currentTerm { - // Unexpected... reject it - msgAsFetchLivenessReportResponse = &FetchLivenessReportResponseStruct{ - MsgType: MsgTypeFetchLivenessReportResponse, - MsgTag: msgAsFetchLivenessReportRequest.MsgTag, - CurrentTerm: globals.currentTerm, - CurrentLeader: "", - Success: false, - LivenessReport: nil, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - } else { // msgAsFetchLivenessReportRequest.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsRequestVoteResponse.CurrentTerm - // Unexpected... reject it and convert to Follower state - msgAsFetchLivenessReportResponse = &FetchLivenessReportResponseStruct{ - MsgType: MsgTypeFetchLivenessReportResponse, - MsgTag: msgAsFetchLivenessReportRequest.MsgTag, - CurrentTerm: globals.currentTerm, - CurrentLeader: "", - Success: false, - LivenessReport: nil, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - globals.nextState = doFollower - return - } - case MsgTypeFetchLivenessReportResponse: - msgAsFetchLivenessReportResponse = recvMsgQueueElement.msg.(*FetchLivenessReportResponseStruct) - if msgAsFetchLivenessReportResponse.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsFetchLivenessReportResponse.CurrentTerm == globals.currentTerm { - // Unexpected... so convert to Follower state - globals.nextState = doFollower - return - } else { // msgAsFetchLivenessReportResponse.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsHeartBeatResponse.CurrentTerm - // Convert to Follower state - globals.nextState = doFollower - return - } - default: - err = fmt.Errorf("Unexpected recvMsgQueueElement.msg: %v", reflect.TypeOf(recvMsgQueueElement.msg)) - panic(err) - } - } - case <-time.After(requestVoteExpirationDurationRemaining): - // We didn't win... but nobody else claims to have either.. so simply return to try again - return - } - } -} - -func doFollower() { - var ( - err error - heartbeatMissTime time.Time - heartbeatMissDurationRemaining time.Duration - msgAsFetchLivenessReportRequest *FetchLivenessReportRequestStruct - msgAsFetchLivenessReportResponse *FetchLivenessReportResponseStruct - msgAsHeartBeatRequest *HeartBeatRequestStruct - msgAsHeartBeatResponse *HeartBeatResponseStruct - msgAsRequestVoteRequest *RequestVoteRequestStruct - msgAsRequestVoteResponse *RequestVoteResponseStruct - observedPeerReport *ObservingPeerStruct - peer *peerStruct - recvMsgQueueElement *recvMsgQueueElementStruct - timeNow time.Time - ) - - if LogLevelStateChanges <= globals.logLevel { - logger.Infof("%s entered Follower state", globals.myUDPAddr) - } - - heartbeatMissTime = time.Now().Add(globals.heartbeatMissDuration) - - for { - timeNow = time.Now() - - if timeNow.After(heartbeatMissTime) || timeNow.Equal(heartbeatMissTime) { - globals.nextState = doCandidate - return - } - - heartbeatMissDurationRemaining = heartbeatMissTime.Sub(timeNow) - - select { - case <-globals.stateMachineStopChan: - globals.stateMachineDone.Done() - runtime.Goexit() - case <-globals.recvMsgChan: - recvMsgQueueElement = popGlobalMsg() - if nil != recvMsgQueueElement { - peer = recvMsgQueueElement.peer - switch recvMsgQueueElement.msgType { - case MsgTypeHeartBeatRequest: - msgAsHeartBeatRequest = recvMsgQueueElement.msg.(*HeartBeatRequestStruct) - if msgAsHeartBeatRequest.LeaderTerm < globals.currentTerm { - // Ignore it - } else if msgAsHeartBeatRequest.LeaderTerm == globals.currentTerm { - // In case this is the first, record .currentLeader - globals.currentLeader = peer - globals.currentVote = nil - // Update RWMode - globals.curRWMode = msgAsHeartBeatRequest.NewRWMode - err = inode.SetRWMode(globals.curRWMode) - if nil != err { - logger.FatalfWithError(err, "inode.SetRWMode(%d) failed", globals.curRWMode) - } - // Compute msgAsHeartBeatResponse.Observed & reset globals.myObservingPeerReport - globals.Lock() - observedPeerReport = convertInternalToExternalObservingPeerReport(globals.myObservingPeerReport) - updateMyObservingPeerReportWhileLocked(convertExternalToInternalObservingPeerReport(msgAsHeartBeatRequest.ToObserve)) - globals.Unlock() - globals.livenessCheckerControlChan <- true - // Send HeartBeat response - msgAsHeartBeatResponse = &HeartBeatResponseStruct{ - MsgType: MsgTypeHeartBeatResponse, - MsgTag: msgAsHeartBeatRequest.MsgTag, - CurrentTerm: globals.currentTerm, - Success: true, - Observed: observedPeerReport, - } - _, err = sendMsg(peer, msgAsHeartBeatResponse) - if nil != err { - panic(err) - } - // Reset heartBeatMissTime - heartbeatMissTime = time.Now().Add(globals.heartbeatMissDuration) - } else { // msgAsHeartBeatRequest.LeaderTerm > globals.currentTerm - globals.currentTerm = msgAsHeartBeatRequest.LeaderTerm - // We missed out on Leader election, so record .currentLeader - globals.currentLeader = peer - globals.currentVote = nil - // Update RWMode - globals.curRWMode = msgAsHeartBeatRequest.NewRWMode - err = inode.SetRWMode(globals.curRWMode) - if nil != err { - logger.FatalfWithError(err, "inode.SetRWMode(%d) failed", globals.curRWMode) - } - // Compute msgAsHeartBeatResponse.Observed & reset globals.myObservingPeerReport - globals.Lock() - observedPeerReport = convertInternalToExternalObservingPeerReport(globals.myObservingPeerReport) - updateMyObservingPeerReportWhileLocked(convertExternalToInternalObservingPeerReport(msgAsHeartBeatRequest.ToObserve)) - globals.Unlock() - globals.livenessCheckerControlChan <- true - // Send HeartBeat response - msgAsHeartBeatResponse = &HeartBeatResponseStruct{ - MsgType: MsgTypeHeartBeatResponse, - MsgTag: msgAsHeartBeatRequest.MsgTag, - CurrentTerm: globals.currentTerm, - Success: true, - Observed: observedPeerReport, - } - _, err = sendMsg(peer, msgAsHeartBeatResponse) - if nil != err { - panic(err) - } - // Reset heartBeatMissTime - heartbeatMissTime = time.Now().Add(globals.heartbeatMissDuration) - } - case MsgTypeHeartBeatResponse: - msgAsHeartBeatResponse = recvMsgQueueElement.msg.(*HeartBeatResponseStruct) - if msgAsHeartBeatResponse.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsHeartBeatResponse.CurrentTerm == globals.currentTerm { - // Unexpected... but ignore it - } else { // msgAsHeartBeatResponse.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsHeartBeatResponse.CurrentTerm - // Unexpected... but ignore it - } - case MsgTypeRequestVoteRequest: - msgAsRequestVoteRequest = recvMsgQueueElement.msg.(*RequestVoteRequestStruct) - if msgAsRequestVoteRequest.CandidateTerm < globals.currentTerm { - // Reject it - msgAsRequestVoteResponse = &RequestVoteResponseStruct{ - MsgType: MsgTypeRequestVoteResponse, - MsgTag: msgAsRequestVoteRequest.MsgTag, - CurrentTerm: globals.currentTerm, - VoteGranted: false, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - } else if msgAsRequestVoteRequest.CandidateTerm == globals.currentTerm { - if nil != globals.currentLeader { - // Candidate missed Leader election, so vote no - msgAsRequestVoteResponse = &RequestVoteResponseStruct{ - MsgType: MsgTypeRequestVoteResponse, - MsgTag: msgAsRequestVoteRequest.MsgTag, - CurrentTerm: globals.currentTerm, - VoteGranted: false, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - } else { - if peer == globals.currentVote { - // Candidate we voted for missed our yes vote and we received msg twice, so vote yes again - msgAsRequestVoteResponse = &RequestVoteResponseStruct{ - MsgType: MsgTypeRequestVoteResponse, - MsgTag: msgAsRequestVoteRequest.MsgTag, - CurrentTerm: globals.currentTerm, - VoteGranted: true, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - // Reset heartBeatMissTime - heartbeatMissTime = time.Now().Add(globals.heartbeatMissDuration) - } else { // peer != globals.currentVote - // We voted for someone else or didn't vote, so vote no - msgAsRequestVoteResponse = &RequestVoteResponseStruct{ - MsgType: MsgTypeRequestVoteResponse, - MsgTag: msgAsRequestVoteRequest.MsgTag, - CurrentTerm: globals.currentTerm, - VoteGranted: false, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - } - } - } else { // msgAsRequestVoteRequest.CandidateTerm > globals.currentTerm - globals.currentTerm = msgAsRequestVoteRequest.CandidateTerm - // Vote yes - globals.currentLeader = nil - globals.currentVote = peer - msgAsRequestVoteResponse = &RequestVoteResponseStruct{ - MsgType: MsgTypeRequestVoteResponse, - MsgTag: msgAsRequestVoteRequest.MsgTag, - CurrentTerm: globals.currentTerm, - VoteGranted: true, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - // Reset heartBeatMissTime - heartbeatMissTime = time.Now().Add(globals.heartbeatMissDuration) - } - case MsgTypeRequestVoteResponse: - msgAsRequestVoteResponse = recvMsgQueueElement.msg.(*RequestVoteResponseStruct) - if msgAsRequestVoteResponse.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsRequestVoteResponse.CurrentTerm == globals.currentTerm { - // Ignore it - } else { // msgAsRequestVoteResponse.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsRequestVoteResponse.CurrentTerm - } - case MsgTypeFetchLivenessReportRequest: - msgAsFetchLivenessReportRequest = recvMsgQueueElement.msg.(*FetchLivenessReportRequestStruct) - if msgAsFetchLivenessReportRequest.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsFetchLivenessReportRequest.CurrentTerm == globals.currentTerm { - // Unexpected... inform requestor of the actual Leader - msgAsFetchLivenessReportResponse = &FetchLivenessReportResponseStruct{ - MsgType: MsgTypeFetchLivenessReportResponse, - MsgTag: msgAsFetchLivenessReportRequest.MsgTag, - CurrentTerm: globals.currentTerm, - CurrentLeader: globals.currentLeader.udpAddr.String(), - Success: false, - LivenessReport: nil, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - } else { // msgAsFetchLivenessReportRequest.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsFetchLivenessReportRequest.CurrentTerm - msgAsFetchLivenessReportResponse = &FetchLivenessReportResponseStruct{ - MsgType: MsgTypeFetchLivenessReportResponse, - MsgTag: msgAsFetchLivenessReportRequest.MsgTag, - CurrentTerm: globals.currentTerm, - CurrentLeader: globals.currentLeader.udpAddr.String(), - Success: false, - LivenessReport: nil, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - } - case MsgTypeFetchLivenessReportResponse: - msgAsFetchLivenessReportResponse = recvMsgQueueElement.msg.(*FetchLivenessReportResponseStruct) - deliverResponse(recvMsgQueueElement.msgTag, msgAsFetchLivenessReportResponse) - default: - err = fmt.Errorf("Unexpected recvMsgQueueElement.msg: %v", reflect.TypeOf(recvMsgQueueElement.msg)) - panic(err) - } - } - case <-time.After(heartbeatMissDurationRemaining): - globals.nextState = doCandidate - return - } - } -} - -func doLeader() { - var ( - awaitingResponses map[*peerStruct]struct{} - err error - heartbeatDurationRemaining time.Duration - heartbeatMsgTag uint64 - heartbeatSendTime time.Time - heartbeatSuccessfulResponses uint64 - heartbeatSuccessfulResponsesRequiredForQuorum uint64 - livenessReportThisHeartBeat *internalLivenessReportStruct - maxDiskUsagePercentage uint8 - msgAsFetchLivenessReportRequest *FetchLivenessReportRequestStruct - msgAsFetchLivenessReportResponse *FetchLivenessReportResponseStruct - msgAsHeartBeatRequest *HeartBeatRequestStruct - msgAsHeartBeatResponse *HeartBeatResponseStruct - msgAsRequestVoteRequest *RequestVoteRequestStruct - msgAsRequestVoteResponse *RequestVoteResponseStruct - ok bool - peer *peerStruct - observingPeerReport *internalObservingPeerReportStruct - quorumMembersLastHeartBeat []string - quorumMembersThisHeartBeat []string - reconEndpointReport *internalReconEndpointReportStruct - recvMsgQueueElement *recvMsgQueueElementStruct - timeNow time.Time - ) - - if LogLevelStateChanges <= globals.logLevel { - logger.Infof("%s entered Leader state", globals.myUDPAddr) - } - - heartbeatSendTime = time.Now() // Force first time through for{} loop to send a heartbeat - - quorumMembersThisHeartBeat = []string{globals.whoAmI} - - globals.Lock() - globals.myObservingPeerReport = &internalObservingPeerReportStruct{ - name: globals.whoAmI, - servingPeer: make(map[string]*internalServingPeerReportStruct), - reconEndpoint: make(map[string]*internalReconEndpointReportStruct), - } - globals.Unlock() - globals.livenessCheckerControlChan <- true - - livenessReportThisHeartBeat = &internalLivenessReportStruct{ - observingPeer: make(map[string]*internalObservingPeerReportStruct), - } - - for { - timeNow = time.Now() - - if timeNow.Before(heartbeatSendTime) { - heartbeatDurationRemaining = heartbeatSendTime.Sub(timeNow) - } else { - globals.Lock() - - mergeObservingPeerReportIntoLivenessReport(globals.myObservingPeerReport, livenessReportThisHeartBeat) - - globals.livenessReport = livenessReportThisHeartBeat - - quorumMembersLastHeartBeat = make([]string, len(quorumMembersThisHeartBeat)) - _ = copy(quorumMembersLastHeartBeat, quorumMembersThisHeartBeat) - - livenessReportThisHeartBeat = computeLivenessCheckAssignments(quorumMembersLastHeartBeat) - - updateMyObservingPeerReportWhileLocked(livenessReportThisHeartBeat.observingPeer[globals.whoAmI]) - - globals.Unlock() - - quorumMembersThisHeartBeat = make([]string, 1, 1+len(globals.peersByName)) - quorumMembersThisHeartBeat[0] = globals.whoAmI - - heartbeatMsgTag = fetchNonce() - - awaitingResponses = make(map[*peerStruct]struct{}) - - for _, peer = range globals.peersByName { - msgAsHeartBeatRequest = &HeartBeatRequestStruct{ - MsgType: MsgTypeHeartBeatRequest, - MsgTag: heartbeatMsgTag, - LeaderTerm: globals.currentTerm, - NewRWMode: globals.curRWMode, - ToObserve: convertInternalToExternalObservingPeerReport(livenessReportThisHeartBeat.observingPeer[peer.name]), - } - - _, err = sendMsg(peer, msgAsHeartBeatRequest) - if nil != err { - panic(err) - } - - awaitingResponses[peer] = struct{}{} - } - - heartbeatSendTime = timeNow.Add(globals.heartbeatDuration) - heartbeatDurationRemaining = globals.heartbeatDuration - - heartbeatSuccessfulResponsesRequiredForQuorum = (uint64(len(awaitingResponses)) + 1) / 2 - heartbeatSuccessfulResponses = 0 - } - - select { - case <-globals.stateMachineStopChan: - globals.stateMachineDone.Done() - runtime.Goexit() - case <-globals.recvMsgChan: - recvMsgQueueElement = popGlobalMsg() - if nil != recvMsgQueueElement { - peer = recvMsgQueueElement.peer - switch recvMsgQueueElement.msgType { - case MsgTypeHeartBeatRequest: - msgAsHeartBeatRequest = recvMsgQueueElement.msg.(*HeartBeatRequestStruct) - if msgAsHeartBeatRequest.LeaderTerm < globals.currentTerm { - // Ignore it - } else if msgAsHeartBeatRequest.LeaderTerm == globals.currentTerm { - // Unexpected... so convert to Candidate state - msgAsHeartBeatResponse = &HeartBeatResponseStruct{ - MsgType: MsgTypeHeartBeatResponse, - MsgTag: msgAsHeartBeatRequest.MsgTag, - CurrentTerm: globals.currentTerm, - Success: false, - } - _, err = sendMsg(peer, msgAsHeartBeatResponse) - if nil != err { - panic(err) - } - globals.nextState = doCandidate - return - } else { // msgAsHeartBeatRequest.LeaderTerm > globals.currentTerm - globals.currentTerm = msgAsHeartBeatRequest.LeaderTerm - // We missed a subsequent election, so convert to Follower state - globals.currentLeader = peer - msgAsHeartBeatResponse = &HeartBeatResponseStruct{ - MsgType: MsgTypeHeartBeatResponse, - MsgTag: msgAsHeartBeatRequest.MsgTag, - CurrentTerm: globals.currentTerm, - Success: true, - } - _, err = sendMsg(peer, msgAsHeartBeatResponse) - if nil != err { - panic(err) - } - globals.nextState = doFollower - return - } - case MsgTypeHeartBeatResponse: - msgAsHeartBeatResponse = recvMsgQueueElement.msg.(*HeartBeatResponseStruct) - if msgAsHeartBeatResponse.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsHeartBeatResponse.CurrentTerm == globals.currentTerm { - if heartbeatMsgTag == msgAsHeartBeatResponse.MsgTag { - _, ok = awaitingResponses[peer] - if ok { - delete(awaitingResponses, peer) - if msgAsHeartBeatResponse.Success { - heartbeatSuccessfulResponses++ - quorumMembersThisHeartBeat = append(quorumMembersThisHeartBeat, peer.name) - if nil != msgAsHeartBeatResponse.Observed { - observingPeerReport = convertExternalToInternalObservingPeerReport(msgAsHeartBeatResponse.Observed) - if nil != observingPeerReport { - mergeObservingPeerReportIntoLivenessReport(observingPeerReport, livenessReportThisHeartBeat) - } - } - } else { - // Unexpected... so convert to Follower state - globals.nextState = doFollower - return - } - } - } else { - // Ignore it - } - } else { // msgAsHeartBeatResponse.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsHeartBeatResponse.CurrentTerm - // Convert to Follower state - globals.nextState = doFollower - return - } - case MsgTypeRequestVoteRequest: - msgAsRequestVoteRequest = recvMsgQueueElement.msg.(*RequestVoteRequestStruct) - if msgAsRequestVoteRequest.CandidateTerm < globals.currentTerm { - // Ignore it - } else if msgAsRequestVoteRequest.CandidateTerm == globals.currentTerm { - // Ignore it - } else { // msgAsRequestVoteRequest.CandidateTerm > globals.currentTerm - globals.currentTerm = msgAsRequestVoteRequest.CandidateTerm - // Abandon our Leadership, vote yes, and convert to Follower - globals.currentLeader = nil - globals.currentVote = peer - msgAsRequestVoteResponse = &RequestVoteResponseStruct{ - MsgType: MsgTypeRequestVoteResponse, - MsgTag: msgAsRequestVoteRequest.MsgTag, - CurrentTerm: globals.currentTerm, - VoteGranted: true, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - globals.nextState = doFollower - return - } - case MsgTypeRequestVoteResponse: - msgAsRequestVoteResponse = recvMsgQueueElement.msg.(*RequestVoteResponseStruct) - if msgAsRequestVoteResponse.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsRequestVoteResponse.CurrentTerm == globals.currentTerm { - // Ignore it - } else { // msgAsRequestVoteResponse.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsRequestVoteResponse.CurrentTerm - // Unexpected... so convert to Follower state - globals.nextState = doFollower - return - } - case MsgTypeFetchLivenessReportRequest: - msgAsFetchLivenessReportRequest = recvMsgQueueElement.msg.(*FetchLivenessReportRequestStruct) - if msgAsFetchLivenessReportRequest.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsFetchLivenessReportRequest.CurrentTerm == globals.currentTerm { - msgAsFetchLivenessReportResponse = &FetchLivenessReportResponseStruct{ - MsgType: MsgTypeFetchLivenessReportResponse, - MsgTag: msgAsFetchLivenessReportRequest.MsgTag, - CurrentTerm: globals.currentTerm, - CurrentLeader: globals.myUDPAddr.String(), - } - msgAsFetchLivenessReportResponse.LivenessReport = convertInternalToExternalLivenessReport(globals.livenessReport) - msgAsFetchLivenessReportResponse.Success = (nil != msgAsFetchLivenessReportResponse.LivenessReport) - _, err = sendMsg(peer, msgAsFetchLivenessReportResponse) - if nil != err { - panic(err) - } - } else { // msgAsFetchLivenessReportRequest.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsFetchLivenessReportRequest.CurrentTerm - // Unexpected... reject it and convert to Follower state - msgAsFetchLivenessReportResponse = &FetchLivenessReportResponseStruct{ - MsgType: MsgTypeFetchLivenessReportResponse, - MsgTag: msgAsFetchLivenessReportRequest.MsgTag, - CurrentTerm: globals.currentTerm, - CurrentLeader: "", - Success: false, - LivenessReport: nil, - } - _, err = sendMsg(peer, msgAsRequestVoteResponse) - if nil != err { - panic(err) - } - globals.nextState = doFollower - return - } - case MsgTypeFetchLivenessReportResponse: - msgAsFetchLivenessReportResponse = recvMsgQueueElement.msg.(*FetchLivenessReportResponseStruct) - if msgAsFetchLivenessReportResponse.CurrentTerm < globals.currentTerm { - // Ignore it - } else if msgAsFetchLivenessReportResponse.CurrentTerm == globals.currentTerm { - // Unexpected... so convert to Follower state - globals.nextState = doFollower - return - } else { // msgAsFetchLivenessReportResponse.CurrentTerm > globals.currentTerm - globals.currentTerm = msgAsHeartBeatResponse.CurrentTerm - // Convert to Follower state - globals.nextState = doFollower - return - } - default: - err = fmt.Errorf("Unexpected recvMsgQueueElement.msg: %v", reflect.TypeOf(recvMsgQueueElement.msg)) - panic(err) - } - } - case <-time.After(heartbeatDurationRemaining): - if heartbeatSuccessfulResponses >= heartbeatSuccessfulResponsesRequiredForQuorum { - // Compute new RWMode - - maxDiskUsagePercentage = 0 - - for _, observingPeerReport = range globals.livenessReport.observingPeer { - for _, reconEndpointReport = range observingPeerReport.reconEndpoint { - if reconEndpointReport.maxDiskUsagePercentage > maxDiskUsagePercentage { - maxDiskUsagePercentage = reconEndpointReport.maxDiskUsagePercentage - } - } - } - - if maxDiskUsagePercentage >= globals.swiftReconReadOnlyThreshold { - globals.curRWMode = inode.RWModeReadOnly - } else if maxDiskUsagePercentage >= globals.swiftReconNoWriteThreshold { - globals.curRWMode = inode.RWModeNoWrite - } else { - globals.curRWMode = inode.RWModeNormal - } - - err = inode.SetRWMode(globals.curRWMode) - if nil != err { - logger.FatalfWithError(err, "inode.SetRWMode(%d) failed", globals.curRWMode) - } - - // Now just loop back and issue a fresh HeartBeat - } else { - // Quorum lost... convert to Candidate state - globals.nextState = doCandidate - return - } - } - } -} diff --git a/make-static-content/.gitignore b/make-static-content/.gitignore deleted file mode 100644 index 61c4948c..00000000 --- a/make-static-content/.gitignore +++ /dev/null @@ -1 +0,0 @@ -make-static-content diff --git a/make-static-content/Makefile b/make-static-content/Makefile deleted file mode 100644 index e8fd6003..00000000 --- a/make-static-content/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/make-static-content - -generatedfiles := \ - make-static-content - -include ../GoMakefile diff --git a/make-static-content/dummy_test.go b/make-static-content/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/make-static-content/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/make-static-content/main.go b/make-static-content/main.go deleted file mode 100644 index 76d90c65..00000000 --- a/make-static-content/main.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" -) - -const bytesPerLine = 16 - -func usage() { - fmt.Println("make-static-content -?") - fmt.Println(" Prints this help text") - fmt.Println("make-static-content ") - fmt.Println(" is the name of the ultimate package for ") - fmt.Println(" is the basename of the desired content resource") - fmt.Println(" is the string to record as the static content's Content-Type") - fmt.Println(" indicates whether the static content is a string (\"s\") or a []byte (\"b\")") - fmt.Println(" is the path to the static content to be embedded") - fmt.Println(" is the name of the generated .go source file containing:") - fmt.Println(" ContentType string holding value of ") - fmt.Println(" Content string or []byte holding contents of ") -} - -var bs = []byte{} - -func main() { - var ( - contentFormat string - contentName string - contentType string - dstFile *os.File - dstFileContents bytes.Buffer - dstFileName string - err error - packageName string - srcFileContentByte byte - srcFileContentIndex int - srcFileContents []byte - srcFileName string - ) - - if (2 == len(os.Args)) && ("-?" == os.Args[1]) { - usage() - os.Exit(0) - } - - if 7 != len(os.Args) { - usage() - os.Exit(1) - } - - packageName = os.Args[1] - contentName = os.Args[2] - contentType = os.Args[3] - contentFormat = os.Args[4] - srcFileName = os.Args[5] - dstFileName = os.Args[6] - - srcFileContents, err = ioutil.ReadFile(srcFileName) - if nil != err { - panic(err.Error()) - } - - dstFile, err = os.Create(dstFileName) - if nil != err { - panic(err.Error()) - } - - _, err = dstFile.Write([]byte(fmt.Sprintf("// Code generated by \"go run make_static_content.go %v %v %v %v %v %v\" - DO NOT EDIT\n\n", packageName, contentName, contentType, contentFormat, srcFileName, dstFileName))) - if nil != err { - panic(err.Error()) - } - _, err = dstFile.Write([]byte(fmt.Sprintf("package %v\n\n", packageName))) - if nil != err { - panic(err.Error()) - } - - _, err = dstFile.Write([]byte(fmt.Sprintf("const %vContentType = \"%v\"\n\n", contentName, contentType))) - if nil != err { - panic(err.Error()) - } - - switch contentFormat { - case "s": - _, err = dstFile.Write([]byte(fmt.Sprintf("const %vContent = `%v`\n", contentName, string(srcFileContents[:])))) - if nil != err { - panic(err.Error()) - } - case "b": - _, err = dstFileContents.Write([]byte(fmt.Sprintf("var %vContent = []byte{", contentName))) - if nil != err { - panic(err.Error()) - } - for srcFileContentIndex, srcFileContentByte = range srcFileContents { - if (srcFileContentIndex % bytesPerLine) == 0 { - _, err = dstFileContents.Write([]byte(fmt.Sprintf("\n\t0x%02X,", srcFileContentByte))) - } else { - _, err = dstFileContents.Write([]byte(fmt.Sprintf(" 0x%02X,", srcFileContentByte))) - } - if nil != err { - panic(err.Error()) - } - } - _, err = dstFileContents.Write([]byte("\n}\n")) - if nil != err { - panic(err.Error()) - } - _, err = dstFile.Write(dstFileContents.Bytes()) - if nil != err { - panic(err.Error()) - } - default: - usage() - os.Exit(1) - } - - err = dstFile.Close() - if nil != err { - panic(err.Error()) - } - - os.Exit(0) -} diff --git a/meta_middleware/meta_middleware/__init__.py b/meta_middleware/meta_middleware/__init__.py deleted file mode 100644 index beaaeb8f..00000000 --- a/meta_middleware/meta_middleware/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -from .middleware import filter_factory, MetaMiddleware # noqa diff --git a/meta_middleware/meta_middleware/middleware.py b/meta_middleware/meta_middleware/middleware.py deleted file mode 100644 index 4339c0a0..00000000 --- a/meta_middleware/meta_middleware/middleware.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -class MetaMiddleware(object): - def __init__(self, app, conf): - self.app = app - - def __call__(self, env, start_response): - hToDel = list() - vToAdd = list() - for h in env: - if h.upper() == 'HTTP_X_PROXYFS_BIMODAL': - hToDel.append(h) - vToAdd.append(env[h]) - for h in hToDel: - del env[h] - for v in vToAdd: - # NB: only last one, if multiple, will determine value - env['HTTP_X_ACCOUNT_SYSMETA_PROXYFS_BIMODAL'] = v - - def meta_response(status, response_headers, exc_info=None): - hvToDel = list() - vToAdd = list() - for (h, v) in response_headers: - if h.upper() == 'X-ACCOUNT-SYSMETA-PROXYFS-BIMODAL': - hvToDel.append((h, v)) - vToAdd.append(v) - for hv in hvToDel: - response_headers.remove(hv) - for v in vToAdd: - # potentially multiple instances of same header - response_headers.append(('X-ProxyFS-BiModal', v)) - return start_response(status, response_headers, exc_info) - - return self.app(env, meta_response) - - -def filter_factory(global_conf, **local_conf): - conf = global_conf.copy() - conf.update(local_conf) - - def meta_filter(app): - return MetaMiddleware(app, conf) - - return meta_filter diff --git a/meta_middleware/setup.py b/meta_middleware/setup.py deleted file mode 100644 index c3d1e8f9..00000000 --- a/meta_middleware/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -from setuptools import setup - - -setup( - name="meta_middleware", - version="0.0.1", - author="SwiftStack Inc.", - description="WSGI middleware component of ProxyFS", - license="Apache", - packages=["meta_middleware"], - - test_suite="nose.collector", - tests_require=['nose'], - - # TODO: classifiers and such - entry_points={ - 'paste.filter_factory': [ - 'meta = meta_middleware.middleware:filter_factory', - ], - }, -) diff --git a/meta_middleware/test-requirements.txt b/meta_middleware/test-requirements.txt deleted file mode 100644 index 3dfbd5d6..00000000 --- a/meta_middleware/test-requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -flake8 -mock diff --git a/meta_middleware/tests/__init__.py b/meta_middleware/tests/__init__.py deleted file mode 100644 index 200c192d..00000000 --- a/meta_middleware/tests/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -# Work around http://bugs.python.org/issue9501 -import logging -logging.raiseExceptions = False diff --git a/meta_middleware/tox.ini b/meta_middleware/tox.ini deleted file mode 100644 index 067b66cc..00000000 --- a/meta_middleware/tox.ini +++ /dev/null @@ -1,23 +0,0 @@ -[tox] -envlist = lint - -[testenv] -deps = -r{toxinidir}/test-requirements.txt -commands = python -m unittest discover - -[testenv:lint] -commands = flake8 {posargs:meta_middleware tests setup.py} - -[flake8] -# flake8 has opinions with which we agree, for the most part. However, -# flake8 has a plugin mechanism by which other people can add their -# opinions; we do not necessarily like those opinions. In particular, -# "hacking" adds many different checks, a significant number of which -# are completely bogus. Fortunately, they have a convention: hacking -# checks start with "H", so that's what we ignore. -ignore = H, - # Both stupid binary opeator things - W503, - W504 -exclude = .venv,.tox,dist,*egg -show-source = true diff --git a/mkproxyfs/Makefile b/mkproxyfs/Makefile deleted file mode 100644 index fd201864..00000000 --- a/mkproxyfs/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/mkproxyfs - -include ../GoMakefile diff --git a/mkproxyfs/api.go b/mkproxyfs/api.go deleted file mode 100644 index c2b92c88..00000000 --- a/mkproxyfs/api.go +++ /dev/null @@ -1,352 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package mkproxyfs - -import ( - "context" - "fmt" - "net/http" - "os" - "strings" - "time" - - etcd "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/etcdclient" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/transitions" - "github.com/NVIDIA/proxyfs/version" -) - -type Mode int - -const ( - ModeNew Mode = iota - ModeOnlyIfNeeded - ModeReformat -) - -func Format(mode Mode, volumeNameToFormat string, confFile string, confStrings []string, execArgs []string) (err error) { - var ( - accountName string - cancel context.CancelFunc - checkpointEtcdKeyName string - confMap conf.ConfMap - containerList []string - containerName string - ctx context.Context - etcdAutoSyncInterval time.Duration - etcdClient *etcd.Client - etcdDialTimeout time.Duration - etcdEnabled bool - etcdCertDir string - etcdEndpoints []string - etcdKV etcd.KV - etcdOpTimeout time.Duration - getResponse *etcd.GetResponse - isEmpty bool - objectList []string - objectName string - replayLogFileName string - whoAmI string - ) - - // Valid mode? - - switch mode { - case ModeNew: - case ModeOnlyIfNeeded: - case ModeReformat: - default: - err = fmt.Errorf("mode (%v) must be one of ModeNew (%v), ModeOnlyIfNeeded (%v), or ModeReformat (%v)", mode, ModeNew, ModeOnlyIfNeeded, ModeReformat) - return - } - - // Load confFile & confStrings (overrides) - - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - err = fmt.Errorf("failed to load config: %v", err) - return - } - - err = confMap.UpdateFromStrings(confStrings) - if nil != err { - err = fmt.Errorf("failed to apply config overrides: %v", err) - return - } - - // Upgrade confMap if necessary - // [should be removed once backwards compatibility is no longer required...] - - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - err = fmt.Errorf("failed to upgrade config: %v", err) - return - } - - // Update confMap to specify an empty FSGlobals.VolumeGroupList - - err = confMap.UpdateFromString("FSGlobals.VolumeGroupList=") - if nil != err { - err = fmt.Errorf("failed to empty config VolumeGroupList: %v", err) - return - } - - // Fetch confMap particulars needed below - - accountName, err = confMap.FetchOptionValueString("Volume:"+volumeNameToFormat, "AccountName") - if nil != err { - return - } - - etcdEnabled, err = confMap.FetchOptionValueBool("FSGlobals", "EtcdEnabled") - if nil != err { - etcdEnabled = false // Current default - } - - if etcdEnabled { - etcdCertDir, err = confMap.FetchOptionValueString("FSGlobals", "EtcdCertDir") - if nil != err { - return - } - etcdEndpoints, err = confMap.FetchOptionValueStringSlice("FSGlobals", "EtcdEndpoints") - if nil != err { - return - } - etcdAutoSyncInterval, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdAutoSyncInterval") - if nil != err { - return - } - etcdDialTimeout, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdDialTimeout") - if nil != err { - return - } - etcdOpTimeout, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdOpTimeout") - if nil != err { - return - } - - checkpointEtcdKeyName, err = confMap.FetchOptionValueString("Volume:"+volumeNameToFormat, "CheckpointEtcdKeyName") - if nil != err { - return - } - - tlsInfo := transport.TLSInfo{ - CertFile: etcdclient.GetCertFilePath(etcdCertDir), - KeyFile: etcdclient.GetKeyFilePath(etcdCertDir), - TrustedCAFile: etcdclient.GetCA(etcdCertDir), - } - - // Initialize etcd Client & KV objects - etcdClient, err = etcdclient.New(&tlsInfo, etcdEndpoints, - etcdAutoSyncInterval, etcdDialTimeout) - if nil != err { - return - } - - etcdKV = etcd.NewKV(etcdClient) - } - - // Call transitions.Up() with empty FSGlobals.VolumeGroupList - - err = transitions.Up(confMap) - if nil != err { - return - } - - logger.Infof("mkproxyfs is starting up (version %s) (PID %d); invoked as '%s'", - version.ProxyFSVersion, os.Getpid(), strings.Join(execArgs, "' '")) - - // Determine if underlying accountName is empty - - _, containerList, err = swiftclient.AccountGet(accountName) - if nil == err { - // accountName exists (possibly auto-created)... consider it empty only if no containers therein - isEmpty = (0 == len(containerList)) - } else { - if http.StatusNotFound == blunder.HTTPCode(err) { - // accountName does not exist, so accountName is empty - isEmpty = true - } else { - _ = transitions.Down(confMap) - err = fmt.Errorf("failed to GET %v: %v", accountName, err) - return - } - } - - // Adjust isEmpty based on etcd - - if isEmpty && etcdEnabled { - ctx, cancel = context.WithTimeout(context.Background(), etcdOpTimeout) - getResponse, err = etcdKV.Get(ctx, checkpointEtcdKeyName) - cancel() - if nil != err { - err = fmt.Errorf("Error contacting etcd [Case 1]: %v", err) - return - } - - if 0 < getResponse.Count { - isEmpty = false - } - } - - if !isEmpty { - switch mode { - case ModeNew: - // If Swift Account is not empty && ModeNew, exit with failure - - _ = transitions.Down(confMap) - err = fmt.Errorf("%v found to be non-empty with mode == ModeNew (%v)", accountName, ModeNew) - return - case ModeOnlyIfNeeded: - // If Swift Account is not empty && ModeOnlyIfNeeded, exit successfully - - _ = transitions.Down(confMap) - err = nil - return - case ModeReformat: - // If Swift Account is not empty && ModeReformat, clear out accountName - - for !isEmpty { - for _, containerName = range containerList { - _, objectList, err = swiftclient.ContainerGet(accountName, containerName) - if nil != err { - _ = transitions.Down(confMap) - err = fmt.Errorf("failed to GET %v/%v: %v", accountName, containerName, err) - return - } - - isEmpty = (0 == len(objectList)) - - for !isEmpty { - for _, objectName = range objectList { - err = swiftclient.ObjectDelete(accountName, containerName, objectName, 0) - if nil != err { - _ = transitions.Down(confMap) - err = fmt.Errorf("failed to DELETE %v/%v/%v: %v", accountName, containerName, objectName, err) - return - } - } - - _, objectList, err = swiftclient.ContainerGet(accountName, containerName) - if nil != err { - _ = transitions.Down(confMap) - err = fmt.Errorf("failed to GET %v/%v: %v", accountName, containerName, err) - return - } - - isEmpty = (0 == len(objectList)) - } - - err = swiftclient.ContainerDelete(accountName, containerName) - if nil != err { - _ = transitions.Down(confMap) - err = fmt.Errorf("failed to DELETE %v/%v: %v", accountName, containerName, err) - return - } - } - - _, containerList, err = swiftclient.AccountGet(accountName) - if nil != err { - _ = transitions.Down(confMap) - err = fmt.Errorf("failed to GET %v: %v", accountName, err) - return - } - - isEmpty = (0 == len(containerList)) - } - - // Clear etcd if it exists... - - if etcdEnabled { - ctx, cancel = context.WithTimeout(context.Background(), etcdOpTimeout) - _, err = etcdKV.Delete(ctx, checkpointEtcdKeyName) - cancel() - if nil != err { - err = fmt.Errorf("Error contacting etcd [Case 2]: %v", err) - return - } - } - - replayLogFileName, err = confMap.FetchOptionValueString("Volume:"+volumeNameToFormat, "ReplayLogFileName") - if nil == err { - if "" != replayLogFileName { - removeReplayLogFileErr := os.Remove(replayLogFileName) - if nil != removeReplayLogFileErr { - if !os.IsNotExist(removeReplayLogFileErr) { - _ = transitions.Down(confMap) - err = fmt.Errorf("os.Remove(replayLogFileName == \"%v\") returned unexpected error: %v", replayLogFileName, removeReplayLogFileErr) - return - } - } - } - } - } - } - - // Call transitions.Down() before restarting to do format - - err = transitions.Down(confMap) - if nil != err { - return - } - - // Update confMap to specify only volumeNameToFormat with AuthFormat set to true - - whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - return - } - - err = confMap.UpdateFromStrings([]string{ - "FSGlobals.VolumeGroupList=MKPROXYFS", - "VolumeGroup:MKPROXYFS.VolumeList=" + volumeNameToFormat, - "VolumeGroup:MKPROXYFS.VirtualIPAddr=", - "VolumeGroup:MKPROXYFS.PrimaryPeer=" + whoAmI, - "Volume:" + volumeNameToFormat + ".AutoFormat=true"}) - if nil != err { - err = fmt.Errorf("failed to retarget config at only %s: %v", volumeNameToFormat, err) - return - } - - // Restart... this time AutoFormat of volumeNameToFormat will be applied - - err = transitions.Up(confMap) - if nil != err { - return - } - - // Make reference to package headhunter to ensure it gets registered with package transitions - - _, err = headhunter.FetchVolumeHandle(volumeNameToFormat) - if nil != err { - return - } - - // With format complete, we can shutdown for good - - err = transitions.Down(confMap) - if nil != err { - return - } - - // Close down etcd - - if etcdEnabled { - etcdKV = nil - - err = etcdClient.Close() - if nil != err { - return - } - } - - return -} diff --git a/mkproxyfs/dummy_test.go b/mkproxyfs/dummy_test.go deleted file mode 100644 index 6ae6c340..00000000 --- a/mkproxyfs/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package mkproxyfs - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/mkproxyfs/mkproxyfs/Makefile b/mkproxyfs/mkproxyfs/Makefile deleted file mode 100644 index 66c1094a..00000000 --- a/mkproxyfs/mkproxyfs/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/mkproxyfs/mkproxyfs - -include ../../GoMakefile diff --git a/mkproxyfs/mkproxyfs/dummy_test.go b/mkproxyfs/mkproxyfs/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/mkproxyfs/mkproxyfs/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/mkproxyfs/mkproxyfs/main.go b/mkproxyfs/mkproxyfs/main.go deleted file mode 100644 index 5d4610ac..00000000 --- a/mkproxyfs/mkproxyfs/main.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// The mkproxyfs program is the command line form invoking the mkproxyfs package's Format() function. - -package main - -import ( - "fmt" - "os" - - "github.com/NVIDIA/proxyfs/mkproxyfs" -) - -func usage() { - fmt.Println("mkproxyfs -?") - fmt.Println(" Prints this help text") - fmt.Println("mkproxyfs -N|-I|-F VolumeNameToFormat ConfFile [ConfFileOverrides]*") - fmt.Println(" -N indicates that VolumeNameToFormat must be empty") - fmt.Println(" -I indicates that VolumeNameToFormat should only be formatted if necessary") - fmt.Println(" -F indicates that VolumeNameToFormat should be formatted regardless") - fmt.Println(" Note: This may take awhile to clear out existing objects/containers") - fmt.Println(" VolumeNameToFormat indicates which Volume in ConfFile is to be formatted") - fmt.Println(" Note: VolumeNameToFormat need not be marked as active on any peer") - fmt.Println(" ConfFile specifies the .conf file as also passed to proxyfsd et. al.") - fmt.Println(" ConfFileOverrides is an optional list of modifications to ConfFile to apply") -} - -func main() { - var ( - err error - mode mkproxyfs.Mode - ) - - if (2 == len(os.Args)) && ("-?" == os.Args[1]) { - usage() - os.Exit(0) - } - - if 4 > len(os.Args) { - usage() - os.Exit(1) - } - - switch os.Args[1] { - case "-N": - mode = mkproxyfs.ModeNew - case "-I": - mode = mkproxyfs.ModeOnlyIfNeeded - case "-F": - mode = mkproxyfs.ModeReformat - default: - usage() - os.Exit(1) - } - - err = mkproxyfs.Format(mode, os.Args[2], os.Args[3], os.Args[4:], os.Args) - if nil == err { - os.Exit(0) - } else { - fmt.Fprintf(os.Stderr, "mkproxyfs: Format() returned error: %v\n", err) - os.Exit(1) - } -} diff --git a/pfs-crash/Makefile b/pfs-crash/Makefile deleted file mode 100644 index 20de57d2..00000000 --- a/pfs-crash/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfs-crash - -include ../GoMakefile diff --git a/pfs-crash/b_tree_load.sh b/pfs-crash/b_tree_load.sh deleted file mode 100755 index e00c9b8c..00000000 --- a/pfs-crash/b_tree_load.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -while true -do - for j in `seq 10` - do - for k in `seq 100` - do - echo "Hi" >> $1/f_$k - done - done - for k in `seq 100` - do - rm $1/f_$k - done -done diff --git a/pfs-crash/dd4Kx256.sh b/pfs-crash/dd4Kx256.sh deleted file mode 100755 index 4399336c..00000000 --- a/pfs-crash/dd4Kx256.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -while true -do - for i in `seq 100` - do - dd if=/dev/zero of=$1/dd1M_$i bs=4k count=256 - done -done diff --git a/pfs-crash/dummy_test.go b/pfs-crash/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfs-crash/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfs-crash/fio.sh b/pfs-crash/fio.sh deleted file mode 100755 index ea9a3860..00000000 --- a/pfs-crash/fio.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -while true -do - fio --name=writetest --size=100MiB --bs=4k --rw=randwrite --direct=1 --buffered=0 --iodepth=32 --filename=$1/fio_scratch_file -done diff --git a/pfs-crash/main.go b/pfs-crash/main.go deleted file mode 100644 index e3954112..00000000 --- a/pfs-crash/main.go +++ /dev/null @@ -1,869 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bufio" - "bytes" - "container/list" - cryptoRand "crypto/rand" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "math/big" - mathRand "math/rand" - "net" - "net/http" - "os" - "os/exec" - "os/signal" - "path/filepath" - "strconv" - "strings" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/httpserver" - "github.com/NVIDIA/proxyfs/transitions" -) - -const ( - proxyfsdHalterMinHaltAfterCount = uint64(400) - proxyfsdHalterMaxHaltAfterCount = uint64(800) - - proxyfsdMinKillDelay = 10 * time.Second - proxyfsdMaxKillDelay = 20 * time.Second - - proxyfsdPollDelay = 100 * time.Millisecond - - openDirPollDelay = 100 * time.Millisecond - - pseudoRandom = false - pseudoRandomSeed = int64(0) -) - -type queryMethodType uint16 - -const ( - queryMethodGET queryMethodType = iota - queryMethodPOST -) - -var ( - confFile string - fuseMountPointName string - haltLabelStrings []string - includeHalterTriggers bool - ipAddrTCPPort string - mathRandSource *mathRand.Rand // A source for pseudo-random numbers (if selected) - proxyfsdArgs []string - proxyfsdCmd *exec.Cmd - proxyfsdCmdWaitChan chan error - timeoutChan chan bool - trafficCmd *exec.Cmd - trafficCmdWaitChan chan error - trafficScript string - volumeName string - - signalExpandedStringMap = map[string]string{"interrupt": "SIGINT(2)", "terminated": "SIGTERM(15)", "killed": "SIGKILL(9)"} -) - -func usage() { - fmt.Printf("%v {+|-}[] []*\n", os.Args[0]) - fmt.Println(" where:") - fmt.Println(" + indicates to include halter trigger to halt ProxyFS") - fmt.Println(" - indicates to not incluce halter trigger to halt ProxyFS") - fmt.Println(" launch trafficScript bash script to generate workload") - fmt.Println() - fmt.Println("Note: If trafficScript is supplied, the script should infinitely loop.") - fmt.Println(" The $1 arg to trafficScript will specify the FUSE MountPoint.") -} - -func main() { - var ( - confMap conf.ConfMap - confStrings []string - contentsAsStrings []string - err error - haltLabelString string - haltLabelStringSplit []string - httpServerTCPPort uint16 - httpStatusCode int - lenArgs int - mkproxyfsArgs []string - mkproxyfsCmd *exec.Cmd - nextHalterTriggerIndex int - peerSectionName string - privateIPAddr string - randomHaltAfterCount uint64 - randomKillDelay time.Duration - signalChan chan os.Signal - signalToSend os.Signal - triggerBoolAndTrafficScript string - whoAmI string - ) - - lenArgs = len(os.Args) - if 1 == lenArgs { - usage() - os.Exit(0) - } - if 4 > lenArgs { - usage() - os.Exit(-1) - } - - triggerBoolAndTrafficScript = os.Args[1] - - if 0 == len(triggerBoolAndTrafficScript) { - usage() - os.Exit(-1) - } - - switch triggerBoolAndTrafficScript[0] { - case '+': - includeHalterTriggers = true - case '-': - includeHalterTriggers = false - default: - usage() - os.Exit(-1) - } - - trafficScript = triggerBoolAndTrafficScript[1:] - - volumeName = os.Args[2] - confFile = os.Args[3] - - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - log.Fatal(err) - } - - mkproxyfsArgs = []string{"-F", volumeName, confFile} - proxyfsdArgs = []string{confFile} - - if 4 < lenArgs { - confStrings = os.Args[4:] - - err = confMap.UpdateFromStrings(confStrings) - if nil != err { - log.Fatalf("failed to apply config overrides: %v", err) - } - - mkproxyfsArgs = append(mkproxyfsArgs, confStrings...) - proxyfsdArgs = append(proxyfsdArgs, confStrings...) - } - - // Upgrade confMap if necessary - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - log.Fatalf("Failed to upgrade config: %v", err) - } - - whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - log.Fatal(err) - } - - peerSectionName = "Peer:" + whoAmI - - privateIPAddr, err = confMap.FetchOptionValueString(peerSectionName, "PrivateIPAddr") - if nil != err { - log.Fatal(err) - } - - httpServerTCPPort, err = confMap.FetchOptionValueUint16("HTTPServer", "TCPPort") - if nil != err { - log.Fatal(err) - } - - ipAddrTCPPort = net.JoinHostPort(privateIPAddr, strconv.Itoa(int(httpServerTCPPort))) - - fuseMountPointName, err = confMap.FetchOptionValueString("Volume:"+volumeName, "FUSEMountPointName") - if nil != err { - log.Fatal(err) - } - - mkproxyfsCmd = exec.Command("mkproxyfs", mkproxyfsArgs...) - - err = mkproxyfsCmd.Run() - if nil != err { - log.Fatalf("mkproxyfsCmd.Run() failed: %v", err) - } - - log.Printf("Call to mkproxyfsCmd.Run() succeeded") - - proxyfsdCmdWaitChan = make(chan error, 1) - - cleanDirectoryUnderFUSEMountPointName() - - launchProxyFSRunSCRUBAndFSCK() - - log.Printf("Initial call to launchProxyFSRunSCRUBAndFSCK() succeeded") - - if includeHalterTriggers { - httpStatusCode, _, contentsAsStrings, err = queryProxyFS(queryMethodGET, "/trigger", "") - if nil != err { - log.Printf("queryProxyFS() failed: %v", err) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if http.StatusOK != httpStatusCode { - log.Printf("queryProxyFS() returned unexpected httpStatusCode: %v", httpStatusCode) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - haltLabelStrings = make([]string, 0) - - for _, contentString := range contentsAsStrings { - haltLabelStringSplit = strings.Split(contentString, " ") - if 0 == len(haltLabelStringSplit) { - log.Printf("queryProxyFS() returned unexpected contentString: %v", contentString) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - haltLabelString = haltLabelStringSplit[0] - if "" == haltLabelString { - log.Printf("queryProxyFS() returned unexpected empty contentString") - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - if !strings.HasPrefix(haltLabelString, "halter.") { - haltLabelStrings = append(haltLabelStrings, haltLabelString) - } - } - - if 0 == len(haltLabelStrings) { - log.Printf("No halter.Arm() calls found - disabling") - includeHalterTriggers = false - } else { - log.Printf("Will arm haltLabelStrings:") - for _, haltLabelString = range haltLabelStrings { - log.Printf(" %v", haltLabelString) - } - } - } else { - log.Printf("No halter.Arm() calls scheduled") - } - - signalChan = make(chan os.Signal, 1) - - signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM) - - timeoutChan = make(chan bool, 1) - - trafficCmdWaitChan = make(chan error, 1) - - // Loop through causing ProxyFS to halt via: - // SIGINT - // SIGTERM - // SIGKILL - // halter.Trigger() on each of haltLabelStrings - // until SIGINT or SIGTERM - - signalToSend = unix.SIGINT - - for { - if nil == signalToSend { - randomHaltAfterCount = proxyfsdRandomHaltAfterCount() - log.Printf("Arming trigger %v with haltAfterCount == %v", haltLabelStrings[nextHalterTriggerIndex], randomHaltAfterCount) - httpStatusCode, _, _, err = queryProxyFS(queryMethodPOST, "/trigger/"+haltLabelStrings[nextHalterTriggerIndex]+"?count="+strconv.FormatUint(randomHaltAfterCount, 10), "") - if nil != err { - log.Printf("queryProxyFS() failed: %v", err) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if http.StatusNoContent != httpStatusCode { - log.Printf("queryProxyFS() returned unexpected httpStatusCode: %v", httpStatusCode) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - } else { - randomKillDelay = proxyfsdRandomKillDelay() - log.Printf("Will fire %v after %v", signalExpandedStringMap[signalToSend.String()], randomKillDelay) - go timeoutWaiter(randomKillDelay) - } - - launchTrafficScript() - - select { - case _ = <-signalChan: - log.Printf("Received SIGINT or SIGTERM... cleanly shutting down ProxyFS") - stopTrafficScript() - stopProxyFS(unix.SIGTERM) - os.Exit(0) - case _ = <-timeoutChan: - log.Printf("Sending %v to ProxyFS", signalExpandedStringMap[signalToSend.String()]) - stopProxyFS(signalToSend) - stopTrafficScript() - case err = <-proxyfsdCmdWaitChan: - log.Printf("ProxyFS has halted due to trigger or other failure") - stopTrafficScript() - case err = <-trafficCmdWaitChan: - log.Printf("trafficScript unexpectedly finished/failed: %v", err) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - cleanDirectoryUnderFUSEMountPointName() - - launchProxyFSRunSCRUBAndFSCK() - - switch signalToSend { - case unix.SIGINT: - signalToSend = unix.SIGTERM - case unix.SIGTERM: - signalToSend = unix.SIGKILL - case unix.SIGKILL: - if includeHalterTriggers { - signalToSend = nil - nextHalterTriggerIndex = 0 - } else { - signalToSend = unix.SIGINT - } - case nil: - nextHalterTriggerIndex++ - if len(haltLabelStrings) == nextHalterTriggerIndex { - signalToSend = unix.SIGINT - } - default: - log.Printf("Logic error... unexpected signalToSend: %v", signalToSend) - stopTrafficScript() - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - } -} - -func timeoutWaiter(randomKillDelay time.Duration) { - time.Sleep(randomKillDelay) - timeoutChan <- true -} - -func trafficCmdWaiter() { - trafficCmdWaitChan <- trafficCmd.Wait() -} - -func stopTrafficScript() { - var ( - err error - pid int - pidSliceLast []int - pidSliceNow []int - umountCmd *exec.Cmd - ) - - if "" != trafficScript { - pidSliceNow = pstree(trafficCmd.Process.Pid) - - // Send SIGSTOP to all pids in pidSliceNow to prevent more trafficScript processes from being created - - for { - for _, pid = range pidSliceNow { - err = unix.Kill(pid, unix.SIGSTOP) - if nil != err { - log.Printf("INFO: unix.Kill(%v, unix.SIGSTOP) failed: %v", pid, err) - } - } - pidSliceLast = pidSliceNow - pidSliceNow = pstree(trafficCmd.Process.Pid) - if pidSliceEqual(pidSliceLast, pidSliceNow) { - break - } - } - - // Send SIGKILL to all pids in pidSliceNow to actually kill all trafficScript processes - - for { - for _, pid = range pidSliceNow { - err = unix.Kill(pid, unix.SIGKILL) - if nil != err { - log.Printf("INFO: unix.Kill(%v, unix.SIGKILL) failed: %v", pid, err) - } - } - pidSliceLast = pidSliceNow - pidSliceNow = pstree(trafficCmd.Process.Pid) - if pidSliceEqual(pidSliceLast, pidSliceNow) { - break - } - } - - // Force an unmount of fewMountPointName incase any processes of trafficScript are hung on if - - umountCmd = exec.Command("fusermount", "-u", fuseMountPointName) - - err = umountCmd.Run() - if nil != err { - log.Printf("INFO: umountCmd.Run() failed: %v", err) - } - - // Finally, await indicated exit of trafficScript - - _ = <-trafficCmdWaitChan - } -} - -func launchTrafficScript() { - var ( - err error - ) - - if "" != trafficScript { - log.Printf("Launching trafficScript: bash %v %v", trafficScript, fuseMountPointName) - - trafficCmd = exec.Command("bash", trafficScript, fuseMountPointName) - - err = trafficCmd.Start() - if nil != err { - log.Fatalf("trafficCmd.Start() failed: %v", err) - } - - go trafficCmdWaiter() - } -} - -func proxyfsdCmdWaiter() { - proxyfsdCmdWaitChan <- proxyfsdCmd.Wait() -} - -func stopProxyFS(signalToSend os.Signal) { - var ( - err error - ) - - err = proxyfsdCmd.Process.Signal(signalToSend) - if nil != err { - log.Fatalf("proxyfsdCmd.Process.Signal(signalToSend) failed: %v", err) - } - _ = <-proxyfsdCmdWaitChan -} - -func launchProxyFSRunSCRUBAndFSCK() { - var ( - contentsAsStrings []string - err error - fsckJob httpserver.JobStatusJSONPackedStruct - fsckJobError string - httpStatusCode int - locationURL string - polling bool - scrubJob httpserver.JobStatusJSONPackedStruct - scrubJobError string - ) - - log.Printf("Launching ProxyFS and performing SCRUB & FSCK of %v", volumeName) - - proxyfsdCmd = exec.Command("proxyfsd", proxyfsdArgs...) - - err = proxyfsdCmd.Start() - if nil != err { - log.Fatalf("proxyfsdCmd.Start() failed: %v", err) - } - - go proxyfsdCmdWaiter() - - polling = true - for polling { - time.Sleep(proxyfsdPollDelay) - - httpStatusCode, locationURL, _, err = queryProxyFS(queryMethodPOST, "/volume/"+volumeName+"/scrub-job", "") - if nil == err { - polling = false - } - } - - if http.StatusCreated != httpStatusCode { - log.Printf("queryProxyFS(queryMethodPOST,\"/volume/%v/scrub-job\",) returned unexpected httpStatusCode: %v", volumeName, httpStatusCode) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - polling = true - for polling { - time.Sleep(proxyfsdPollDelay) - - httpStatusCode, _, contentsAsStrings, err = queryProxyFS(queryMethodGET, locationURL+"?compact=true", "application/json") - if nil != err { - log.Printf("queryProxyFS(queryMethodGET,\"%v?compact=true\",) failed: %v", locationURL, err) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if http.StatusOK != httpStatusCode { - log.Printf("queryProxyFS(queryMethodGET,\"%v?compact=true\",) returned unexpected httpStatusCode: %v", locationURL, httpStatusCode) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if 1 != len(contentsAsStrings) { - log.Printf("queryProxyFS(queryMethodGET,\"%v?compact=true\",) returned unexpected len(contentsAsStrings): %v", locationURL, len(contentsAsStrings)) - } - - err = json.Unmarshal([]byte(contentsAsStrings[0]), &scrubJob) - if nil != err { - log.Printf("queryProxyFS(queryMethodGET,\"%v?compact=true\",) returned undecodable content: %v (err == %v)", locationURL, contentsAsStrings[0], err) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if "" == scrubJob.StartTime { - log.Printf("scrubJob unexpectantly missing StartTime value") - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if "" != scrubJob.HaltTime { - log.Printf("scrubJob contained unexpected HaltTime value: %v", scrubJob.HaltTime) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - if "" != scrubJob.DoneTime { - polling = false - } - } - - if 0 < len(scrubJob.ErrorList) { - if 1 == len(scrubJob.ErrorList) { - log.Printf("scrubJob contained unexpected error: %v", scrubJob.ErrorList[0]) - } else { - log.Printf("scrubJob contained unexpected errors:") - for _, scrubJobError = range scrubJob.ErrorList { - log.Printf(" %v", scrubJobError) - } - } - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - httpStatusCode, locationURL, _, err = queryProxyFS(queryMethodPOST, "/volume/"+volumeName+"/fsck-job", "") - if nil != err { - log.Printf("queryProxyFS(queryMethodPOST,\"/volume/%v/fsck-job\",) failed: %v", volumeName, err) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if http.StatusCreated != httpStatusCode { - log.Printf("queryProxyFS(queryMethodPOST,\"/volume/%v/fsck-job\",) returned unexpected httpStatusCode: %v", volumeName, httpStatusCode) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - polling = true - for polling { - time.Sleep(proxyfsdPollDelay) - - httpStatusCode, _, contentsAsStrings, err = queryProxyFS(queryMethodGET, locationURL+"?compact=true", "application/json") - if nil != err { - log.Printf("queryProxyFS(queryMethodGET,\"%v?compact=true\",) failed: %v", locationURL, err) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if http.StatusOK != httpStatusCode { - log.Printf("queryProxyFS(queryMethodGET,\"%v?compact=true\",) returned unexpected httpStatusCode: %v", locationURL, httpStatusCode) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if 1 != len(contentsAsStrings) { - log.Printf("queryProxyFS(queryMethodGET,\"%v?compact=true\",) returned unexpected len(contentsAsStrings): %v", locationURL, len(contentsAsStrings)) - } - - err = json.Unmarshal([]byte(contentsAsStrings[0]), &fsckJob) - if nil != err { - log.Printf("queryProxyFS(queryMethodGET,\"%v?compact=true\",) returned undecodable content: %v (err == %v)", locationURL, contentsAsStrings[0], err) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if "" == fsckJob.StartTime { - log.Printf("fsckJob unexpectantly missing StartTime value") - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - if "" != fsckJob.HaltTime { - log.Printf("fsckJob contained unexpected HaltTime value: %v", fsckJob.HaltTime) - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - if "" != fsckJob.DoneTime { - polling = false - } - } - - if 0 < len(fsckJob.ErrorList) { - if 1 == len(fsckJob.ErrorList) { - log.Printf("fsckJob contained unexpected error: %v", fsckJob.ErrorList[0]) - } else { - log.Printf("fsckJob contained unexpected errors:") - for _, fsckJobError = range fsckJob.ErrorList { - log.Printf(" %v", fsckJobError) - } - } - stopProxyFS(unix.SIGTERM) - os.Exit(-1) - } - - log.Printf("ProxyFS launched; SCRUB & FSCK of %v reported no errors", volumeName) -} - -func queryProxyFS(queryMethod queryMethodType, queryURL string, acceptHeader string) (httpStatusCode int, locationURL string, contentsAsStrings []string, err error) { - var ( - client *http.Client - contentsAsByteSlice []byte - queryURLWithHost string - request *http.Request - response *http.Response - ) - - queryURLWithHost = "http://" + ipAddrTCPPort + queryURL - - switch queryMethod { - case queryMethodGET: - request, err = http.NewRequest("GET", queryURLWithHost, nil) - case queryMethodPOST: - request, err = http.NewRequest("POST", queryURLWithHost, nil) - default: - log.Fatalf("queryProxyFS(queryMethod==%v,,) invalid", queryMethod) - } - - if "" != acceptHeader { - request.Header.Add("Accept", acceptHeader) - } - - client = &http.Client{} - - response, err = client.Do(request) - if nil != err { - return - } - - defer response.Body.Close() - - httpStatusCode = response.StatusCode - - locationURL = response.Header.Get("Location") - - contentsAsByteSlice, err = ioutil.ReadAll(response.Body) - if nil != err { - return - } - - contentsAsStrings = strings.Split(string(contentsAsByteSlice), "\n") - if "" == contentsAsStrings[len(contentsAsStrings)-1] { - contentsAsStrings = contentsAsStrings[:len(contentsAsStrings)-1] - } - - return -} - -func proxyfsdRandomHaltAfterCount() (haltAfterCount uint64) { - var ( - bigN *big.Int - bigR *big.Int - err error - ) - - if pseudoRandom { - if nil == mathRandSource { - mathRandSource = mathRand.New(mathRand.NewSource(pseudoRandomSeed)) - } - haltAfterCount = uint64(mathRandSource.Int63n(int64(proxyfsdHalterMaxHaltAfterCount-proxyfsdHalterMinHaltAfterCount)+1)) + proxyfsdHalterMinHaltAfterCount - } else { - bigN = big.NewInt(int64(proxyfsdHalterMaxHaltAfterCount-proxyfsdHalterMinHaltAfterCount) + 1) - bigR, err = cryptoRand.Int(cryptoRand.Reader, bigN) - if nil != err { - log.Fatalf("cryptoRand.Int(cryptoRand.Reader, bigN) failed: %v", err) - } - haltAfterCount = bigR.Uint64() + proxyfsdHalterMinHaltAfterCount - } - - return -} - -func proxyfsdRandomKillDelay() (killDelay time.Duration) { - var ( - bigN *big.Int - bigR *big.Int - err error - ) - - if pseudoRandom { - if nil == mathRandSource { - mathRandSource = mathRand.New(mathRand.NewSource(pseudoRandomSeed)) - } - killDelay = time.Duration(mathRandSource.Int63n(int64(proxyfsdMaxKillDelay-proxyfsdMinKillDelay)+1)) + proxyfsdMinKillDelay - } else { - bigN = big.NewInt(int64(proxyfsdMaxKillDelay-proxyfsdMinKillDelay) + 1) - bigR, err = cryptoRand.Int(cryptoRand.Reader, bigN) - if nil != err { - log.Fatalf("cryptoRand.Int(cryptoRand.Reader, bigN) failed: %v", err) - } - killDelay = time.Duration(bigR.Uint64()) + proxyfsdMinKillDelay - } - - return -} - -func pstree(topPid int) (pidSlice []int) { - var ( - err error - ok bool - pid int - pidChildren []int - pidChildrenMap map[int][]int - pidList *list.List - pidOnList *list.Element - ppid int - psOutputByteSlice []byte - psOutputBufioReader *bufio.Reader - psOutputBytesReader *bytes.Reader - psOutputLine string - sscanfN int - ) - - psOutputByteSlice, err = exec.Command("ps", "-e", "-o", "pid,ppid").Output() - if nil != err { - log.Fatalf("exec.Command(\"ps\", \"-e\", \"-o\", \"pid,ppid\").Output failed: %v", err) - } - - psOutputBytesReader = bytes.NewReader(psOutputByteSlice) - psOutputBufioReader = bufio.NewReader(psOutputBytesReader) - - pidChildrenMap = make(map[int][]int) - - for { - psOutputLine, err = psOutputBufioReader.ReadString(byte(0x0A)) - if nil != err { - break - } - sscanfN, err = fmt.Sscanf(psOutputLine, "%d %d\n", &pid, &ppid) - if nil != err { - continue - } - if 2 != sscanfN { - continue - } - - pidChildren, ok = pidChildrenMap[ppid] - if !ok { - pidChildren = make([]int, 0, 1) - } - pidChildren = append(pidChildren, pid) - pidChildrenMap[ppid] = pidChildren - } - - pidList = list.New() - - pstreeStep(topPid, pidChildrenMap, pidList) - - pidSlice = make([]int, 0, pidList.Len()) - - for pidOnList = pidList.Front(); nil != pidOnList; pidOnList = pidOnList.Next() { - pid, ok = pidOnList.Value.(int) - if !ok { - log.Fatalf("pidOnList.Value.(int) failed") - } - pidSlice = append(pidSlice, pid) - } - - return -} - -func pstreeStep(ppid int, pidChildrenMap map[int][]int, pidList *list.List) { - var ( - ok bool - pid int - pidChildren []int - ) - - _ = pidList.PushBack(ppid) - - pidChildren, ok = pidChildrenMap[ppid] - if !ok { - return - } - - for _, pid = range pidChildren { - pstreeStep(pid, pidChildrenMap, pidList) - } -} - -func pidSliceEqual(pidSlice1 []int, pidSlice2 []int) (equal bool) { - var ( - ok bool - pid int - pidSlice1Map map[int]struct{} // Go's version of a "set" - ) - - if len(pidSlice1) != len(pidSlice2) { - equal = false - return - } - - pidSlice1Map = make(map[int]struct{}) - - for _, pid = range pidSlice1 { - pidSlice1Map[pid] = struct{}{} - } - - for _, pid = range pidSlice2 { - _, ok = pidSlice1Map[pid] - if !ok { - equal = false - return - } - } - - equal = true - return -} - -func cleanDirectoryUnderFUSEMountPointName() { - var ( - dir *os.File - err error - name string - nameJoined string - names []string - umountCmd *exec.Cmd - ) - - for { - time.Sleep(openDirPollDelay) - - dir, err = os.Open(fuseMountPointName) - if nil == err { - break - } - - log.Printf("Retrying os.Open(\"%v\") after \"fusermount -u\" due to err == %v", fuseMountPointName, err) - - umountCmd = exec.Command("fusermount", "-u", fuseMountPointName) - - err = umountCmd.Run() - if nil != err { - log.Printf("umountCmd.Run() failed: %v", err) - } - } - - names, err = dir.Readdirnames(-1) - if nil != err { - _ = dir.Close() - log.Fatalf("dir.Readdirnames(-1) failed: %v", err) - } - - err = dir.Close() - if nil != err { - log.Fatalf("dir.Close() failed: %v", err) - } - - for _, name = range names { - nameJoined = filepath.Join(fuseMountPointName, name) - err = os.RemoveAll(nameJoined) - if nil != err { - log.Fatalf("os.RemoveAll(%v) failed: %v", nameJoined, err) - } - } -} diff --git a/pfs-crash/touch.sh b/pfs-crash/touch.sh deleted file mode 100755 index c278f417..00000000 --- a/pfs-crash/touch.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -while true -do - for i in `seq 100` - do - touch $1/touch_$i - done -done diff --git a/pfs-fsck/Makefile b/pfs-fsck/Makefile deleted file mode 100644 index e8e6dce1..00000000 --- a/pfs-fsck/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfs-fsck - -include ../GoMakefile diff --git a/pfs-fsck/dummy_test.go b/pfs-fsck/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfs-fsck/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfs-fsck/main.go b/pfs-fsck/main.go deleted file mode 100644 index e903cf5f..00000000 --- a/pfs-fsck/main.go +++ /dev/null @@ -1,1091 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net" - "net/http" - "os" - "strconv" - "strings" - "time" - - etcd "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" - - "github.com/NVIDIA/cstruct" - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/etcdclient" - "github.com/NVIDIA/proxyfs/headhunter" -) - -const ( - bPlusTreeCacheEvictHighLimitDefault = uint64(10010) - bPlusTreeCacheEvictLowLimitDefault = uint64(10000) -) - -type globalsStruct struct { - accountName string - alsoDump bool - bPlusTreeCache sortedmap.BPlusTreeCache - bPlusTreeCacheEvictHighLimit uint64 // [FSCK]BPlusTreeCacheEvictHighLimit - bPlusTreeCacheEvictLowLimit uint64 // [FSCK]BPlusTreeCacheEvictLowLimit - bPlusTreeObjectBPlusTreeLayout sortedmap.LayoutReport - checkpointEtcdKeyName string - checkpointContainerName string - checkpointObjectTrailerV3 *headhunter.CheckpointObjectTrailerV3Struct - checkpointObjectTrailerV3Buf []byte - checkpointObjectTrailerV3BufPos uint64 - createdObjectsBPlusTreeLayout sortedmap.LayoutReport - deletedObjectsBPlusTreeLayout sortedmap.LayoutReport - elementOfBPlusTreeLayoutStructSize uint64 - etcdAutoSyncInterval time.Duration - etcdClient *etcd.Client - etcdDialTimeout time.Duration - etcdEnabled bool - etcdCertDir string - etcdEndpoints []string - etcdKV etcd.KV - etcdOpTimeout time.Duration - initialCheckpointHeader *headhunter.CheckpointHeaderStruct - inodeRecBPlusTreeLayout sortedmap.LayoutReport - logSegmentBPlusTreeLayout sortedmap.LayoutReport - noAuthIPAddr string - noAuthTCPPort uint16 - noAuthURL string - recycleBinRefMap map[string]uint64 // Key: object path ; Value: # references - snapShotList []*headhunter.ElementOfSnapShotListStruct - uint64Size uint64 - volumeName string -} - -var globals globalsStruct - -func main() { - var ( - bPlusTree sortedmap.BPlusTree - elementOfSnapShotList *headhunter.ElementOfSnapShotListStruct - err error - recycleBinObjectName string - recycleBinObjectReferences uint64 - ) - - setup() - - globals.initialCheckpointHeader = fetchCheckpointHeader() - - globals.recycleBinRefMap = make(map[string]uint64) - - err = fetchCheckpointObjectTrailerV3() - if nil != err { - log.Fatal(err) - } - - globals.inodeRecBPlusTreeLayout, err = fetchBPlusTreeLayout(globals.checkpointObjectTrailerV3.InodeRecBPlusTreeLayoutNumElements) - if nil != err { - log.Fatalf("fetching inodeRecBPlusTreeLayout failed: %v", err) - } - globals.logSegmentBPlusTreeLayout, err = fetchBPlusTreeLayout(globals.checkpointObjectTrailerV3.LogSegmentRecBPlusTreeLayoutNumElements) - if nil != err { - log.Fatalf("fetching logSegmentBPlusTreeLayout failed: %v", err) - } - globals.bPlusTreeObjectBPlusTreeLayout, err = fetchBPlusTreeLayout(globals.checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeLayoutNumElements) - if nil != err { - log.Fatalf("fetching bPlusTreeObjectBPlusTreeLayout failed: %v", err) - } - globals.createdObjectsBPlusTreeLayout, err = fetchBPlusTreeLayout(globals.checkpointObjectTrailerV3.CreatedObjectsBPlusTreeLayoutNumElements) - if nil != err { - log.Fatalf("fetching createdObjectsBPlusTreeLayout failed: %v", err) - } - globals.deletedObjectsBPlusTreeLayout, err = fetchBPlusTreeLayout(globals.checkpointObjectTrailerV3.DeletedObjectsBPlusTreeLayoutNumElements) - if nil != err { - log.Fatalf("fetching deletedObjectsBPlusTreeLayout failed: %v", err) - } - - err = fetchSnapShotList(globals.checkpointObjectTrailerV3.SnapShotListNumElements) - if nil != err { - log.Fatalf("fetching snapShotList failed: %v", err) - } - - if globals.checkpointObjectTrailerV3BufPos < globals.initialCheckpointHeader.CheckpointObjectTrailerStructObjectLength { - log.Fatalf("unmarshalling checkpointObjectTrailerV3Buf not fully consumed") - } - - globals.bPlusTreeCache = sortedmap.NewBPlusTreeCache(globals.bPlusTreeCacheEvictLowLimit, globals.bPlusTreeCacheEvictHighLimit) - - if uint64(0) != globals.checkpointObjectTrailerV3.InodeRecBPlusTreeObjectNumber { - bPlusTree, err = sortedmap.OldBPlusTree(globals.checkpointObjectTrailerV3.InodeRecBPlusTreeObjectNumber, - globals.checkpointObjectTrailerV3.InodeRecBPlusTreeObjectOffset, - globals.checkpointObjectTrailerV3.InodeRecBPlusTreeObjectLength, - sortedmap.CompareUint64, - &globals, - globals.bPlusTreeCache) - if nil != err { - log.Fatalf("loading of InodeRecBPlusTree failed: %v", err) - } - - err = bPlusTree.Validate() - if nil != err { - log.Fatalf("validate of InodeRecBPlusTree failed: %v", err) - } - - dumpKeysIfRequested(bPlusTree, "InodeRecBPlusTree") - } - - if uint64(0) != globals.checkpointObjectTrailerV3.LogSegmentRecBPlusTreeObjectNumber { - bPlusTree, err = sortedmap.OldBPlusTree(globals.checkpointObjectTrailerV3.LogSegmentRecBPlusTreeObjectNumber, - globals.checkpointObjectTrailerV3.LogSegmentRecBPlusTreeObjectOffset, - globals.checkpointObjectTrailerV3.LogSegmentRecBPlusTreeObjectLength, - sortedmap.CompareUint64, - &globals, - globals.bPlusTreeCache) - if nil != err { - log.Fatalf("loading of LogSegmentRecBPlusTree failed: %v", err) - } - - err = bPlusTree.Validate() - if nil != err { - log.Fatalf("validate of LogSegmentRecBPlusTree failed: %v", err) - } - - dumpKeysIfRequested(bPlusTree, "LogSegmentRecBPlusTree") - } - - if uint64(0) != globals.checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeObjectNumber { - bPlusTree, err = sortedmap.OldBPlusTree(globals.checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeObjectNumber, - globals.checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeObjectOffset, - globals.checkpointObjectTrailerV3.BPlusTreeObjectBPlusTreeObjectLength, - sortedmap.CompareUint64, - &globals, - globals.bPlusTreeCache) - if nil != err { - log.Fatalf("loading of BPlusTreeObjectBPlusTree failed: %v", err) - } - - err = bPlusTree.Validate() - if nil != err { - log.Fatalf("validate of BPlusTreeObjectBPlusTree failed: %v", err) - } - - dumpKeysIfRequested(bPlusTree, "BPlusTreeObjectBPlusTree") - } - - for _, elementOfSnapShotList = range globals.snapShotList { - if uint64(0) != elementOfSnapShotList.InodeRecBPlusTreeObjectNumber { - bPlusTree, err = sortedmap.OldBPlusTree(elementOfSnapShotList.InodeRecBPlusTreeObjectNumber, - elementOfSnapShotList.InodeRecBPlusTreeObjectOffset, - elementOfSnapShotList.InodeRecBPlusTreeObjectLength, - sortedmap.CompareUint64, - &globals, - globals.bPlusTreeCache) - if nil != err { - log.Fatalf("loading of SnapShot.ID==%d InodeRecBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - err = bPlusTree.Validate() - if nil != err { - log.Fatalf("validate of SnapShot.ID==%d InodeRecBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - dumpKeysIfRequested(bPlusTree, fmt.Sprintf("InodeRecBPlusTree for SnapShotID==%d", elementOfSnapShotList.ID)) - } - - if uint64(0) != elementOfSnapShotList.LogSegmentRecBPlusTreeObjectNumber { - bPlusTree, err = sortedmap.OldBPlusTree(elementOfSnapShotList.LogSegmentRecBPlusTreeObjectNumber, - elementOfSnapShotList.LogSegmentRecBPlusTreeObjectOffset, - elementOfSnapShotList.LogSegmentRecBPlusTreeObjectLength, - sortedmap.CompareUint64, - &globals, - globals.bPlusTreeCache) - if nil != err { - log.Fatalf("loading of SnapShot.ID==%d LogSegmentRecBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - err = bPlusTree.Validate() - if nil != err { - log.Fatalf("validate of SnapShot.ID==%d LogSegmentRecBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - dumpKeysIfRequested(bPlusTree, fmt.Sprintf("LogSegmentRecBPlusTree for SnapShotID==%d", elementOfSnapShotList.ID)) - } - - if uint64(0) != elementOfSnapShotList.BPlusTreeObjectBPlusTreeObjectNumber { - bPlusTree, err = sortedmap.OldBPlusTree(elementOfSnapShotList.BPlusTreeObjectBPlusTreeObjectNumber, - elementOfSnapShotList.BPlusTreeObjectBPlusTreeObjectOffset, - elementOfSnapShotList.BPlusTreeObjectBPlusTreeObjectLength, - sortedmap.CompareUint64, - &globals, - globals.bPlusTreeCache) - if nil != err { - log.Fatalf("loading of SnapShot.ID==%d BPlusTreeObjectBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - err = bPlusTree.Validate() - if nil != err { - log.Fatalf("validate of SnapShot.ID==%d BPlusTreeObjectBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - dumpKeysIfRequested(bPlusTree, fmt.Sprintf("BPlusTreeObjectBPlusTree for SnapShotID==%d", elementOfSnapShotList.ID)) - } - - if uint64(0) != elementOfSnapShotList.CreatedObjectsBPlusTreeObjectNumber { - bPlusTree, err = sortedmap.OldBPlusTree(elementOfSnapShotList.CreatedObjectsBPlusTreeObjectNumber, - elementOfSnapShotList.CreatedObjectsBPlusTreeObjectOffset, - elementOfSnapShotList.CreatedObjectsBPlusTreeObjectLength, - sortedmap.CompareUint64, - &globals, - globals.bPlusTreeCache) - if nil != err { - log.Fatalf("loading of SnapShot.ID==%d CreatedObjectsBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - err = bPlusTree.Validate() - if nil != err { - log.Fatalf("validate of SnapShot.ID==%d CreatedObjectsBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - dumpKeysIfRequested(bPlusTree, fmt.Sprintf("CreatedObjectsBPlusTree for SnapShotID==%d", elementOfSnapShotList.ID)) - } - - if uint64(0) != elementOfSnapShotList.DeletedObjectsBPlusTreeObjectNumber { - bPlusTree, err = sortedmap.OldBPlusTree(elementOfSnapShotList.DeletedObjectsBPlusTreeObjectNumber, - elementOfSnapShotList.DeletedObjectsBPlusTreeObjectOffset, - elementOfSnapShotList.DeletedObjectsBPlusTreeObjectLength, - sortedmap.CompareUint64, - &globals, - globals.bPlusTreeCache) - if nil != err { - log.Fatalf("loading of SnapShot.ID==%d DeletedObjectsBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - err = bPlusTree.Validate() - if nil != err { - log.Fatalf("validate of SnapShot.ID==%d DeletedObjectsBPlusTree failed: %v", elementOfSnapShotList.ID, err) - } - - dumpKeysIfRequested(bPlusTree, fmt.Sprintf("DeletedObjectsBPlusTree for SnapShotID==%d", elementOfSnapShotList.ID)) - } - } - - // TODO: Validate InodeRec, LogSegmentRec, BPlusTreeObject, CreatedObjects, & DeletedObjects B+Tree Layouts - - if 0 < len(globals.recycleBinRefMap) { - for recycleBinObjectName, recycleBinObjectReferences = range globals.recycleBinRefMap { - log.Printf("Recycled Object %s referenced %d times", recycleBinObjectName, recycleBinObjectReferences) - } - os.Exit(1) - } -} - -func usage() { - fmt.Printf("%v [-D] []*\n", os.Args[0]) -} - -func setup() { - var ( - confFile string - confMap conf.ConfMap - confOverrides []string - dummyElementOfBPlusTreeLayout headhunter.ElementOfBPlusTreeLayoutStruct - dummyUint64 uint64 - err error - volumeSectionName string - ) - - if 3 > len(os.Args) { - usage() - os.Exit(1) - } - - if "-D" == os.Args[1] { - globals.alsoDump = true - - if 4 > len(os.Args) { - usage() - os.Exit(1) - } - - globals.volumeName = os.Args[2] - confFile = os.Args[3] - confOverrides = os.Args[4:] - } else { - globals.alsoDump = false - - globals.volumeName = os.Args[1] - confFile = os.Args[2] - confOverrides = os.Args[3:] - } - - volumeSectionName = "Volume:" + globals.volumeName - - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - log.Fatalf("confFile (\"%s\") not parseable: %v", confFile, err) - } - - err = confMap.UpdateFromStrings(confOverrides) - if nil != err { - log.Fatalf("confOverrides (%v) not parseable: %v", confOverrides, err) - } - - globals.noAuthIPAddr, err = confMap.FetchOptionValueString("SwiftClient", "NoAuthIPAddr") - if nil != err { - globals.noAuthIPAddr = "127.0.0.1" // TODO: Eventually just return - } - globals.noAuthTCPPort, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") - if nil != err { - log.Fatal(err) - } - - globals.noAuthURL = "http://" + net.JoinHostPort(globals.noAuthIPAddr, fmt.Sprintf("%d", globals.noAuthTCPPort)) + "/" - - globals.accountName, err = confMap.FetchOptionValueString(volumeSectionName, "AccountName") - if nil != err { - log.Fatal(err) - } - globals.checkpointContainerName, err = confMap.FetchOptionValueString(volumeSectionName, "CheckpointContainerName") - if nil != err { - log.Fatal(err) - } - - globals.etcdEnabled, err = confMap.FetchOptionValueBool("FSGlobals", "EtcdEnabled") - if nil != err { - globals.etcdEnabled = false // TODO: Current default... perhaps eventually just log.Fatal(err) - } - - if globals.etcdEnabled { - globals.etcdAutoSyncInterval, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdAutoSyncInterval") - if nil != err { - log.Fatal(err) - } - globals.etcdCertDir, err = confMap.FetchOptionValueString("FSGlobals", "EtcdCertDir") - if nil != err { - return - } - globals.etcdDialTimeout, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdDialTimeout") - if nil != err { - log.Fatal(err) - } - globals.etcdEndpoints, err = confMap.FetchOptionValueStringSlice("FSGlobals", "EtcdEndpoints") - if nil != err { - log.Fatal(err) - } - globals.etcdOpTimeout, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdOpTimeout") - if nil != err { - log.Fatal(err) - } - - tlsInfo := transport.TLSInfo{ - CertFile: etcdclient.GetCertFilePath(globals.etcdCertDir), - KeyFile: etcdclient.GetKeyFilePath(globals.etcdCertDir), - TrustedCAFile: etcdclient.GetCA(globals.etcdCertDir), - } - - globals.etcdClient, err = etcdclient.New(&tlsInfo, globals.etcdEndpoints, - globals.etcdAutoSyncInterval, globals.etcdDialTimeout) - if nil != err { - log.Fatalf("unable to create etcdClient: %v\n", err) - } - - globals.etcdKV = etcd.NewKV(globals.etcdClient) - - globals.checkpointEtcdKeyName, err = confMap.FetchOptionValueString(volumeSectionName, "CheckpointEtcdKeyName") - } - - globals.bPlusTreeCacheEvictLowLimit, err = confMap.FetchOptionValueUint64("FSCK", "BPlusTreeCacheEvictLowLimit") - if nil != err { - globals.bPlusTreeCacheEvictLowLimit = bPlusTreeCacheEvictLowLimitDefault - } - globals.bPlusTreeCacheEvictHighLimit, err = confMap.FetchOptionValueUint64("FSCK", "BPlusTreeCacheEvictHighLimit") - if nil != err { - globals.bPlusTreeCacheEvictHighLimit = bPlusTreeCacheEvictHighLimitDefault - } - - globals.elementOfBPlusTreeLayoutStructSize, _, err = cstruct.Examine(&dummyElementOfBPlusTreeLayout) - if nil != err { - log.Fatal(err) - } - globals.uint64Size, _, err = cstruct.Examine(&dummyUint64) - if nil != err { - log.Fatal(err) - } -} - -func fetchCheckpointHeader() (checkpointHeader *headhunter.CheckpointHeaderStruct) { - var ( - cancel context.CancelFunc - checkpointContainerHeaderMap http.Header - checkpointContainerHeaderString string - checkpointContainerHeaderStringSlice []string - checkpointContainerHeaderStringSplit []string - ctx context.Context - err error - etcdGetResponse *etcd.GetResponse - ok bool - ) - - checkpointHeader = &headhunter.CheckpointHeaderStruct{} - - if globals.etcdEnabled { - ctx, cancel = context.WithTimeout(context.Background(), globals.etcdOpTimeout) - etcdGetResponse, err = globals.etcdKV.Get(ctx, globals.checkpointEtcdKeyName) - cancel() - if nil != err { - log.Fatalf("error contacting etcd: %v", err) - } - - if 1 == etcdGetResponse.Count { - err = json.Unmarshal(etcdGetResponse.Kvs[0].Value, checkpointHeader) - if nil != err { - log.Fatalf("error unmarshalling %s's Value (%s): %v", globals.checkpointEtcdKeyName, string(etcdGetResponse.Kvs[0].Value[:]), err) - } - - if headhunter.CheckpointVersion3 != checkpointHeader.CheckpointVersion { - log.Fatalf("unsupported CheckpointVersion (%v)...must be == CheckpointVersion3 (%v)", checkpointHeader.CheckpointVersion, headhunter.CheckpointVersion3) - } - - return - } - } - - checkpointContainerHeaderMap, err = doHead(globals.accountName + "/" + globals.checkpointContainerName) - if nil != err { - log.Fatalf("error fetching checkpointContainerHeaderMap: %v", err) - } - - checkpointContainerHeaderStringSlice, ok = checkpointContainerHeaderMap[headhunter.CheckpointHeaderName] - if !ok { - log.Fatalf("error fetching checkpointContainerHeaderStringSlice") - } - if 1 != len(checkpointContainerHeaderStringSlice) { - log.Fatalf("checkpointContainerHeaderStringSlice must be single-valued") - } - - checkpointContainerHeaderString = checkpointContainerHeaderStringSlice[0] - - checkpointContainerHeaderStringSplit = strings.Split(checkpointContainerHeaderString, " ") - if 4 != len(checkpointContainerHeaderStringSplit) { - log.Fatalf("checkpointContainerHeaderStringSplit must be four-valued") - } - - checkpointHeader.CheckpointVersion, err = strconv.ParseUint(checkpointContainerHeaderStringSplit[0], 16, 64) - if nil != err { - log.Fatalf("error parsing CheckpointVersion: %v", err) - } - if headhunter.CheckpointVersion3 != checkpointHeader.CheckpointVersion { - log.Fatalf("unsupported CheckpointVersion (%v)...must be == CheckpointVersion3 (%v)", checkpointHeader.CheckpointVersion, headhunter.CheckpointVersion3) - } - - checkpointHeader.CheckpointObjectTrailerStructObjectNumber, err = strconv.ParseUint(checkpointContainerHeaderStringSplit[1], 16, 64) - if nil != err { - log.Fatalf("error parsing CheckpointObjectTrailerStructObjectNumber:%v", err) - } - - checkpointHeader.CheckpointObjectTrailerStructObjectLength, err = strconv.ParseUint(checkpointContainerHeaderStringSplit[2], 16, 64) - if nil != err { - log.Fatalf("error parsing CheckpointObjectTrailerStructObjectLength:%v", err) - } - - checkpointHeader.ReservedToNonce, err = strconv.ParseUint(checkpointContainerHeaderStringSplit[3], 16, 64) - if nil != err { - log.Fatalf("error parsing ReservedToNonce:%v", err) - } - - return -} - -func checkpointHeadersAreEqual(checkpointHeader1 *headhunter.CheckpointHeaderStruct, checkpointHeader2 *headhunter.CheckpointHeaderStruct) (areEqual bool) { - areEqual = (checkpointHeader1.CheckpointVersion == checkpointHeader2.CheckpointVersion) && - (checkpointHeader1.CheckpointObjectTrailerStructObjectNumber == checkpointHeader2.CheckpointObjectTrailerStructObjectNumber) && - (checkpointHeader1.CheckpointObjectTrailerStructObjectLength == checkpointHeader2.CheckpointObjectTrailerStructObjectLength) && - (checkpointHeader1.ReservedToNonce == checkpointHeader2.ReservedToNonce) - return // Note: Comparing CheckpointObjectTrailerStructObjectNumber's would actually have been sufficient -} - -func fetchCheckpointObjectTrailerV3() (err error) { - var ( - checkpointObjectHeaderMap http.Header - checkpointObjectPath string - inRecycleBinRefMap bool - refs uint64 - ) - - checkpointObjectPath = globals.accountName + "/" + globals.checkpointContainerName + "/" + fmt.Sprintf("%016X", globals.initialCheckpointHeader.CheckpointObjectTrailerStructObjectNumber) - - checkpointObjectHeaderMap, globals.checkpointObjectTrailerV3Buf, err = doGetTail(checkpointObjectPath, globals.initialCheckpointHeader.CheckpointObjectTrailerStructObjectLength) - if nil != err { - err = fmt.Errorf("error reading checkpointObjectTrailerV3Buf: %v", err) - return - } - - if metadataRecycleBinHeaderPresent(checkpointObjectHeaderMap) { - refs, inRecycleBinRefMap = globals.recycleBinRefMap[checkpointObjectPath] - if inRecycleBinRefMap { - refs++ - } else { - refs = 1 - } - globals.recycleBinRefMap[checkpointObjectPath] = refs - } - - globals.checkpointObjectTrailerV3 = &headhunter.CheckpointObjectTrailerV3Struct{} - - globals.checkpointObjectTrailerV3BufPos, err = cstruct.Unpack(globals.checkpointObjectTrailerV3Buf, globals.checkpointObjectTrailerV3, headhunter.LittleEndian) - if nil != err { - err = fmt.Errorf("unable to Unpack checkpointObjectTrailerV3Buf: %v", err) - return - } - - return -} - -func fetchBPlusTreeLayout(numElements uint64) (treeLayout sortedmap.LayoutReport, err error) { - var ( - alreadyInTreeLayout bool - bytesNeeded uint64 - elementIndex uint64 - elementOfBPlusTreeLayout headhunter.ElementOfBPlusTreeLayoutStruct - ) - - bytesNeeded = numElements * globals.elementOfBPlusTreeLayoutStructSize - if bytesNeeded > (globals.initialCheckpointHeader.CheckpointObjectTrailerStructObjectLength - globals.checkpointObjectTrailerV3BufPos) { - err = fmt.Errorf("insufficient bytes left in checkpointObjectTrailerV3Buf") - return - } - - treeLayout = make(sortedmap.LayoutReport) - - for elementIndex = 0; elementIndex < numElements; elementIndex++ { - _, err = cstruct.Unpack(globals.checkpointObjectTrailerV3Buf[globals.checkpointObjectTrailerV3BufPos:], &elementOfBPlusTreeLayout, headhunter.LittleEndian) - if nil != err { - return - } - - globals.checkpointObjectTrailerV3BufPos += globals.elementOfBPlusTreeLayoutStructSize - - _, alreadyInTreeLayout = treeLayout[elementOfBPlusTreeLayout.ObjectNumber] - if alreadyInTreeLayout { - err = fmt.Errorf("duplicate elementOfBPlusTreeLayout.ObjectNumber (0x%016X) encountered", elementOfBPlusTreeLayout.ObjectNumber) - return - } - - treeLayout[elementOfBPlusTreeLayout.ObjectNumber] = elementOfBPlusTreeLayout.ObjectNumber - } - - return -} - -func fetchSnapShotList(numElements uint64) (err error) { - var ( - alreadySeenThisID bool - alreadySeenThisIDSet map[uint64]struct{} - alreadySeenThisName bool - alreadySeenThisNameSet map[string]struct{} - alreadySeenThisNonce bool - alreadySeenThisNonceSet map[uint64]struct{} - elementIndex uint64 - elementOfSnapShotList *headhunter.ElementOfSnapShotListStruct - ) - - globals.snapShotList = make([]*headhunter.ElementOfSnapShotListStruct, numElements) - - alreadySeenThisNonceSet = make(map[uint64]struct{}) - alreadySeenThisIDSet = make(map[uint64]struct{}) - alreadySeenThisNameSet = make(map[string]struct{}) - - for elementIndex = 0; elementIndex < numElements; elementIndex++ { - elementOfSnapShotList = &headhunter.ElementOfSnapShotListStruct{} - - elementOfSnapShotList.Nonce, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.ID, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.TimeStamp, err = consumeCheckpointObjectTrailerV3BufTimeStamp() - if nil != err { - return - } - elementOfSnapShotList.Name, err = consumeCheckpointObjectTrailerV3BufString() - if nil != err { - return - } - elementOfSnapShotList.InodeRecBPlusTreeObjectNumber, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.InodeRecBPlusTreeObjectOffset, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.InodeRecBPlusTreeObjectLength, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.LogSegmentRecBPlusTreeObjectNumber, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.LogSegmentRecBPlusTreeObjectOffset, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.LogSegmentRecBPlusTreeObjectLength, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.BPlusTreeObjectBPlusTreeObjectNumber, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.BPlusTreeObjectBPlusTreeObjectOffset, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.BPlusTreeObjectBPlusTreeObjectLength, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.CreatedObjectsBPlusTreeObjectNumber, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.CreatedObjectsBPlusTreeObjectOffset, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.CreatedObjectsBPlusTreeObjectLength, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.DeletedObjectsBPlusTreeObjectNumber, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.DeletedObjectsBPlusTreeObjectOffset, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - elementOfSnapShotList.DeletedObjectsBPlusTreeObjectLength, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - - _, alreadySeenThisNonce = alreadySeenThisNonceSet[elementOfSnapShotList.Nonce] - if alreadySeenThisNonce { - err = fmt.Errorf("duplicate elementOfSnapShotList.Nonce found: 0x%16X", elementOfSnapShotList.Nonce) - return - } - alreadySeenThisNonceSet[elementOfSnapShotList.Nonce] = struct{}{} - - _, alreadySeenThisID = alreadySeenThisIDSet[elementOfSnapShotList.ID] - if alreadySeenThisID { - err = fmt.Errorf("duplicate elementOfSnapShotList.ID found: 0x%16X", elementOfSnapShotList.ID) - return - } - alreadySeenThisIDSet[elementOfSnapShotList.ID] = struct{}{} - - _, alreadySeenThisName = alreadySeenThisNameSet[elementOfSnapShotList.Name] - if alreadySeenThisName { - err = fmt.Errorf("duplicate elementOfSnapShotList.Name found: 0x%16X", elementOfSnapShotList.Name) - return - } - alreadySeenThisNameSet[elementOfSnapShotList.Name] = struct{}{} - - globals.snapShotList[elementIndex] = elementOfSnapShotList - } - - return -} - -func consumeCheckpointObjectTrailerV3BufUint64() (u64 uint64, err error) { - var ( - u64Copy uint64 - ) - - if globals.uint64Size > (globals.initialCheckpointHeader.CheckpointObjectTrailerStructObjectLength - globals.checkpointObjectTrailerV3BufPos) { - err = fmt.Errorf("insufficient bytes left in checkpointObjectTrailerV3Buf") - return - } - - _, err = cstruct.Unpack(globals.checkpointObjectTrailerV3Buf[globals.checkpointObjectTrailerV3BufPos:], &u64Copy, headhunter.LittleEndian) - if nil != err { - return - } - - globals.checkpointObjectTrailerV3BufPos += globals.uint64Size - - u64 = u64Copy - - return -} - -func consumeCheckpointObjectTrailerV3BufTimeStamp() (timeStamp time.Time, err error) { - var ( - timeStampBuf []byte - timeStampBufLen uint64 - ) - - timeStampBufLen, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - - if timeStampBufLen > (globals.initialCheckpointHeader.CheckpointObjectTrailerStructObjectLength - globals.checkpointObjectTrailerV3BufPos) { - err = fmt.Errorf("insufficient bytes left in checkpointObjectTrailerV3Buf") - return - } - - timeStampBuf = globals.checkpointObjectTrailerV3Buf[globals.checkpointObjectTrailerV3BufPos : globals.checkpointObjectTrailerV3BufPos+timeStampBufLen] - - err = timeStamp.UnmarshalBinary(timeStampBuf) - if nil != err { - return - } - - globals.checkpointObjectTrailerV3BufPos += timeStampBufLen - - return -} - -func consumeCheckpointObjectTrailerV3BufString() (str string, err error) { - var ( - strLen uint64 - ) - - strLen, err = consumeCheckpointObjectTrailerV3BufUint64() - if nil != err { - return - } - - if strLen > (globals.initialCheckpointHeader.CheckpointObjectTrailerStructObjectLength - globals.checkpointObjectTrailerV3BufPos) { - err = fmt.Errorf("insufficient bytes left in checkpointObjectTrailerV3Buf") - return - } - - str = string(globals.checkpointObjectTrailerV3Buf[globals.checkpointObjectTrailerV3BufPos : globals.checkpointObjectTrailerV3BufPos+strLen]) - - globals.checkpointObjectTrailerV3BufPos += strLen - - return -} - -func dumpKeysIfRequested(bPlusTree sortedmap.BPlusTree, bPlusTreeName string) { - var ( - err error - itemIndex int - keyAsKey sortedmap.Key - keyAsUint64 uint64 - numberOfItems int - ok bool - ) - - if globals.alsoDump { - fmt.Printf("%s:\n", bPlusTreeName) - - numberOfItems, err = bPlusTree.Len() - if nil != err { - log.Fatalf(" bPlusTree.Len() failed: %v", err) - } - - if 0 == numberOfItems { - fmt.Printf(" \n") - return - } - - for itemIndex = 0; itemIndex < numberOfItems; itemIndex++ { - keyAsKey, _, ok, err = bPlusTree.GetByIndex(itemIndex) - if nil != err { - log.Fatalf(" bPlusTree.GetByIndex(%d) failed: %v", itemIndex, err) - } - if !ok { - log.Fatalf(" bPlusTree.GetByIndex(%d) returned !ok", itemIndex) - } - - keyAsUint64, ok = keyAsKey.(uint64) - if !ok { - log.Fatalf(" bPlusTree.GetByIndex(%d) returned non-uint64 Key", itemIndex) - } - - fmt.Printf(" 0x%016X\n", keyAsUint64) - } - } -} - -func (dummy *globalsStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - err = fmt.Errorf("not implemented") - return -} - -func (dummy *globalsStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - err = fmt.Errorf("not implemented") - return -} - -func (dummy *globalsStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { - var ( - inRecycleBinRefMap bool - objectHeaderMap http.Header - objectPath string - refs uint64 - ) - - objectPath = globals.accountName + "/" + globals.checkpointContainerName + "/" + fmt.Sprintf("%016X", objectNumber) - - objectHeaderMap, nodeByteSlice, err = doGetRange(objectPath, objectOffset, objectLength) - if nil != err { - err = fmt.Errorf("error reading %s (offset=0x%016X,length=0x%016X): %v", objectPath, objectOffset, objectLength, err) - return - } - - if metadataRecycleBinHeaderPresent(objectHeaderMap) { - refs, inRecycleBinRefMap = globals.recycleBinRefMap[objectPath] - if inRecycleBinRefMap { - refs++ - } else { - refs = 1 - } - globals.recycleBinRefMap[objectPath] = refs - } - - return -} - -func (dummy *globalsStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { - err = fmt.Errorf("not implemented") - return -} - -func (dummy *globalsStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { - err = nil - return -} - -func (dummy *globalsStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { - err = fmt.Errorf("not implemented") - return -} - -func (dummy *globalsStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { - var ( - keyAsUint64 uint64 - ) - - _, err = cstruct.Unpack(payloadData, &keyAsUint64, headhunter.LittleEndian) - if nil != err { - return - } - - key = keyAsUint64 - bytesConsumed = globals.uint64Size - - return -} - -func (dummy *globalsStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { - err = fmt.Errorf("not implemented") - return -} - -func (dummy *globalsStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { - var ( - valueLen uint64 - ) - - _, err = cstruct.Unpack(payloadData, &valueLen, headhunter.LittleEndian) - if nil != err { - return - } - - if (globals.uint64Size + valueLen) > uint64(len(payloadData)) { - err = fmt.Errorf("payloadData insufficient to UnpackValue()") - return - } - - value = make([]byte, valueLen) - _ = copy(value.([]byte), payloadData) - - bytesConsumed = globals.uint64Size + valueLen - - return -} - -func doHead(path string) (headerMap http.Header, err error) { - var ( - headResponse *http.Response - ) - - headResponse, err = http.Head(globals.noAuthURL + "v1/" + path) - if nil != err { - return - } - - _, err = ioutil.ReadAll(headResponse.Body) - if nil != err { - return - } - err = headResponse.Body.Close() - if nil != err { - return - } - - if (http.StatusOK != headResponse.StatusCode) && (http.StatusNoContent != headResponse.StatusCode) { - err = fmt.Errorf("unexpected getResponse.Status: %s", headResponse.Status) - return - } - - headerMap = headResponse.Header - - return -} - -func doGetRange(path string, offset uint64, length uint64) (headerMap http.Header, buf []byte, err error) { - var ( - getRequest *http.Request - getResponse *http.Response - httpClient *http.Client - ) - - getRequest, err = http.NewRequest("GET", globals.noAuthURL+"v1/"+path, nil) - if nil != err { - return - } - - getRequest.Header.Set("Range", "bytes="+strconv.FormatUint(offset, 10)+"-"+strconv.FormatUint((offset+length-1), 10)) - - httpClient = &http.Client{} - - getResponse, err = httpClient.Do(getRequest) - if nil != err { - return - } - - buf, err = ioutil.ReadAll(getResponse.Body) - if nil != err { - return - } - err = getResponse.Body.Close() - if nil != err { - return - } - - if (http.StatusOK != getResponse.StatusCode) && (http.StatusNoContent != getResponse.StatusCode) && (http.StatusPartialContent != getResponse.StatusCode) { - err = fmt.Errorf("unexpected getResponse.Status: %s", getResponse.Status) - return - } - - if uint64(len(buf)) != length { - err = fmt.Errorf("unexpected getResponse.Body length") - return - } - - headerMap = getResponse.Header - - return -} - -func doGetTail(path string, length uint64) (headerMap http.Header, buf []byte, err error) { - var ( - getRequest *http.Request - getResponse *http.Response - httpClient *http.Client - ) - - getRequest, err = http.NewRequest("GET", globals.noAuthURL+"v1/"+path, nil) - if nil != err { - return - } - - getRequest.Header.Set("Range", "bytes=-"+strconv.FormatUint(length, 10)) - - httpClient = &http.Client{} - - getResponse, err = httpClient.Do(getRequest) - if nil != err { - return - } - - buf, err = ioutil.ReadAll(getResponse.Body) - if nil != err { - return - } - err = getResponse.Body.Close() - if nil != err { - return - } - - if (http.StatusOK != getResponse.StatusCode) && (http.StatusNoContent != getResponse.StatusCode) && (http.StatusPartialContent != getResponse.StatusCode) { - err = fmt.Errorf("unexpected getResponse.Status: %s", getResponse.Status) - return - } - - if uint64(len(buf)) != length { - err = fmt.Errorf("unexpected getResponse.Body length") - return - } - - headerMap = getResponse.Header - - return -} - -func doGetList(path string) (headerMap http.Header, list []string, err error) { - var ( - buf []byte - getResponse *http.Response - ) - - list = make([]string, 0) - - for { - if 0 == len(list) { - getResponse, err = http.Get(globals.noAuthURL + "v1/" + path) - } else { - getResponse, err = http.Get(globals.noAuthURL + "v1/" + path + "?marker=" + list[len(list)-1]) - } - if nil != err { - return - } - - buf, err = ioutil.ReadAll(getResponse.Body) - if nil != err { - return - } - err = getResponse.Body.Close() - if nil != err { - return - } - - if (http.StatusOK != getResponse.StatusCode) && (http.StatusNoContent != getResponse.StatusCode) && (http.StatusPartialContent != getResponse.StatusCode) { - err = fmt.Errorf("unexpected getResponse.Status: %s", getResponse.Status) - return - } - - time.Sleep(time.Second) - if 0 == len(buf) { - break - } - - list = append(list, strings.Split(string(buf[:]), "\n")...) - list = list[:len(list)-1] - } - - headerMap = getResponse.Header - - return -} - -func metadataRecycleBinHeaderPresent(headerMap http.Header) (present bool) { - _, present = headerMap[headhunter.MetadataRecycleBinHeaderName] - return -} diff --git a/pfs-restart-test/Makefile b/pfs-restart-test/Makefile deleted file mode 100644 index 83b2e56c..00000000 --- a/pfs-restart-test/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfs-restart-test - -include ../GoMakefile diff --git a/pfs-restart-test/dummy_test.go b/pfs-restart-test/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfs-restart-test/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfs-restart-test/macramswift0.conf b/pfs-restart-test/macramswift0.conf deleted file mode 100644 index b45c1ff1..00000000 --- a/pfs-restart-test/macramswift0.conf +++ /dev/null @@ -1,10 +0,0 @@ -[RestartTest] -Volume: CommonVolume # Must be referenced by [FSGlobals]VolumeGroupList with [Cluster]WhoAmI's PrimaryPeer -UseRAMSwift: true # If true, ramswift will be launched; if false, running Swift NoAuth Proxy is used -NumLoops: 100 # Number (>=1) of iterations to perform -NumCopies: 3 # Number (>=1) of copies of to perform each iteration -NumOverwrites: 2 # Number (>=0, <=) of copies of that are overwrites each iteration -SkipWrites: true # If true, every other iteration skips the copy step -DirPath: ../test # Identifies the source directory tree to copy/compare - -.include ../ramswift/macramswift0.conf diff --git a/pfs-restart-test/main.go b/pfs-restart-test/main.go deleted file mode 100644 index befd6c08..00000000 --- a/pfs-restart-test/main.go +++ /dev/null @@ -1,664 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "os" - "os/exec" - "strconv" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "golang.org/x/sys/unix" -) - -const ( - compareChunkSize = 65536 - dstDirPerm = os.FileMode(0700) - dstFilePerm = os.FileMode(0600) - startupPollingDelay = 100 * time.Millisecond -) - -type argsStruct struct { - Volume string - UseRAMSwift bool - NumLoops uint64 - NumCopies uint64 - NumOverwrites uint64 - SkipWrites bool - DirPath string -} - -var ( - args argsStruct - volumeGroupOfVolume string - checkpointInterval time.Duration - dirIndexAdvancePerNonSkippedLoop uint64 - fuseMountPointName string - proxyfsdCmd *exec.Cmd - proxyfsdVersionURL string - ramswiftCmd *exec.Cmd - testConfFileName string -) - -func main() { - var ( - loopIndex uint64 - ) - - setup() - - dirIndexAdvancePerNonSkippedLoop = args.NumCopies - args.NumOverwrites - - for loopIndex = 0; loopIndex < args.NumLoops; loopIndex++ { - log.Printf("Loop # %d\n", loopIndex) - doMount() - if 0 < loopIndex { - doCompares(loopIndex) - if (0 != dirIndexAdvancePerNonSkippedLoop) && (!args.SkipWrites || (0 == loopIndex%2)) { - doDeletions(loopIndex) - } - } - if !args.SkipWrites || (0 == loopIndex%2) { - doWrites(loopIndex) - } - doUnmount() - } - - if !args.SkipWrites || (0 != args.NumLoops%2) { - log.Printf("Final Compare\n") - doMount() - doCompares(args.NumLoops) - doUnmount() - } - - teardown() -} - -func setup() { - var ( - confFile string - confOverride string - confOverrides []string - err error - getInfoResponse *http.Response - getInfoURL string - mkproxyfsCmd *exec.Cmd - noAuthTCPPort uint16 - primaryPeerSlice []string - proxyfsdPrivateIPAddr string - proxyfsdTCPPortAsString string - proxyfsdTCPPortAsUint16 uint16 - testConfFile *os.File - testConfMap conf.ConfMap - volumeGroupList []string - volumeGroupListElement string - volumeGroup string - volumeList []string - volumeListElement string - whoAmI string - ) - - // Parse arguments - - if 2 > len(os.Args) { - fmt.Printf("%v []*\n", os.Args[0]) - os.Exit(1) - } - - confFile = os.Args[1] - confOverrides = os.Args[2:] - - testConfMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - log.Fatalf("confFile (\"%s\") not parseable: %v", confFile, err) - } - - for _, confOverride = range confOverrides { - err = testConfMap.UpdateFromString(confOverride) - if nil != err { - log.Fatalf("confOverride (\"%s\") not parseable: %v", confOverride, err) - } - } - - args.Volume, err = testConfMap.FetchOptionValueString("RestartTest", "Volume") - if nil != err { - log.Fatalf("unable to fetch RestartTest.Volume: %v", err) - } - - args.UseRAMSwift, err = testConfMap.FetchOptionValueBool("RestartTest", "UseRAMSwift") - if nil != err { - log.Fatalf("unable to fetch RestartTest.UseRAMSwift: %v", err) - } - - args.NumLoops, err = testConfMap.FetchOptionValueUint64("RestartTest", "NumLoops") - if nil != err { - log.Fatalf("unable to fetch RestartTest.NumLoops: %v", err) - } - if 0 == args.NumLoops { - log.Fatalf("RestartTest.NumLoops (%v) not valid", args.NumLoops) - } - - args.NumCopies, err = testConfMap.FetchOptionValueUint64("RestartTest", "NumCopies") - if nil != err { - log.Fatalf("unable to fetch RestartTest.NumCopies: %v", err) - } - if 0 == args.NumCopies { - log.Fatalf("RestartTest.NumCopies (%v) not valid", args.NumCopies) - } - - args.NumOverwrites, err = testConfMap.FetchOptionValueUint64("RestartTest", "NumOverwrites") - if nil != err { - log.Fatalf("unable to fetch RestartTest.NumOverwrites: %v", err) - } - if args.NumCopies < args.NumOverwrites { - log.Fatalf("RestartTest.NumOverwrites (%v) must be <= RestartTest.NumCopies (%v)", args.NumOverwrites, args.NumCopies) - } - - args.SkipWrites, err = testConfMap.FetchOptionValueBool("RestartTest", "SkipWrites") - if nil != err { - log.Fatalf("unable to fetch RestartTest.SkipWrites: %v", err) - } - - args.DirPath, err = testConfMap.FetchOptionValueString("RestartTest", "DirPath") - if nil != err { - log.Fatalf("unable to fetch RestartTest.DirPath: %v", err) - } - - // Fetch (RAMSwift &) ProxyFS values from testConfMap and validate Volume is served by this Peer - - whoAmI, err = testConfMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - log.Fatalf("unable to fetch Cluster.WhoAmI: %v", err) - } - - volumeGroupList, err = testConfMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - log.Fatalf("unable to fetch FSGlobals.VolumeGroupList: %v", err) - } - - for _, volumeGroupListElement = range volumeGroupList { - primaryPeerSlice, err = testConfMap.FetchOptionValueStringSlice("VolumeGroup:"+volumeGroupListElement, "PrimaryPeer") - if nil != err { - log.Fatalf("unable to fetch VolumeGroup:%s.PrimaryPeer: %v", volumeGroupListElement, err) - } - if (1 == len(primaryPeerSlice)) && (whoAmI == primaryPeerSlice[0]) { - volumeList, err = testConfMap.FetchOptionValueStringSlice("VolumeGroup:"+volumeGroupListElement, "VolumeList") - if nil != err { - log.Fatalf("unable to fetch VolumeGroup:%s.VolumeList: %v", volumeGroupListElement, err) - } - for _, volumeListElement = range volumeList { - if volumeListElement == args.Volume { - volumeGroup = volumeGroupListElement - goto FoundVolume - } - } - } - } - - log.Fatalf("unable to find Volume (\"%s\") in list of served Volumes for this peer (\"%s\")", args.Volume, whoAmI) - -FoundVolume: - - proxyfsdPrivateIPAddr, err = testConfMap.FetchOptionValueString("Peer:"+whoAmI, "PrivateIPAddr") - if nil != err { - log.Fatalf("unable to fetch Peer:%s.PrivateIPAddr: %v", whoAmI, err) - } - proxyfsdTCPPortAsUint16, err = testConfMap.FetchOptionValueUint16("HTTPServer", "TCPPort") - if nil != err { - log.Fatalf("unable to fetch HTTPServer.TCPPort: %v", err) - } - proxyfsdTCPPortAsString = fmt.Sprintf("%d", proxyfsdTCPPortAsUint16) - proxyfsdVersionURL = "http://" + net.JoinHostPort(proxyfsdPrivateIPAddr, proxyfsdTCPPortAsString) + "/version" - - testConfMap["FSGlobals"]["VolumeGroupList"] = []string{volumeGroup} - testConfMap["VolumeGroup:"+volumeGroup]["VolumeList"] = []string{args.Volume} - - fuseMountPointName, err = testConfMap.FetchOptionValueString("Volume:"+args.Volume, "FUSEMountPointName") - if nil != err { - log.Fatalf("unable to fetch Volume:%s.FUSEMountPointName: %v", args.Volume, err) - } - - noAuthTCPPort, err = testConfMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") - if nil != err { - log.Fatalf("unable to fetch SwiftClient.NoAuthTCPPort: %v", err) - } - - // Construct ConfFile to pass to ramswift, mkproxyfs, and proxyfsd - - testConfFile, err = ioutil.TempFile("", "pfs-fsck-*.conf") - if nil != err { - log.Fatalf("unable to create testConfFile: %v", err) - } - testConfFileName = testConfFile.Name() - _, err = testConfFile.Write([]byte(testConfMap.Dump())) - if nil != err { - log.Fatalf("unable to populate testConfFile (\"%s\"): %v", testConfFileName, err) - } - err = testConfFile.Close() - if nil != err { - log.Fatalf("unable to close testConfFile (\"%s\"): %v", testConfFileName, err) - } - - // Start-up ramswift if necessary - - if args.UseRAMSwift { - ramswiftCmd = exec.Command("ramswift", testConfFileName) - - err = ramswiftCmd.Start() - if nil != err { - log.Fatalf("unable to start ramswift: %v", err) - } - - getInfoURL = "http://127.0.0.1:" + strconv.Itoa(int(noAuthTCPPort)) + "/info" - - for { - getInfoResponse, err = http.Get(getInfoURL) - if nil == err { - _, err = ioutil.ReadAll(getInfoResponse.Body) - if nil != err { - log.Fatalf("unable to read getInfoResponse.Body: %v", err) - } - err = getInfoResponse.Body.Close() - if nil != err { - log.Fatalf("unable to close getInfoResponse.Body: %v", err) - } - if http.StatusOK == getInfoResponse.StatusCode { - break - } - } - time.Sleep(startupPollingDelay) - } - } - - // Format Volume - - mkproxyfsCmd = exec.Command("mkproxyfs", "-F", args.Volume, testConfFileName, "SwiftClient.RetryLimit=0") - - err = mkproxyfsCmd.Run() - if nil != err { - log.Fatalf("mkproxyfs -F Volume \"%s\" failed: %v", args.Volume, err) - } -} - -func doMount() { - var ( - err error - getVersionResponse *http.Response - ) - - // Start-up proxyfsd - - proxyfsdCmd = exec.Command("proxyfsd", testConfFileName) - - err = proxyfsdCmd.Start() - if nil != err { - log.Fatalf("unable to start proxyfsd: %v", err) - } - - for { - getVersionResponse, err = http.Get(proxyfsdVersionURL) - if nil == err { - _, err = ioutil.ReadAll(getVersionResponse.Body) - if nil != err { - log.Fatalf("unable to read getVersionResponse.Body: %v", err) - } - err = getVersionResponse.Body.Close() - if nil != err { - log.Fatalf("unable to close getVersionResponse.Body: %v", err) - } - if http.StatusOK == getVersionResponse.StatusCode { - break - } - } - time.Sleep(startupPollingDelay) - } -} - -func doCompares(loopIndex uint64) { - var ( - dirIndex uint64 - dirIndexEnd uint64 - dirIndexStart uint64 - dstDirPath string - nonSkippedLoopIndex uint64 - ) - - if args.SkipWrites { - nonSkippedLoopIndex = (loopIndex + 1) / 2 - } else { - nonSkippedLoopIndex = loopIndex - } - - dirIndexStart = (nonSkippedLoopIndex - 1) * dirIndexAdvancePerNonSkippedLoop - dirIndexEnd = dirIndexStart + (args.NumCopies - 1) - - for dirIndex = dirIndexStart; dirIndex <= dirIndexEnd; dirIndex++ { - dstDirPath = fmt.Sprintf(fuseMountPointName+"/%016X", dirIndex) - compareDir(args.DirPath, dstDirPath) - } -} - -func compareDir(srcDirPath string, dstDirPath string) { - var ( - dirEntryListIndex int - dstDirEntry os.FileInfo - dstDirEntryList []os.FileInfo - dstDirEntryListPruned []os.FileInfo - dstDirEntryPath string - err error - srcDirEntry os.FileInfo - srcDirEntryList []os.FileInfo - srcDirEntryListPruned []os.FileInfo - srcDirEntryPath string - ) - - srcDirEntryList, err = ioutil.ReadDir(srcDirPath) - if nil != err { - log.Fatalf("unable to read srcDirPath (\"%s\"): %v", srcDirPath, err) - } - dstDirEntryList, err = ioutil.ReadDir(dstDirPath) - if nil != err { - log.Fatalf("unable to read dstDirPath (\"%s\"): %v", dstDirPath, err) - } - - srcDirEntryListPruned = make([]os.FileInfo, 0, len(srcDirEntryList)) - dstDirEntryListPruned = make([]os.FileInfo, 0, len(dstDirEntryList)) - - for _, srcDirEntry = range srcDirEntryList { - if ".fseventsd" != srcDirEntry.Name() { - srcDirEntryListPruned = append(srcDirEntryListPruned, srcDirEntry) - } - } - for _, dstDirEntry = range dstDirEntryList { - if ".fseventsd" != dstDirEntry.Name() { - dstDirEntryListPruned = append(dstDirEntryListPruned, dstDirEntry) - } - } - - if len(srcDirEntryListPruned) != len(dstDirEntryListPruned) { - log.Fatalf("found unexpected discrepency between directory contents between \"%s\" & \"%s\"", srcDirPath, dstDirPath) - } - - for dirEntryListIndex = 0; dirEntryListIndex < len(srcDirEntryListPruned); dirEntryListIndex++ { // || len(dstDirEntryListPruned) - srcDirEntry = srcDirEntryListPruned[dirEntryListIndex] - dstDirEntry = dstDirEntryListPruned[dirEntryListIndex] - srcDirEntryPath = srcDirPath + "/" + srcDirEntry.Name() - dstDirEntryPath = dstDirPath + "/" + dstDirEntry.Name() - if srcDirEntry.IsDir() { - if !dstDirEntry.IsDir() { - log.Fatalf("dstDirPath/%s was not expected to be a dir", dstDirEntryPath) - } - compareDir(srcDirEntryPath, dstDirEntryPath) - } else { // !srcDirEntry.IsDir() - if dstDirEntry.IsDir() { - log.Fatalf("dstDirPath/%s was expected to be a dir", dstDirEntryPath) - } - compareFile(srcDirEntryPath, dstDirEntryPath) - } - } -} - -func compareFile(srcFilePath string, dstFilePath string) { - var ( - err error - dstFile *os.File - dstFileBuf []byte - dstFileLen int - dstFileStat os.FileInfo - fileLen int64 - fileOffset int64 - srcFile *os.File - srcFileBuf []byte - srcFileLen int - srcFileStat os.FileInfo - ) - - srcFile, err = os.Open(srcFilePath) - if nil != err { - log.Fatalf("unable to open srcFilePath (\"%s\"): %v", srcFilePath, err) - } - srcFileStat, err = srcFile.Stat() - if nil != err { - log.Fatalf("unable to stat srcFilePath (\"%s\"): %v", srcFilePath, err) - } - dstFile, err = os.Open(dstFilePath) - if nil != err { - log.Fatalf("unable to open dstFilePath (\"%s\"): %v", dstFilePath, err) - } - dstFileStat, err = dstFile.Stat() - if nil != err { - log.Fatalf("unable to stat srcFilePath (\"%s\"): %v", srcFilePath, err) - } - - if srcFileStat.Size() != dstFileStat.Size() { - log.Fatalf("srcFilePath (\"%s\") and dstFilePath (\"%s\") sizes done match", srcFilePath, dstFilePath) - } - - srcFileBuf = make([]byte, compareChunkSize, compareChunkSize) - dstFileBuf = make([]byte, compareChunkSize, compareChunkSize) - - fileLen = srcFileStat.Size() // || dstFileStat.Size() - fileOffset = 0 - - for fileLen < fileOffset { - srcFileLen, err = srcFile.Read(srcFileBuf) - if nil != err { - log.Fatalf("unable to read srcFilePath (\"%s\"): %v", srcFilePath, err) - } - dstFileLen, err = srcFile.Read(dstFileBuf) - if nil != err { - log.Fatalf("unable to read dstFilePath (\"%s\"): %v", dstFilePath, err) - } - - if srcFileLen != dstFileLen { - log.Fatalf("unexpectedly read differing buffer sizes from srcFilePath (\"%s\") [%d] & dstFilePath (\"%s\") [%d]", srcFilePath, srcFileLen, dstFilePath, dstFileLen) - } - - if 0 == bytes.Compare(srcFileBuf[:srcFileLen], dstFileBuf[:srcFileLen]) { - log.Fatalf("unexpectedly read differing buffer contents from srcFilePath (\"%s\") and dstFilePath (\"%s\") at offset %d", srcFilePath, dstFilePath, fileOffset) - } - - fileOffset += int64(srcFileLen) // || int64(dstFileLen) - } - - err = srcFile.Close() - if nil != err { - log.Fatalf("unable to close srcFilePath (\"%s\"): %v", srcFilePath, err) - } - err = dstFile.Close() - if nil != err { - log.Fatalf("unable to close dstFilePath (\"%s\"): %v", dstFilePath, err) - } -} - -func doDeletions(loopIndex uint64) { - var ( - dstDirPath string - dirIndex uint64 - dirIndexEnd uint64 - dirIndexStart uint64 - err error - nonSkippedLoopIndex uint64 - ) - - if args.SkipWrites { - nonSkippedLoopIndex = (loopIndex + 1) / 2 - } else { - nonSkippedLoopIndex = loopIndex - } - - dirIndexStart = (nonSkippedLoopIndex - 1) * dirIndexAdvancePerNonSkippedLoop - dirIndexEnd = dirIndexStart + (dirIndexAdvancePerNonSkippedLoop - 1) - - for dirIndex = dirIndexStart; dirIndex <= dirIndexEnd; dirIndex++ { - dstDirPath = fmt.Sprintf(fuseMountPointName+"/%016X", dirIndex) - err = os.RemoveAll(dstDirPath) - if nil != err { - log.Fatalf("unable to remove dstDirPath (\"%s\"): %v", dstDirPath, err) - } - } -} - -func doWrites(loopIndex uint64) { - var ( - dirIndex uint64 - dirIndexEnd uint64 - dirIndexStart uint64 - dstDirPath string - nonSkippedLoopIndex uint64 - numOverwritesLeft uint64 - ) - - if args.SkipWrites { - nonSkippedLoopIndex = loopIndex / 2 - } else { - nonSkippedLoopIndex = loopIndex - } - - dirIndexStart = nonSkippedLoopIndex * dirIndexAdvancePerNonSkippedLoop - dirIndexEnd = dirIndexStart + (args.NumCopies - 1) - - if 0 == loopIndex { - numOverwritesLeft = 0 - } else { - numOverwritesLeft = args.NumOverwrites - } - - for dirIndex = dirIndexStart; dirIndex <= dirIndexEnd; dirIndex++ { - dstDirPath = fmt.Sprintf(fuseMountPointName+"/%016X", dirIndex) - if 0 < numOverwritesLeft { - cloneDir(args.DirPath, dstDirPath, true) - numOverwritesLeft-- - } else { - cloneDir(args.DirPath, dstDirPath, false) - } - } -} - -func cloneDir(srcDirPath string, dstDirPath string, overwrite bool) { - var ( - dstDirEntryPath string - err error - srcDirEntry os.FileInfo - srcDirEntryList []os.FileInfo - srcDirEntryPath string - ) - - if !overwrite { - err = os.Mkdir(dstDirPath, dstDirPerm) - if nil != err { - log.Fatalf("unable to create dstDirPath (\"%s\"): %v", dstDirPath, err) - } - } - - srcDirEntryList, err = ioutil.ReadDir(srcDirPath) - if nil != err { - log.Fatalf("unable to read srcDirPath (\"%s\"): %v", srcDirPath, err) - } - - for _, srcDirEntry = range srcDirEntryList { - if ("." != srcDirEntry.Name()) && (".." != srcDirEntry.Name()) { - srcDirEntryPath = srcDirPath + "/" + srcDirEntry.Name() - dstDirEntryPath = dstDirPath + "/" + srcDirEntry.Name() - if srcDirEntry.IsDir() { - cloneDir(srcDirEntryPath, dstDirEntryPath, overwrite) - } else { - cloneFile(srcDirEntryPath, dstDirEntryPath, overwrite) - } - } - } -} - -func cloneFile(srcFilePath string, dstFilePath string, overwrite bool) { - var ( - err error - dstFile *os.File - srcFile *os.File - ) - - srcFile, err = os.Open(srcFilePath) - if nil != err { - log.Fatalf("unable to open srcFilePath (\"%s\"): %v", srcFilePath, err) - } - - if overwrite { - dstFile, err = os.OpenFile(dstFilePath, os.O_WRONLY|os.O_TRUNC, dstFilePerm) - if nil != err { - log.Fatalf("unable to open existing dstFilePath (\"%s\"): %v", dstFilePath, err) - } - } else { - dstFile, err = os.OpenFile(dstFilePath, os.O_WRONLY|os.O_EXCL|os.O_CREATE, dstFilePerm) - if nil != err { - log.Fatalf("unable to create dstFilePath (\"%s\"): %v", dstFilePath, err) - } - } - - _, err = io.Copy(dstFile, srcFile) - if nil != err { - log.Fatalf("unable to copp srcFilePath (\"%s\") to dstFilePath (\"%s\"): %v", srcFilePath, dstFilePath, err) - } - - err = dstFile.Close() - if nil != err { - log.Fatalf("unable to close dstFilePath (\"%s\"): %v", dstFilePath, err) - } - - err = srcFile.Close() - if nil != err { - log.Fatalf("unable to close srcFilePath (\"%s\"): %v", srcFilePath, err) - } -} - -func doUnmount() { - var ( - err error - ) - - // Stop proxyfsd - - err = proxyfsdCmd.Process.Signal(unix.SIGTERM) - if nil != err { - log.Fatalf("unable to send SIGTERM to proxyfsd: %v", err) - } - - err = proxyfsdCmd.Wait() - if nil != err { - log.Printf("failure waiting for proxyfsd to exit: %v", err) - } -} - -func teardown() { - var ( - err error - ) - - // Stop ramswift if started earlier - - if args.UseRAMSwift { - err = ramswiftCmd.Process.Signal(unix.SIGTERM) - if nil != err { - log.Fatalf("unable to send SIGTERM to ramswift: %v", err) - } - - err = ramswiftCmd.Wait() - if nil != err { - log.Fatalf("failure waiting for ramswift to exit: %v", err) - } - } - - // Clean-up - - err = os.Remove(testConfFileName) - if nil != err { - log.Fatalf("unable to delete testConfFile (\"%s\"): %v", testConfFileName, err) - } -} diff --git a/pfs-restart-test/saioproxyfsd0.conf b/pfs-restart-test/saioproxyfsd0.conf deleted file mode 100644 index 9ecafa13..00000000 --- a/pfs-restart-test/saioproxyfsd0.conf +++ /dev/null @@ -1,10 +0,0 @@ -[RestartTest] -Volume: CommonVolume # Must be referenced by [FSGlobals]VolumeGroupList with [Cluster]WhoAmI's PrimaryPeer -UseRAMSwift: false # If true, ramswift will be launched; if false, running Swift NoAuth Proxy is used -NumLoops: 100 # Number (>=1) of iterations to perform -NumCopies: 3 # Number (>=1) of copies of to perform each iteration -NumOverwrites: 2 # Number (>=0, <=) of copies of that are overwrites each iteration -SkipWrites: true # If true, every other iteration skips the copy step -DirPath: ../test # Identifies the source directory tree to copy/compare - -.include ../proxyfsd/saioproxyfsd0.conf diff --git a/pfs-restart-test/saioramswift0.conf b/pfs-restart-test/saioramswift0.conf deleted file mode 100644 index 65e6b02f..00000000 --- a/pfs-restart-test/saioramswift0.conf +++ /dev/null @@ -1,10 +0,0 @@ -[RestartTest] -Volume: CommonVolume # Must be referenced by [FSGlobals]VolumeGroupList with [Cluster]WhoAmI's PrimaryPeer -UseRAMSwift: true # If true, ramswift will be launched; if false, running Swift NoAuth Proxy is used -NumLoops: 100 # Number (>=1) of iterations to perform -NumCopies: 3 # Number (>=1) of copies of to perform each iteration -NumOverwrites: 2 # Number (>=0, <=) of copies of that are overwrites each iteration -SkipWrites: true # If true, every other iteration skips the copy step -DirPath: ../test # Identifies the source directory tree to copy/compare - -.include ../ramswift/saioramswift0.conf diff --git a/pfs-stress/Makefile b/pfs-stress/Makefile deleted file mode 100644 index 48c00556..00000000 --- a/pfs-stress/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfs-stress - -include ../GoMakefile diff --git a/pfs-stress/dummy_test.go b/pfs-stress/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfs-stress/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfs-stress/main.go b/pfs-stress/main.go deleted file mode 100644 index 08b7d566..00000000 --- a/pfs-stress/main.go +++ /dev/null @@ -1,385 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "crypto/rand" - "fmt" - "log" - "math" - "math/big" - "os" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/NVIDIA/proxyfs/conf" -) - -var ( - displayUpdateInterval time.Duration - filePathPrefix string - fileSize uint64 - maxExtentSize uint64 - minExtentSize uint64 - numExtentsToWriteInTotal uint64 - numExtentsToWritePerFile uint32 - numExtentWritesPerFlush uint32 - numExtentWritesPerValidate uint32 - numExtentsWrittenInTotal uint64 - numFiles uint16 - waitGroup sync.WaitGroup -) - -func main() { - var ( - args []string - confMap conf.ConfMap - displayUpdateIntervalAsString string - displayUpdateIntervalPad string - err error - filePathPrefixPad string - fileSizeAsString string - fileSizePad string - maxExtentSizeAsString string - maxExtentSizePad string - maxParameterStringLen int - minExtentSizeAsString string - minExtentSizePad string - numExtentsToWritePerFileAsString string - numExtentsToWritePerFilePad string - numExtentWritesPerFlushAsString string - numExtentWritesPerFlushPad string - numExtentWritesPerValidateAsString string - numExtentWritesPerValidatePad string - numFilesAsString string - numFilesPad string - progressPercentage uint64 - stresserIndex uint16 - ) - - // Parse arguments - - args = os.Args[1:] - - // Read in the program's os.Arg[1]-specified (and required) .conf file - if 0 == len(args) { - log.Fatalf("no .conf file specified") - } - - confMap, err = conf.MakeConfMapFromFile(args[0]) - if nil != err { - log.Fatalf("failed to load config: %v", err) - } - - // Update confMap with any extra os.Args supplied - err = confMap.UpdateFromStrings(args[1:]) - if nil != err { - log.Fatalf("failed to load config overrides: %v", err) - } - - // Process resultant confMap - - filePathPrefix, err = confMap.FetchOptionValueString("StressParameters", "FilePathPrefix") - if nil != err { - log.Fatal(err) - } - - numFiles, err = confMap.FetchOptionValueUint16("StressParameters", "NumFiles") - if nil != err { - log.Fatal(err) - } - if 0 == numFiles { - log.Fatalf("NumFiles must be > 0") - } - - fileSize, err = confMap.FetchOptionValueUint64("StressParameters", "FileSize") - if nil != err { - log.Fatal(err) - } - if 0 == fileSize { - log.Fatalf("FileSize must be > 0") - } - if fileSize > math.MaxInt64 { - log.Fatalf("FileSize(%v) must be <= math.MaxInt64(%v)", fileSize, uint64(math.MaxInt64)) - } - - minExtentSize, err = confMap.FetchOptionValueUint64("StressParameters", "MinExtentSize") - if nil != err { - log.Fatal(err) - } - if 0 == minExtentSize { - log.Fatalf("MinExtentSize must be > 0") - } - if minExtentSize > fileSize { - log.Fatalf("MinExtentSize(%v) must be <= FileSize(%v)", minExtentSize, fileSize) - } - - maxExtentSize, err = confMap.FetchOptionValueUint64("StressParameters", "MaxExtentSize") - if nil != err { - log.Fatal(err) - } - if maxExtentSize < minExtentSize { - log.Fatalf("MaxExtentSize(%v) must be >= MinExtentSize(%v)", maxExtentSize, minExtentSize) - } - if maxExtentSize > fileSize { - log.Fatalf("MaxExtentSize(%v) must be <= FileSize(%v)", maxExtentSize, fileSize) - } - - numExtentsToWritePerFile, err = confMap.FetchOptionValueUint32("StressParameters", "NumExtentsToWritePerFile") - if nil != err { - log.Fatal(err) - } - if 0 == numExtentsToWritePerFile { - log.Fatalf("NumExtentsToWritePerFile must be > 0") - } - - numExtentWritesPerFlush, err = confMap.FetchOptionValueUint32("StressParameters", "NumExtentWritesPerFlush") - if nil != err { - log.Fatal(err) - } - if 0 == numExtentWritesPerFlush { - numExtentWritesPerFlush = numExtentsToWritePerFile - } - - numExtentWritesPerValidate, err = confMap.FetchOptionValueUint32("StressParameters", "NumExtentWritesPerValidate") - if nil != err { - log.Fatal(err) - } - if 0 == numExtentWritesPerValidate { - numExtentWritesPerValidate = numExtentsToWritePerFile - } - - displayUpdateInterval, err = confMap.FetchOptionValueDuration("StressParameters", "DisplayUpdateInterval") - if nil != err { - log.Fatal(err) - } - - // Display parameters - - numFilesAsString = fmt.Sprintf("%d", numFiles) - fileSizeAsString = fmt.Sprintf("%d", fileSize) - minExtentSizeAsString = fmt.Sprintf("%d", minExtentSize) - maxExtentSizeAsString = fmt.Sprintf("%d", maxExtentSize) - numExtentsToWritePerFileAsString = fmt.Sprintf("%d", numExtentsToWritePerFile) - numExtentWritesPerFlushAsString = fmt.Sprintf("%d", numExtentWritesPerFlush) - numExtentWritesPerValidateAsString = fmt.Sprintf("%d", numExtentWritesPerValidate) - displayUpdateIntervalAsString = fmt.Sprintf("%s", displayUpdateInterval) - - maxParameterStringLen = len(filePathPrefix) - if len(numFilesAsString) > maxParameterStringLen { - maxParameterStringLen = len(numFilesAsString) - } - if len(fileSizeAsString) > maxParameterStringLen { - maxParameterStringLen = len(fileSizeAsString) - } - if len(minExtentSizeAsString) > maxParameterStringLen { - maxParameterStringLen = len(minExtentSizeAsString) - } - if len(maxExtentSizeAsString) > maxParameterStringLen { - maxParameterStringLen = len(maxExtentSizeAsString) - } - if len(numExtentsToWritePerFileAsString) > maxParameterStringLen { - maxParameterStringLen = len(numExtentsToWritePerFileAsString) - } - if len(numExtentWritesPerFlushAsString) > maxParameterStringLen { - maxParameterStringLen = len(numExtentWritesPerFlushAsString) - } - if len(numExtentWritesPerValidateAsString) > maxParameterStringLen { - maxParameterStringLen = len(numExtentWritesPerValidateAsString) - } - if len(displayUpdateIntervalAsString) > maxParameterStringLen { - maxParameterStringLen = len(displayUpdateIntervalAsString) - } - - filePathPrefixPad = strings.Repeat(" ", maxParameterStringLen-len(filePathPrefix)) - numFilesPad = strings.Repeat(" ", maxParameterStringLen-len(numFilesAsString)) - fileSizePad = strings.Repeat(" ", maxParameterStringLen-len(fileSizeAsString)) - minExtentSizePad = strings.Repeat(" ", maxParameterStringLen-len(minExtentSizeAsString)) - maxExtentSizePad = strings.Repeat(" ", maxParameterStringLen-len(maxExtentSizeAsString)) - numExtentsToWritePerFilePad = strings.Repeat(" ", maxParameterStringLen-len(numExtentsToWritePerFileAsString)) - numExtentWritesPerFlushPad = strings.Repeat(" ", maxParameterStringLen-len(numExtentWritesPerFlushAsString)) - numExtentWritesPerValidatePad = strings.Repeat(" ", maxParameterStringLen-len(numExtentWritesPerValidateAsString)) - displayUpdateIntervalPad = strings.Repeat(" ", maxParameterStringLen-len(displayUpdateIntervalAsString)) - - fmt.Println("[StressParameters]") - fmt.Printf("FilePathPrefix: %s%s\n", filePathPrefixPad, filePathPrefix) - fmt.Printf("NumFiles: %s%s\n", numFilesPad, numFilesAsString) - fmt.Printf("FileSize: %s%s\n", fileSizePad, fileSizeAsString) - fmt.Printf("MinExtentSize: %s%s\n", minExtentSizePad, minExtentSizeAsString) - fmt.Printf("MaxExtentSize: %s%s\n", maxExtentSizePad, maxExtentSizeAsString) - fmt.Printf("NumExtentsToWritePerFile: %s%s\n", numExtentsToWritePerFilePad, numExtentsToWritePerFileAsString) - fmt.Printf("NumExtentWritesPerFlush: %s%s\n", numExtentWritesPerFlushPad, numExtentWritesPerFlushAsString) - fmt.Printf("NumExtentWritesPerValidate: %s%s\n", numExtentWritesPerValidatePad, numExtentWritesPerValidateAsString) - fmt.Printf("DisplayUpdateInterval: %s%s\n", displayUpdateIntervalPad, displayUpdateIntervalAsString) - - // Setup monitoring parameters - - numExtentsToWriteInTotal = uint64(numFiles) * uint64(numExtentsToWritePerFile) - numExtentsWrittenInTotal = uint64(0) - - // Launch fileStresser goroutines - - waitGroup.Add(int(numFiles)) - - for stresserIndex = 0; stresserIndex < numFiles; stresserIndex++ { - go fileStresser(stresserIndex) - } - - // Monitor fileStresser goroutines - - for { - time.Sleep(displayUpdateInterval) - progressPercentage = 100 * atomic.LoadUint64(&numExtentsWrittenInTotal) / numExtentsToWriteInTotal - fmt.Printf("\rProgress: %3d%%", progressPercentage) - if 100 == progressPercentage { - break - } - } - - waitGroup.Wait() - - fmt.Println("... done!") -} - -type fileStresserContext struct { - filePath string - file *os.File - written []byte -} - -func fileStresser(stresserIndex uint16) { - var ( - b uint8 - err error - extentIndex uint32 - fSC *fileStresserContext - l int64 - mustBeLessThanBigIntPtr *big.Int - numExtentWritesSinceLastFlush uint32 - numExtentWritesSinceLastValidate uint32 - off int64 - u64BigIntPtr *big.Int - ) - - // Construct this instance's fileStresserContext - - fSC = &fileStresserContext{ - filePath: fmt.Sprintf("%s%04X", filePathPrefix, stresserIndex), - written: make([]byte, fileSize), - } - - fSC.file, err = os.OpenFile(fSC.filePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) - if nil != err { - log.Fatal(err) - } - - fSC.fileStresserWriteAt(int64(0), int64(fileSize), 0x00) - fSC.fileStresserValidate() - - // Perform extent writes - - b = 0x00 - numExtentWritesSinceLastFlush = 0 - numExtentWritesSinceLastValidate = 0 - - for extentIndex = 0; extentIndex < numExtentsToWritePerFile; extentIndex++ { - // Pick an l value such that minExtentSize <= l <= maxExtentSize - mustBeLessThanBigIntPtr = big.NewInt(int64(maxExtentSize - minExtentSize + 1)) - u64BigIntPtr, err = rand.Int(rand.Reader, mustBeLessThanBigIntPtr) - if nil != err { - log.Fatal(err) - } - l = int64(minExtentSize) + u64BigIntPtr.Int64() - - // Pick an off value such that 0 <= off <= (fileSize - l) - mustBeLessThanBigIntPtr = big.NewInt(int64(int64(fileSize) - l)) - u64BigIntPtr, err = rand.Int(rand.Reader, mustBeLessThanBigIntPtr) - if nil != err { - log.Fatal(err) - } - off = u64BigIntPtr.Int64() - - // Pick next b value (skipping 0x00 for as-yet-un-over-written bytes) - b++ - if 0x00 == b { - b = 0x01 - } - - fSC.fileStresserWriteAt(off, l, b) - - numExtentWritesSinceLastFlush++ - - if numExtentWritesPerFlush == numExtentWritesSinceLastFlush { - fSC.fileStresserFlush() - numExtentWritesSinceLastFlush = 0 - } - - numExtentWritesSinceLastValidate++ - - if numExtentWritesPerValidate == numExtentWritesSinceLastValidate { - fSC.fileStresserValidate() - numExtentWritesSinceLastValidate = 0 - } - - atomic.AddUint64(&numExtentsWrittenInTotal, uint64(1)) - } - - // Do one final fileStresserFlush() call if necessary to flush final writes - - if 0 < numExtentWritesSinceLastFlush { - fSC.fileStresserFlush() - } - - // Do one final fileStresserValidate() call if necessary to validate final writes - - if 0 < numExtentWritesSinceLastValidate { - fSC.fileStresserValidate() - } - - // Clean up and exit - - err = fSC.file.Close() - if nil != err { - log.Fatal(err) - } - err = os.Remove(fSC.filePath) - if nil != err { - log.Fatal(err) - } - - waitGroup.Done() -} - -func (fSC *fileStresserContext) fileStresserWriteAt(off int64, l int64, b byte) { - buf := make([]byte, l) - for i := int64(0); i < l; i++ { - buf[i] = b - fSC.written[off+i] = b - } - _, err := fSC.file.WriteAt(buf, off) - if nil != err { - log.Fatal(err) - } -} - -func (fSC *fileStresserContext) fileStresserFlush() { - err := fSC.file.Sync() - if nil != err { - log.Fatal(err) - } -} - -func (fSC *fileStresserContext) fileStresserValidate() { - buf := make([]byte, fileSize) - _, err := fSC.file.ReadAt(buf, int64(0)) - if nil != err { - log.Fatal(err) - } - if 0 != bytes.Compare(fSC.written, buf) { - log.Fatalf("Miscompare in filePath %s\n", fSC.filePath) - } -} diff --git a/pfs-stress/pfs-stress.conf b/pfs-stress/pfs-stress.conf deleted file mode 100644 index d1ce1fb0..00000000 --- a/pfs-stress/pfs-stress.conf +++ /dev/null @@ -1,10 +0,0 @@ -[StressParameters] -FilePathPrefix: _pfs_stress_file_ # unless starting with '/', relative to $CWD -NumFiles: 10 -FileSize: 10000000 -MinExtentSize: 1 -MaxExtentSize: 10000 -NumExtentsToWritePerFile: 1000 -NumExtentWritesPerFlush: 50 # 0 means only perform Flush function at the end -NumExtentWritesPerValidate: 100 # 0 means only perform Validate function at the end -DisplayUpdateInterval: 100ms diff --git a/pfs-swift-load/.gitignore b/pfs-swift-load/.gitignore deleted file mode 100644 index 5ceb3864..00000000 --- a/pfs-swift-load/.gitignore +++ /dev/null @@ -1 +0,0 @@ -venv diff --git a/pfs-swift-load/Makefile b/pfs-swift-load/Makefile deleted file mode 100644 index fdcb9dc1..00000000 --- a/pfs-swift-load/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfs-swift-load - -include ../GoMakefile diff --git a/pfs-swift-load/dummy_test.go b/pfs-swift-load/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfs-swift-load/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfs-swift-load/main.go b/pfs-swift-load/main.go deleted file mode 100644 index a1880199..00000000 --- a/pfs-swift-load/main.go +++ /dev/null @@ -1,1000 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "crypto/rand" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "strings" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/conf" -) - -const ( - fileWrite = "write" - fileStat = "stat" - fileRead = "read" - fileDelete = "delete" -) - -type workerStruct struct { - sync.Mutex - - name string - - methodList []string // One or more of: - // fileWrite http.MethodPut - // fileStat http.MethodHead - // fileRead http.MethodGet - // fileDelete http.MethodDelete - - mountPoint string // Typically corresponds to swiftAccount - directory string // Typically matches swiftContainer - subDirectoryPath string // Optional - but, if present, typically matches swiftObjectPrefix (plus trailing '/') - fileBlockCount uint64 // Typically, - fileBlockSize uint64 // (fileBlockCount * fileBlockSize) == objectSize - - swiftProxyURL string - swiftAuthUser string - swiftAuthKey string - swiftAccount string // Typically corresponds to mountPoint - swiftContainer string // Typically matches directory - swiftContainerStoragePolicy string // Optional - but, if present, typically matches subDirectoryPath (minus trailing '/') - swiftObjectPrefix string // - objectSize uint64 // Typically matches (fileBlockCount * fileBlockSize) - - iterations uint64 - numThreads uint64 - - methodListIncludesFileMethod bool - methodListIncludesSwiftMethod bool - - fileWriteBuffer []byte - swiftPutBuffer []byte - - swiftAuthToken string - swiftAuthTokenUpdateWaitGroup *sync.WaitGroup - authorizationsPerformed uint64 - - nextIteration uint64 - iterationsCompleted uint64 - priorIterationsCompleted uint64 -} - -type globalsStruct struct { - sync.Mutex - optionDestroy bool - optionFormat bool - logFile *os.File - workersDoneGate sync.WaitGroup - workersReadyGate sync.WaitGroup - workersGoGate sync.WaitGroup - liveThreadCount uint64 - elapsedTime time.Duration -} - -var globals globalsStruct - -type workerIntervalValues struct { - iterationsCompleted uint64 - iterationsDelta uint64 -} - -var columnTitles = []string{"ElapsedTime", "Completed", "Delta"} - -func main() { - var ( - args []string - confMap conf.ConfMap - displayInterval time.Duration - err error - iterationsCompleted uint64 - iterationsDelta uint64 - logHeaderLine string - logPath string - method string - methodListIncludesFileWrite bool - methodListIncludesSwiftPut bool - optionString string - worker *workerStruct - workerArray []*workerStruct - workerName string - workerList []string - workerSectionName string - workersIntervals []workerIntervalValues - ) - - // Parse arguments - - args = os.Args[1:] - - // Check that os.Args[1] was supplied... it might be a .conf or an option list (followed by a .conf) - if 0 == len(args) { - log.Fatalf("No .conf file specified") - } - - if "-" == args[0][:1] { - // Peel off option list from args - optionString = args[0] - args = args[1:] - - // Check that os.Args[2]-specified (and required) .conf was supplied - if 0 == len(args) { - log.Fatalf("No .conf file specified") - } - - switch optionString { - case "-F": - globals.optionFormat = true - globals.optionDestroy = false - case "-D": - globals.optionFormat = false - globals.optionDestroy = true - case "-DF": - globals.optionFormat = true - globals.optionDestroy = true - case "-FD": - globals.optionFormat = true - globals.optionDestroy = true - default: - log.Fatalf("unexpected option supplied: %s", optionString) - } - } else { - globals.optionDestroy = false - globals.optionFormat = false - } - - confMap, err = conf.MakeConfMapFromFile(args[0]) - if nil != err { - log.Fatalf("Failed to load config: %v", err) - } - - // Update confMap with any extra os.Args supplied - err = confMap.UpdateFromStrings(args[1:]) - if nil != err { - log.Fatalf("Failed to load config overrides: %v", err) - } - - // Process resultant confMap - - workerList, err = confMap.FetchOptionValueStringSlice("LoadParameters", "WorkerList") - if nil != err { - log.Fatalf("Fetching LoadParameters.WorkerList failed: failed: %v", err) - } - if 0 == len(workerList) { - log.Fatalf("LoadParameters.WorkerList must not be empty") - } - displayInterval, err = confMap.FetchOptionValueDuration("LoadParameters", "DisplayInterval") - if nil != err { - log.Fatalf("Fetching LoadParameters.DisplayInterval failed: %v", err) - } - logPath, err = confMap.FetchOptionValueString("LoadParameters", "LogPath") - if nil != err { - log.Fatalf("Fetching LoadParameters.LogPath failed: %v", err) - } - - workerArray = make([]*workerStruct, 0, len(workerList)) - - if globals.optionDestroy && !globals.optionFormat { - // We will be exiting once optionDestroy is complete - } else { - globals.logFile, err = os.Create(logPath) - if nil != err { - log.Fatalf("os.Create(\"%s\") failed: %v", logPath, err) - } - - globals.liveThreadCount = 0 - - globals.workersGoGate.Add(1) - } - - for _, workerName = range workerList { - worker = &workerStruct{name: workerName} - - workerSectionName = "Worker:" + workerName - - worker.methodList, err = confMap.FetchOptionValueStringSlice(workerSectionName, "MethodList") - if nil != err { - log.Fatalf("confMap.FetchOptionValueStringSlice(\"%s\", \"MethodList\") failed: %v", workerSectionName, err) - } - if 0 == len(worker.methodList) { - log.Fatalf("%v.MethodList must not be empty", workerSectionName) - } - - worker.methodListIncludesFileMethod = false - worker.methodListIncludesSwiftMethod = false - - methodListIncludesFileWrite = false - methodListIncludesSwiftPut = false - - for _, method = range worker.methodList { - switch method { - case fileWrite: - worker.methodListIncludesFileMethod = true - methodListIncludesFileWrite = true - case fileStat: - worker.methodListIncludesFileMethod = true - case fileRead: - worker.methodListIncludesFileMethod = true - case fileDelete: - worker.methodListIncludesFileMethod = true - case http.MethodPut: - worker.methodListIncludesSwiftMethod = true - methodListIncludesSwiftPut = true - case http.MethodHead: - worker.methodListIncludesSwiftMethod = true - case http.MethodGet: - worker.methodListIncludesSwiftMethod = true - case http.MethodDelete: - worker.methodListIncludesSwiftMethod = true - default: - log.Fatalf("Method %s in %s.MethodList not supported", method, workerSectionName) - } - } - - if worker.methodListIncludesFileMethod { - worker.mountPoint, err = confMap.FetchOptionValueString(workerSectionName, "MountPoint") - if nil != err { - log.Fatalf("Fetching %s.MountPoint failed: %v", workerSectionName, err) - } - worker.directory, err = confMap.FetchOptionValueString(workerSectionName, "Directory") - if nil != err { - log.Fatalf("Fetching %s.Directory failed: %v", workerSectionName, err) - } - worker.subDirectoryPath, err = confMap.FetchOptionValueString(workerSectionName, "SubDirectoryPath") - if nil == err { - if strings.HasSuffix(worker.subDirectoryPath, "/") { - worker.subDirectoryPath = worker.subDirectoryPath[:len(worker.subDirectoryPath)-1] - } - } else { - worker.subDirectoryPath = "" - } - worker.fileBlockCount, err = confMap.FetchOptionValueUint64(workerSectionName, "FileBlockCount") - if nil != err { - log.Fatalf("Fetching %s.FileBlockCount failed: %v", workerSectionName, err) - } - worker.fileBlockSize, err = confMap.FetchOptionValueUint64(workerSectionName, "FileBlockSize") - if nil != err { - log.Fatalf("Fetching %s.FileBlockSize failed: %v", workerSectionName, err) - } - - if methodListIncludesFileWrite { - worker.fileWriteBuffer = make([]byte, worker.fileBlockSize) - rand.Read(worker.fileWriteBuffer) - } - } - - if worker.methodListIncludesSwiftMethod { - worker.swiftProxyURL, err = confMap.FetchOptionValueString(workerSectionName, "SwiftProxyURL") - if nil != err { - log.Fatalf("Fetching %s.SwiftProxyURL failed: %v", workerSectionName, err) - } - worker.swiftAuthUser, err = confMap.FetchOptionValueString(workerSectionName, "SwiftAuthUser") - if nil != err { - log.Fatalf("Fetching %s.SwiftAuthUser failed: %v", workerSectionName, err) - } - worker.swiftAuthKey, err = confMap.FetchOptionValueString(workerSectionName, "SwiftAuthKey") - if nil != err { - log.Fatalf("Fetching %s.SwiftAuthKey failed: %v", workerSectionName, err) - } - worker.swiftAccount, err = confMap.FetchOptionValueString(workerSectionName, "SwiftAccount") - if nil != err { - log.Fatalf("Fetching %s.SwiftAccount failed: %v", workerSectionName, err) - } - worker.swiftContainer, err = confMap.FetchOptionValueString(workerSectionName, "SwiftContainer") - if nil != err { - log.Fatalf("Fetching %s.SwiftContainer failed: %v", workerSectionName, err) - } - worker.swiftContainerStoragePolicy, err = confMap.FetchOptionValueString(workerSectionName, "SwiftContainerStoragePolicy") - if nil != err { - log.Fatalf("Fetching %s.SwiftContainerStoragePolicy failed: %v", workerSectionName, err) - } - worker.swiftObjectPrefix, err = confMap.FetchOptionValueString(workerSectionName, "SwiftObjectPrefix") - if nil != err { - worker.swiftObjectPrefix = "" - } - worker.objectSize, err = confMap.FetchOptionValueUint64(workerSectionName, "ObjectSize") - if nil != err { - log.Fatalf("Fetching %s.ObjectSize failed: %v", workerSectionName, err) - } - - if methodListIncludesSwiftPut { - worker.swiftPutBuffer = make([]byte, worker.objectSize) - rand.Read(worker.swiftPutBuffer) - } - } - - worker.iterations, err = confMap.FetchOptionValueUint64(workerSectionName, "Iterations") - if nil != err { - log.Fatalf("Fetching %s.Iterations failed: %v", workerSectionName, err) - } - worker.numThreads, err = confMap.FetchOptionValueUint64(workerSectionName, "NumThreads") - if nil != err { - log.Fatalf("Fetching %s.NumThreads failed: %v", workerSectionName, err) - } - - workerArray = append(workerArray, worker) - - if globals.optionDestroy && !globals.optionFormat { - globals.workersDoneGate.Add(1) - } else { - globals.workersReadyGate.Add(1) - } - - go worker.workerLauncher() - } - - if globals.optionDestroy && !globals.optionFormat { - // Wait for optionDestroy to be completed and exit - globals.workersDoneGate.Wait() - os.Exit(0) - } - - // Wait for all workers to be ready - - globals.workersReadyGate.Wait() - - // Print column heads - - fmt.Printf(" ElapsedTime ") - for _, worker = range workerArray { - fmt.Printf(" %-20s", worker.name) - } - fmt.Println() - - // Log column heads - - logHeaderLine = "" - for _, worker = range workerArray { - for range columnTitles { - if "" == logHeaderLine { - logHeaderLine = worker.name - } else { - logHeaderLine += "," + worker.name - } - } - } - _, _ = globals.logFile.WriteString(logHeaderLine + "\n") - - logHeaderLine = "" - for _, worker = range workerArray { - if "" == logHeaderLine { - logHeaderLine = strings.Join(columnTitles, ",") - } else { - logHeaderLine += "," + strings.Join(columnTitles, ",") - } - } - _, _ = globals.logFile.WriteString(logHeaderLine + "\n") - - // Kick off workers (and their threads) - - globals.workersGoGate.Done() - - // Display ongoing results until globals.liveThreadCount == 0 - - globals.elapsedTime = time.Duration(0) - - for { - time.Sleep(displayInterval) - - globals.elapsedTime += displayInterval - - workersIntervals = make([]workerIntervalValues, 0, len(workerArray)) - - for _, worker = range workerArray { - worker.Lock() - iterationsCompleted = worker.iterationsCompleted - iterationsDelta = iterationsCompleted - worker.priorIterationsCompleted - worker.priorIterationsCompleted = iterationsCompleted - worker.Unlock() - - workersIntervals = append(workersIntervals, workerIntervalValues{iterationsCompleted, iterationsDelta}) - } - - printPlainTextInterval(workersIntervals) - logCSVInterval(workersIntervals) - - globals.Lock() - if 0 == globals.liveThreadCount { - globals.Unlock() - // All threads exited, so just exit (indicating success if all worker iterations succeeded) - for _, worker = range workerArray { - if worker.iterationsCompleted != worker.iterations { - os.Exit(1) - } - } - os.Exit(0) - } - globals.Unlock() - } -} - -func printPlainTextInterval(workersIntervals []workerIntervalValues) { - var ( - elapsedTimeString string - iterationsCompletedString string - iterationsDeltaString string - workerInterval workerIntervalValues - ) - - elapsedTimeString = fmt.Sprintf("%v", globals.elapsedTime) - - fmt.Printf("%10s", elapsedTimeString) - - for _, workerInterval = range workersIntervals { - iterationsCompletedString = fmt.Sprintf("%d", workerInterval.iterationsCompleted) - iterationsDeltaString = fmt.Sprintf("(+%d)", workerInterval.iterationsDelta) - - fmt.Printf(" %12s %-10s", iterationsCompletedString, iterationsDeltaString) - } - - fmt.Println() -} - -func logCSVInterval(workersIntervals []workerIntervalValues) { - var ( - csvString string - workerInterval workerIntervalValues - workerIntervalStrings []string - ) - - for _, workerInterval = range workersIntervals { - csvString = fmt.Sprintf("%v,%v,%v", globals.elapsedTime.Seconds(), workerInterval.iterationsCompleted, workerInterval.iterationsDelta) - workerIntervalStrings = append(workerIntervalStrings, csvString) - } - - _, _ = globals.logFile.WriteString(strings.Join(workerIntervalStrings, ",") + "\n") -} - -func (worker *workerStruct) fetchNextIteration() (iteration uint64, allDone bool) { - worker.Lock() - if worker.nextIteration < worker.iterations { - iteration = worker.nextIteration - worker.nextIteration = iteration + 1 - allDone = false - } else { - allDone = true - } - worker.Unlock() - return -} - -func (worker *workerStruct) incIterationsCompleted() { - worker.Lock() - worker.iterationsCompleted++ - worker.Unlock() -} - -func (worker *workerStruct) workerLauncher() { - var ( - containerDeleteRequest *http.Request - containerDeleteResponse *http.Response - containerGetBody []byte - containerGetRequest *http.Request - containerGetResponse *http.Response - containerPutRequest *http.Request - containerPutResponse *http.Response - containerURL string - directoryPath string - err error - httpClient *http.Client - objectDeleteRequest *http.Request - objectDeleteResponse *http.Response - objectList []string - objectListIndex int - objectListLen int - objectListReversed []string - objectName string - objectURL string - subDirectoryFullPath string - swiftAuthToken string - threadIndex uint64 - ) - - if worker.methodListIncludesFileMethod { - directoryPath = worker.mountPoint + "/" + worker.directory - if "" == worker.subDirectoryPath { - subDirectoryFullPath = directoryPath - } else { - subDirectoryFullPath = directoryPath + "/" + worker.subDirectoryPath - } - } - if worker.methodListIncludesSwiftMethod { - worker.authorizationsPerformed = 0 - - worker.updateSwiftAuthToken() - - httpClient = &http.Client{} - - containerURL = fmt.Sprintf("%sv1/%s/%s", worker.swiftProxyURL, worker.swiftAccount, worker.swiftContainer) - } - - if globals.optionDestroy { - if worker.methodListIncludesFileMethod { - err = os.RemoveAll(directoryPath) - if nil != err { - log.Fatalf("os.RemoveAll(\"%s\") failed: %v", directoryPath, err) - } - } else { // worker.methodListIncludesSwiftMethod must be true - // First, empty the containerURL - - optionDestroyLoop: - for { - swiftAuthToken = worker.fetchSwiftAuthToken() - containerGetRequest, err = http.NewRequest("GET", containerURL, nil) - if nil != err { - log.Fatalf("http.NewRequest(\"GET\", \"%s\", nil) failed: %v", containerURL, err) - } - containerGetRequest.Header.Set("X-Auth-Token", swiftAuthToken) - containerGetResponse, err = httpClient.Do(containerGetRequest) - if nil != err { - log.Fatalf("httpClient.Do(containerGetRequest) failed: %v", err) - } - containerGetBody, err = ioutil.ReadAll(containerGetResponse.Body) - containerGetResponse.Body.Close() - if nil != err { - log.Fatalf("ioutil.ReadAll(containerGetResponse.Body) failed: %v", err) - } - - switch containerGetResponse.StatusCode { - case http.StatusOK: - objectList = strings.Split(string(containerGetBody[:]), "\n") - - objectListLen = len(objectList) - - if (0 < objectListLen) && ("" == objectList[objectListLen-1]) { - objectList = objectList[:objectListLen-1] - objectListLen-- - } - - if 0 == objectListLen { - break optionDestroyLoop - } - - objectListReversed = make([]string, len(objectList)) - - for objectListIndex, objectName = range objectList { - objectListReversed[objectListLen-objectListIndex-1] = objectName - } - - for _, objectName = range objectListReversed { - objectURL = containerURL + "/" + objectName - - objectDeleteRetryLoop: - for { - swiftAuthToken = worker.fetchSwiftAuthToken() - objectDeleteRequest, err = http.NewRequest("DELETE", objectURL, nil) - if nil != err { - log.Fatalf("http.NewRequest(\"DELETE\", \"%s\", nil) failed: %v", objectURL, err) - } - objectDeleteRequest.Header.Set("X-Auth-Token", swiftAuthToken) - objectDeleteResponse, err = httpClient.Do(objectDeleteRequest) - if nil != err { - log.Fatalf("httpClient.Do(objectDeleteRequest) failed: %v", err) - } - _, err = ioutil.ReadAll(objectDeleteResponse.Body) - objectDeleteResponse.Body.Close() - if nil != err { - log.Fatalf("ioutil.ReadAll(objectDeleteResponse.Body) failed: %v", err) - } - - switch objectDeleteResponse.StatusCode { - case http.StatusNoContent: - break objectDeleteRetryLoop - case http.StatusNotFound: - // Object apparently already deleted... Container just doesn't know it yet - break objectDeleteRetryLoop - case http.StatusUnauthorized: - worker.updateSwiftAuthToken() - default: - log.Fatalf("DELETE response had unexpected status: %s (%d)", objectDeleteResponse.Status, objectDeleteResponse.StatusCode) - } - } - } - case http.StatusNoContent: - break optionDestroyLoop - case http.StatusNotFound: - // Container apparently already deleted... - break optionDestroyLoop - default: - log.Fatalf("GET response had unexpected status: %s (%d)", containerGetResponse.Status, containerGetResponse.StatusCode) - } - } - - // Now, delete containerURL - - containerDeleteRetryLoop: - for { - swiftAuthToken = worker.fetchSwiftAuthToken() - containerDeleteRequest, err = http.NewRequest("DELETE", containerURL, nil) - if nil != err { - log.Fatalf("http.NewRequest(\"DELETE\", \"%s\", nil) failed: %v", containerURL, err) - } - containerDeleteRequest.Header.Set("X-Auth-Token", swiftAuthToken) - containerDeleteResponse, err = httpClient.Do(containerDeleteRequest) - if nil != err { - log.Fatalf("httpClient.Do(containerDeleteRequest) failed: %v", err) - } - _, err = ioutil.ReadAll(containerDeleteResponse.Body) - containerDeleteResponse.Body.Close() - if nil != err { - log.Fatalf("ioutil.ReadAll(containerDeleteResponse.Body) failed: %v", err) - } - - switch containerDeleteResponse.StatusCode { - case http.StatusNoContent: - break containerDeleteRetryLoop - case http.StatusNotFound: - // Container apparently already deleted... - break containerDeleteRetryLoop - case http.StatusUnauthorized: - worker.updateSwiftAuthToken() - default: - log.Fatalf("DELETE response had unexpected status: %s (%d)", containerDeleteResponse.Status, containerDeleteResponse.StatusCode) - } - } - } - - if !globals.optionFormat { - // We are just going to exit if all we were asked to do was optionDestroy - globals.workersDoneGate.Done() - return - } - } - - if globals.optionFormat { - if worker.methodListIncludesFileMethod { - err = os.MkdirAll(subDirectoryFullPath, os.ModePerm) - if nil != err { - log.Fatalf("os.MkdirAll(\"%s\", os.ModePerm) failed: %v", subDirectoryFullPath, err) - } - } else { //worker.methodListIncludesSwiftMethod must be true - containerPutRetryLoop: - for { - swiftAuthToken = worker.fetchSwiftAuthToken() - containerPutRequest, err = http.NewRequest("PUT", containerURL, nil) - if nil != err { - log.Fatalf("http.NewRequest(\"PUT\", \"%s\", nil) failed: %v", containerURL, err) - } - containerPutRequest.Header.Set("X-Auth-Token", swiftAuthToken) - containerPutRequest.Header.Set("X-Storage-Policy", worker.swiftContainerStoragePolicy) - containerPutResponse, err = httpClient.Do(containerPutRequest) - if nil != err { - log.Fatalf("httpClient.Do(containerPutRequest) failed: %v", err) - } - _, err = ioutil.ReadAll(containerPutResponse.Body) - containerPutResponse.Body.Close() - if nil != err { - log.Fatalf("ioutil.ReadAll(containerPutResponse.Body) failed: %v", err) - } - - switch containerPutResponse.StatusCode { - case http.StatusCreated: - break containerPutRetryLoop - case http.StatusNoContent: - worker.updateSwiftAuthToken() - default: - log.Fatalf("PUT response had unexpected status: %s (%d)", containerPutResponse.Status, containerPutResponse.StatusCode) - } - } - } - } - - worker.nextIteration = 0 - worker.iterationsCompleted = 0 - worker.priorIterationsCompleted = 0 - - for threadIndex = 0; threadIndex < worker.numThreads; threadIndex++ { - globals.workersReadyGate.Add(1) - - go worker.workerThreadLauncher() - } - - globals.workersReadyGate.Done() -} - -func (worker *workerStruct) workerThreadLauncher() { - var ( - abandonedIteration bool - allDone bool - doSwiftMethod bool - err error - file *os.File - fileBlockIndex uint64 - fileReadAt uint64 - fileReadBuffer []byte - httpClient *http.Client - iteration uint64 - method string - methodRequest *http.Request - methodResponse *http.Response - path string - subDirectoryFullPath string - swiftAuthToken string - swiftPutReader *bytes.Reader - url string - ) - - globals.Lock() - globals.liveThreadCount++ - globals.Unlock() - - if worker.methodListIncludesFileMethod { - if "" == worker.subDirectoryPath { - subDirectoryFullPath = worker.mountPoint + "/" + worker.directory - } else { - subDirectoryFullPath = worker.mountPoint + "/" + worker.directory + "/" + worker.subDirectoryPath - } - fileReadBuffer = make([]byte, worker.fileBlockSize) - } - if worker.methodListIncludesSwiftMethod { - httpClient = &http.Client{} - } - - globals.workersReadyGate.Done() - - globals.workersGoGate.Wait() - - defer func() { - globals.Lock() - globals.liveThreadCount-- - globals.Unlock() - }() - - iteration, allDone = worker.fetchNextIteration() - if allDone { - return - } - - for { - if worker.methodListIncludesFileMethod { - path = fmt.Sprintf("%s/%016X", subDirectoryFullPath, iteration) - } - if worker.methodListIncludesSwiftMethod { - url = fmt.Sprintf("%sv1/%s/%s/%s%016X", worker.swiftProxyURL, worker.swiftAccount, worker.swiftContainer, worker.swiftObjectPrefix, iteration) - } - - abandonedIteration = false - - for _, method = range worker.methodList { - if !abandonedIteration { - switch method { - case fileWrite: - doSwiftMethod = false - file, err = os.Create(path) - if nil == err { - fileBlockIndex = 0 - for (nil == err) && (fileBlockIndex < worker.fileBlockCount) { - _, err = file.Write(worker.fileWriteBuffer) - fileBlockIndex++ - } - if nil == err { - err = file.Sync() - if nil == err { - err = file.Close() - if nil != err { - abandonedIteration = true - } - } else { - abandonedIteration = true - _ = file.Close() - } - } else { - abandonedIteration = true - _ = file.Close() - } - } else { - abandonedIteration = true - } - case fileStat: - doSwiftMethod = false - file, err = os.Open(path) - if nil == err { - _, err = file.Stat() - if nil == err { - err = file.Close() - if nil != err { - abandonedIteration = true - } - } else { - abandonedIteration = true - _ = file.Close() - } - } else { - abandonedIteration = true - } - case fileRead: - doSwiftMethod = false - file, err = os.Open(path) - if nil == err { - fileBlockIndex = 0 - fileReadAt = 0 - for (nil == err) && (fileBlockIndex < worker.fileBlockCount) { - _, err = file.ReadAt(fileReadBuffer, int64(fileReadAt)) - fileBlockIndex++ - fileReadAt += worker.fileBlockSize - } - if nil == err { - err = file.Close() - if nil != err { - abandonedIteration = true - } - } else { - abandonedIteration = true - _ = file.Close() - } - } else { - abandonedIteration = true - } - case fileDelete: - doSwiftMethod = false - err = os.Remove(path) - if nil != err { - abandonedIteration = true - } - case http.MethodPut: - doSwiftMethod = true - case http.MethodHead: - doSwiftMethod = true - case http.MethodGet: - doSwiftMethod = true - case http.MethodDelete: - doSwiftMethod = true - default: - log.Fatalf("Method %s in Worker:%s.MethodList not supported", method, worker.name) - } - - for doSwiftMethod { - swiftAuthToken = worker.fetchSwiftAuthToken() - - switch method { - case http.MethodPut: - swiftPutReader = bytes.NewReader(worker.swiftPutBuffer) - methodRequest, err = http.NewRequest("PUT", url, swiftPutReader) - if nil != err { - log.Fatalf("http.NewRequest(\"PUT\", \"%s\", swiftPutReader) failed: %v", url, err) - } - case http.MethodHead: - methodRequest, err = http.NewRequest("HEAD", url, nil) - if nil != err { - log.Fatalf("http.NewRequest(\"HEAD\", \"%s\", nil) failed: %v", url, err) - } - case http.MethodGet: - methodRequest, err = http.NewRequest("GET", url, nil) - if nil != err { - log.Fatalf("http.NewRequest(\"GET\", \"%s\", nil) failed: %v", url, err) - } - case http.MethodDelete: - methodRequest, err = http.NewRequest("DELETE", url, nil) - if nil != err { - log.Fatalf("http.NewRequest(\"DELETE\", \"%s\", nil) failed: %v", url, err) - } - default: - log.Fatalf("Method %s in Worker:%s.MethodList not supported", method, worker.name) - } - - methodRequest.Header.Set("X-Auth-Token", swiftAuthToken) - methodResponse, err = httpClient.Do(methodRequest) - if nil != err { - log.Fatalf("httpClient.Do(methodRequest) failed: %v", err) - } - _, err = ioutil.ReadAll(methodResponse.Body) - methodResponse.Body.Close() - if nil != err { - log.Fatalf("ioutil.ReadAll(methodResponse.Body) failed: %v", err) - } - switch methodResponse.StatusCode { - case http.StatusOK: - doSwiftMethod = false - case http.StatusNoContent: - doSwiftMethod = false - case http.StatusNotFound: - doSwiftMethod = false - abandonedIteration = true - case http.StatusCreated: - doSwiftMethod = false - case http.StatusUnauthorized: - worker.updateSwiftAuthToken() - default: - log.Fatalf("%s response for url \"%s\" had unexpected status: %s (%d)", method, url, methodResponse.Status, methodResponse.StatusCode) - } - } - } - } - - if !abandonedIteration { - worker.incIterationsCompleted() - } - - iteration, allDone = worker.fetchNextIteration() - if allDone { - return - } - } -} - -func (worker *workerStruct) fetchSwiftAuthToken() (swiftAuthToken string) { - var ( - swiftAuthTokenUpdateWaitGroup *sync.WaitGroup - ) - - for { - worker.Lock() - - swiftAuthTokenUpdateWaitGroup = worker.swiftAuthTokenUpdateWaitGroup - - if nil == swiftAuthTokenUpdateWaitGroup { - swiftAuthToken = worker.swiftAuthToken - worker.Unlock() - return - } - - worker.Unlock() - - swiftAuthTokenUpdateWaitGroup.Wait() - } -} - -func (worker *workerStruct) updateSwiftAuthToken() { - var ( - err error - getRequest *http.Request - getResponse *http.Response - httpClient *http.Client - ) - - worker.Lock() - - if nil != worker.swiftAuthTokenUpdateWaitGroup { - worker.Unlock() - - _ = worker.fetchSwiftAuthToken() - - return - } - - worker.swiftAuthTokenUpdateWaitGroup = &sync.WaitGroup{} - worker.swiftAuthTokenUpdateWaitGroup.Add(1) - - worker.Unlock() - - getRequest, err = http.NewRequest("GET", worker.swiftProxyURL+"auth/v1.0", nil) - if nil != err { - log.Fatalf("http.NewRequest(\"GET\", \"%sauth/v1.0\", nil) failed: %v", worker.swiftProxyURL, err) - } - - getRequest.Header.Set("X-Auth-User", worker.swiftAuthUser) - getRequest.Header.Set("X-Auth-Key", worker.swiftAuthKey) - - httpClient = &http.Client{} - - getResponse, err = httpClient.Do(getRequest) - if nil != err { - log.Fatalf("httpClient.Do(getRequest) failed: %v", err) - } - _, err = ioutil.ReadAll(getResponse.Body) - getResponse.Body.Close() - if nil != err { - log.Fatalf("ioutil.ReadAll(getResponse.Body) failed: %v", err) - } - - worker.Lock() - - worker.swiftAuthToken = getResponse.Header.Get("X-Auth-Token") - - worker.authorizationsPerformed++ - - worker.swiftAuthTokenUpdateWaitGroup.Done() - - worker.swiftAuthTokenUpdateWaitGroup = nil - - worker.Unlock() -} diff --git a/pfs-swift-load/pfs-swift-load-plot b/pfs-swift-load/pfs-swift-load-plot deleted file mode 100755 index 992c6f32..00000000 --- a/pfs-swift-load/pfs-swift-load-plot +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import itertools -import matplotlib -import pandas as pd -import sys - -matplotlib.use("Agg") -import matplotlib.pyplot as plt - - -ELAPSED_TIME_COL = "ElapsedTime" -ELAPSED_TIME_TEXT = "Elapsed Time (s)" -COMPLETED_COL = "Completed" -COMPLETED_TEXT = "Completed" -DELTA_COL = "Delta" -DELTA_TEXT = "Delta" -OBJECTS_PER_SECOND_COL = "ObjectsPerSecond" -OBJECTS_PER_SECOND_TEXT = "Objects/Second" -PLOT_COLORS = ["r", "g", "b", "c", "m", "y", "k"] -PLOT_COLS = 1 -PLOT_ROWS = 2 -FIG_SIZE = (18, 12) - - -def plot(plot_position, x_axis_data, x_axis_label, warning_msg=None): - color_iterator = itertools.cycle(PLOT_COLORS) - - ax = fig.add_subplot(PLOT_ROWS, PLOT_COLS, plot_position) - for worker in workers: - ax.plot( - x_axis_data, - data_frame[worker][OBJECTS_PER_SECOND_COL], - color=next(color_iterator), - label=worker, - ) - ax.set_xlabel(x_axis_label, color="k") - ax.set_ylabel(OBJECTS_PER_SECOND_TEXT, color="k") - ax.grid(color="tab:gray", linestyle="dashdot", linewidth=0.4) - plt.legend() - if warning_msg: - plt.title(warning_msg, bbox=dict(facecolor="red", alpha=0.5)) - - -def parse_args(): - parser = argparse.ArgumentParser(description="Plot results from " "pfs-swift-load") - parser.add_argument("-i", "--input", type=str, required=True, help="Input CSV file") - parser.add_argument( - "-o", "--output", type=str, required=True, help="Output PNG file" - ) - parser.add_argument( - "-w", - "--warning", - type=str, - required=False, - default=None, - help="Warning message to print on the " "PNG file", - ) - args = parser.parse_args() - options = { - "input": args.input, - "output": args.output, - "warning": args.warning, - } - return options - - -if __name__ == "__main__": - options = parse_args() - - data_frame = pd.read_csv(options["input"], header=[0, 1]) - workers = list(data_frame.keys().levels[0]) - # We only have len(PLOT_COLORS) colors to use, and we don't want 2 workers - # to be plotted with the same color... - if len(workers) > len(PLOT_COLORS): - print("Too many workers: {}, max: {}".format(len(workers), len(PLOT_COLORS))) - sys.exit(1) - - # Check how long are the intervals between samples - interval = data_frame[workers[0]].ElapsedTime[0] - # We want an extra column for objects/second. Warning! It doesn't take into - # account the fact that several threads could have been working in - # parallel. Maybe we could add an option? - for worker in workers: - data_frame[worker, OBJECTS_PER_SECOND_COL] = ( - data_frame[worker][DELTA_COL] / interval - ) - data_frame = data_frame.sort_index(axis=1) - - # Draw the chart(s) - fig = plt.figure(figsize=FIG_SIZE) - plot(1, data_frame[worker][ELAPSED_TIME_COL], ELAPSED_TIME_TEXT, options["warning"]) - plot(2, data_frame[worker][COMPLETED_COL], COMPLETED_TEXT) - plt.savefig(options["output"]) diff --git a/pfs-swift-load/pfs-swift-load.conf b/pfs-swift-load/pfs-swift-load.conf deleted file mode 100644 index 53bfaaf1..00000000 --- a/pfs-swift-load/pfs-swift-load.conf +++ /dev/null @@ -1,49 +0,0 @@ -[LoadParameters] - -WorkerList: file bimodal pipeline # Must be non-empty... example file, bimodal, and pipeline workers given below -DisplayInterval: 1s -LogPath: /dev/null # Log will be in .csv format - -[Worker:file] - -MethodList: write # One of write, stat, read, or delete -MountPoint: /CommonMountPoint -Directory: pfs-swift-load-file -SubDirectoryPath: # Optional - trailing slash will be automatically appended if missing -FileBlockCount: 256 -FileBlockSize: 4096 -Iterations: 1000 -NumThreads: 10 - -[Worker:bimodal] - -MethodList: PUT # One of PUT, HEAD, GET, or DELETE -SwiftProxyURL: http://127.0.0.1:8080/ -SwiftAuthUser: test:tester -SwiftAuthKey: testing -SwiftAccount: AUTH_test -SwiftContainer: pfs-swift-load-bimodal -SwiftContainerStoragePolicy: silver -SwiftObjectPrefix: # Optional -ObjectSize: 1048576 -Iterations: 1000 -NumThreads: 10 - -[Worker:pipeline] - -MethodList: write GET DELETE # A non-empty list of write, stat, read, delete, PUT, HEAD, GET, and DELETE -MountPoint: /CommonMountPoint # Typically corresponds to SwiftAccount -Directory: pfs-swift-load-pipeline # Typically matches SwiftContainer -SubDirectoryPath: # Typically matches SwiftObjectPrefix -FileBlockCount: 256 # Typically, -FileBlockSize: 4096 # (FileBlockCount * FileBlockSize) == ObjectSize -SwiftProxyURL: http://127.0.0.1:8080/ -SwiftAuthUser: test:tester -SwiftAuthKey: testing -SwiftAccount: AUTH_test # Typically corresponds to MountPoint -SwiftContainer: pfs-swift-load-pipeline # Typically matches Directory -SwiftContainerStoragePolicy: silver -SwiftObjectPrefix: # Typically matches SubDirectoryPath -ObjectSize: 1048576 # Typically matches (FileBlockCount * FileBlockSize) -Iterations: 1000 -NumThreads: 10 diff --git a/pfs-swift-load/requirements.txt b/pfs-swift-load/requirements.txt deleted file mode 100644 index 122d84fc..00000000 --- a/pfs-swift-load/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy==1.14.5 -pandas==0.23.3 -matplotlib==2.2.2 diff --git a/pfs_middleware/pfs_middleware/__init__.py b/pfs_middleware/pfs_middleware/__init__.py deleted file mode 100644 index 8bd903f7..00000000 --- a/pfs_middleware/pfs_middleware/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -from .bimodal_checker import BimodalChecker # flake8: noqa -from .middleware import PfsMiddleware # flake8: noqa -from .s3_compat import S3Compat # flake8: noqa -from swift.common.utils import register_swift_info - - -def filter_factory(global_conf, **local_conf): - conf = global_conf.copy() - conf.update(local_conf) - - register_swift_info('pfs', - bypass_mode=conf.get('bypass_mode', 'off')) - - def pfs_filter(app): - return BimodalChecker(S3Compat(PfsMiddleware(app, conf), conf), conf) - - return pfs_filter diff --git a/pfs_middleware/pfs_middleware/bimodal_checker.py b/pfs_middleware/pfs_middleware/bimodal_checker.py deleted file mode 100644 index ff17ca13..00000000 --- a/pfs_middleware/pfs_middleware/bimodal_checker.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -""" -Middleware to check whether a given Swift account corresponds to a -ProxyFS volume or not. - -This is part of the ProxyFS suite of middlewares needed to enable Swift-API -access to ProxyFS volumes. It doesn't have any user-visible functionality on -its own, but it is required for the other middlewares to work. -""" - -import random -import socket -import time - -from swift.common import swob, constraints -from swift.common.utils import get_logger -from swift.proxy.controllers.base import get_account_info - -from . import rpc, utils, swift_code - - -# full header is "X-Account-Sysmeta-Proxyfs-Bimodal" -SYSMETA_BIMODAL_INDICATOR = 'proxyfs-bimodal' - -RPC_FINDER_TIMEOUT_DEFAULT = 3.0 - - -class BimodalChecker(object): - def __init__(self, app, conf, logger=None): - self.app = app - self.conf = conf - self._cached_is_bimodal = {} - self.logger = logger or get_logger( - conf, log_route='pfs.bimodal_checker') - - proxyfsd_hosts = [h.strip() for h - in conf.get('proxyfsd_host', '127.0.0.1').split(',')] - self.proxyfsd_port = int(conf.get('proxyfsd_port', '12345')) - - self.proxyfsd_addrinfos = set() - for host in proxyfsd_hosts: - try: - # If hostname resolution fails, we'll cause the proxy to - # fail to start. This is probably better than returning 500 - # to every single request, but maybe not. - # - # To ensure that proxy startup works, use IP addresses for - # proxyfsd_host. Then socket.getaddrinfo() will always work. - addrinfo = socket.getaddrinfo( - host, self.proxyfsd_port, - socket.AF_UNSPEC, socket.SOCK_STREAM)[0] - self.proxyfsd_addrinfos.add(addrinfo) - except socket.gaierror: - self.logger.error("Error resolving hostname %r", host) - raise - - self.proxyfsd_rpc_timeout = float(conf.get('rpc_finder_timeout', - RPC_FINDER_TIMEOUT_DEFAULT)) - self.bimodal_recheck_interval = float(conf.get( - 'bimodal_recheck_interval', '60.0')) - - def _rpc_call(self, addrinfos, rpc_request): - addrinfos = list(addrinfos) - random.shuffle(addrinfos) - - # We can get fast errors or slow errors here; we retry across all - # hosts on fast errors, but immediately raise a slow error. HTTP - # clients won't wait forever for a response, so we can't retry slow - # errors across all hosts. - # - # Fast errors are things like "connection refused" or "no route to - # host". Slow errors are timeouts. - result = None - while addrinfos: - addrinfo = addrinfos.pop() - rpc_client = utils.JsonRpcClient(addrinfo) - try: - result = rpc_client.call(rpc_request, - self.proxyfsd_rpc_timeout) - except socket.error as err: - if addrinfos: - self.logger.debug("Error communicating with %r: %s. " - "Trying again with another host.", - addrinfo, err) - continue - else: - raise - - errstr = result.get("error") - if errstr: - errno = utils.extract_errno(errstr) - raise utils.RpcError(errno, errstr) - - return result["result"] - - def _fetch_owning_proxyfs(self, req, account_name): - """ - Checks to see if an account is bimodal or not, and if so, which proxyfs - daemon owns it. Performs any necessary DNS resolution on the owner's - address, raising an error if the owner is an unresolvable hostname. - - Will check a local cache first, falling back to a ProxyFS RPC call - if necessary. - - Results are cached in memory locally, not memcached. ProxyFS has an - in-memory list (or whatever data structure it uses) of known - accounts, so it can answer the RPC request very quickly. Retrieving - that result from memcached wouldn't be any faster than asking - ProxyFS. - - :returns: 2-tuple (is-bimodal, proxyfsd-addrinfo). - - :raises utils.NoSuchHostnameError: if owning proxyfsd is - unresolvable - - :raises utils.RpcTimeout: if proxyfsd is too slow in responding - - :raises utils.RpcError: if proxyfsd's response indicates an error - """ - cached_result = self._cached_is_bimodal.get(account_name) - if cached_result: - res, res_time = cached_result - if res_time + self.bimodal_recheck_interval >= time.time(): - # cache is populated and fresh; use it - return res - - # First, ask Swift if the account is bimodal. This lets us keep - # non-ProxyFS accounts functional during a ProxyFS outage. - env_copy = req.environ.copy() - if env_copy["PATH_INFO"].startswith("/proxyfs/"): - env_copy["PATH_INFO"] = env_copy["PATH_INFO"].replace( - "/proxyfs/", "/v1/", 1) - env_copy[utils.ENV_IS_BIMODAL] = False - account_info = get_account_info(env_copy, self.app, - swift_source="PFS") - if not swift_code.config_true_value(account_info["sysmeta"].get( - SYSMETA_BIMODAL_INDICATOR)): - res = (False, None) - self._cached_is_bimodal[account_name] = (res, time.time()) - return res - - # We know where one proxyfsd is, and they'll all respond identically - # to the same query. - iab_req = rpc.is_account_bimodal_request(account_name) - is_bimodal, proxyfsd_ip_or_hostname = \ - rpc.parse_is_account_bimodal_response( - self._rpc_call(self.proxyfsd_addrinfos, iab_req)) - - if not is_bimodal: - # Swift account says bimodal, ProxyFS says otherwise. - raise swob.HTTPServiceUnavailable( - request=req, - headers={"Content-Type": "text/plain"}, - body=("The Swift account says it has bimodal access, but " - "ProxyFS disagrees. Unable to proceed.")) - - if not proxyfsd_ip_or_hostname: - # When an account is moving between proxyfsd nodes, it's - # bimodal but nobody owns it. This is usually a - # very-short-lived temporary condition, so we don't cache - # this result. - return (True, None) - - # Just run whatever we got through socket.getaddrinfo(). If we - # got an IPv4 or IPv6 address, it'll come out the other side - # unchanged. If we got a hostname, it'll get resolved. - try: - # Someday, ProxyFS will probably start giving us port - # numbers, too. Until then, assume they all use the same - # port. - addrinfos = socket.getaddrinfo( - proxyfsd_ip_or_hostname, self.proxyfsd_port, - socket.AF_UNSPEC, socket.SOCK_STREAM) - except socket.gaierror: - raise utils.NoSuchHostnameError( - "Owning ProxyFS is at %s, but that could not " - "be resolved to an IP address" % ( - proxyfsd_ip_or_hostname)) - - # socket.getaddrinfo returns things already sorted according - # to the various rules in RFC 3484, so instead of thinking - # we're smarter than Python and glibc, we'll just take the - # first (i.e. best) one. - # - # Since we didn't get an exception, we resolved the hostname - # to *something*, which means there's at least one element - # in addrinfos. - res = (True, addrinfos[0]) - self._cached_is_bimodal[account_name] = (res, time.time()) - return res - - @swob.wsgify - def __call__(self, req): - vrs, acc, con, obj = utils.parse_path(req.path) - - if not acc or not (constraints.valid_api_version(vrs) or - vrs == 'proxyfs'): - # could be a GET /info request or something made up by some - # other middleware; get out of the way. - return self.app - - try: - is_bimodal, owner_addrinfo = self._fetch_owning_proxyfs(req, acc) - except (utils.RpcError, utils.RpcTimeout) as err: - return swob.HTTPServiceUnavailable( - request=req, - headers={'Content-Type': 'text/plain'}, - body=str(err)) - - # Other middlewares will find and act on this - req.environ[utils.ENV_IS_BIMODAL] = is_bimodal - req.environ[utils.ENV_OWNING_PROXYFS] = owner_addrinfo - req.environ[utils.ENV_BIMODAL_CHECKER] = self - - return self.app diff --git a/pfs_middleware/pfs_middleware/middleware.py b/pfs_middleware/pfs_middleware/middleware.py deleted file mode 100644 index f95d2b28..00000000 --- a/pfs_middleware/pfs_middleware/middleware.py +++ /dev/null @@ -1,2174 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -""" -Middleware that will provide a Swift-ish API for a ProxyFS account. - -It tries to mimic the Swift API as much as possible, but there are some -differences that must be called out. - -* ETags are sometimes different. In Swift, an object's ETag is the MD5 - checksum of its contents. With this middleware, an object's ETag is - sometimes an opaque value sufficient to provide a strong identifier as per - RFC 7231, but it is not the MD5 checksum of the object's contents. - - ETags start out as MD5 checksums, but if files are subsequently modified - via SMB or NFS, the ETags become opaque values. - -* Container listings lack object count. To get an object count, it would be - necessary to traverse the entire directory structure underneath the - container directory. This would be unbearably slow and resource-intensive. - -* Account HEAD responses contain a header "ProxyFS-Enabled: yes". This way, - clients can know what they're dealing with. - -* Support for the COALESCE verb. Since static large objects will not work - with this middleware's ETag values, another solution was found. A client - can combine (or coalesce, if you will) a number of small files together - into one large file, allowing for parallel uploads like a client can get - with static large objects. - - Once the small files are uploaded, one makes a COALESCE request to the - destination path. The request body is a JSON object containing a key - "elements" whose value is a list of account-relative object names. - - Example: - - COALESCE /v1/AUTH_me/c/rainbow HTTP/1.1 - X-Auth-Token: t - ... more headers ... - - { - "elements": [ - "/c/red", - "/c/orange", - "/c/yellow", - "/c/green", - "/c/blue", - "/c/indigo", - "/c/violet" - ] - } - - This will combine the files /c/red, ..., /c/violet into a single file - /c/rainbow. The files /c/red et cetera will be deleted as a result of this - request. -""" -import contextlib -import datetime -import eventlet -import eventlet.queue -import hashlib -import itertools -import json -import math -import mimetypes -import re -import six -import socket -import time -import uuid -import xml.etree.ElementTree as ET -from six.moves.urllib import parse as urllib_parse -from io import BytesIO - -from swift.common.middleware.acl import parse_acl, format_acl -from . import pfs_errno, rpc, swift_code, utils - -# Generally speaking, let's try to keep the use of Swift code to a -# reasonable level. Using dozens of random functions from swift.common.utils -# will ensure that this middleware breaks with every new Swift release. On -# the other hand, there's some really good, actually-works-in-production -# code in Swift that does things we need. - -# Were we to make an account HEAD request instead of calling -# get_account_info, we'd lose the benefit of Swift's caching. This would -# slow down requests *a lot*. Same for containers. -from swift.proxy.controllers.base import ( - get_account_info, get_container_info, clear_info_cache) - -# Plain WSGI is annoying to work with, and nobody wants a dependency on -# webob. -from swift.common import swob, constraints - -# Our logs should go to the same place as everyone else's. Plus, this logger -# works well in an eventlet-ified process, and SegmentedIterable needs one. -from swift.common.utils import config_true_value, get_logger, Timestamp - - -# POSIX file-path limits. Taken from Linux's limits.h, which is also where -# ProxyFS gets them. -NAME_MAX = 255 -PATH_MAX = 4096 - -# Used for content type of directories in container listings -DIRECTORY_CONTENT_TYPE = "application/directory" - -ZERO_FILL_PATH = "/0" - -LEASE_RENEWAL_INTERVAL = 5 # seconds - -# Beware: ORIGINAL_MD5_HEADER is random case, not title case, but is -# stored on-disk just as defined here. Care must be taken when comparing -# it to incoming headers which are title case. -ORIGINAL_MD5_HEADER = "X-Object-Sysmeta-ProxyFS-Initial-MD5" -S3API_ETAG_HEADER = "X-Object-Sysmeta-S3Api-Etag" -LISTING_ETAG_OVERRIDE_HEADER = \ - "X-Object-Sysmeta-Container-Update-Override-Etag" - -# They don't start with X-Object-(Meta|Sysmeta)-, but we save them anyway. -SPECIAL_OBJECT_METADATA_HEADERS = { - "Content-Type", - "Content-Disposition", - "Content-Encoding", - "X-Object-Manifest", - "X-Static-Large-Object"} - -# These are not mutated on object POST. -STICKY_OBJECT_METADATA_HEADERS = { - "X-Static-Large-Object", - ORIGINAL_MD5_HEADER} - -SPECIAL_CONTAINER_METADATA_HEADERS = { - "X-Container-Read", - "X-Container-Write", - "X-Container-Sync-Key", - "X-Container-Sync-To", - "X-Versions-Location"} - -# ProxyFS directories don't know how many objects are under them, nor how -# many bytes each one uses. (Yes, a directory knows how many files and -# subdirectories it contains, but that doesn't include things in those -# subdirectories.) -CONTAINER_HEADERS_WE_LIE_ABOUT = { - "X-Container-Object-Count": "0", - "X-Container-Bytes-Used": "0", -} - -SWIFT_OWNER_HEADERS = { - "X-Container-Read", - "X-Container-Write", - "X-Container-Sync-Key", - "X-Container-Sync-To", - "X-Account-Meta-Temp-Url-Key", - "X-Account-Meta-Temp-Url-Key-2", - "X-Container-Meta-Temp-Url-Key", - "X-Container-Meta-Temp-Url-Key-2", - "X-Account-Access-Control"} - -MD5_ETAG_RE = re.compile("^[a-f0-9]{32}$") - -EMPTY_OBJECT_ETAG = "d41d8cd98f00b204e9800998ecf8427e" - -RPC_TIMEOUT_DEFAULT = 30.0 -MAX_RPC_BODY_SIZE = 2 ** 20 - - -def listing_iter_from_read_plan(read_plan): - """ - Takes a read plan from proxyfsd and turns it into an iterable of - tuples suitable for passing to SegmentedIterable. - - Example read plan: - - [ - { - "Length": 4, - "ObjectPath": "/v1/AUTH_test/Replicated3Way_1/0000000000000074", - "Offset": 0 - }, - { - "Length": 17, - "ObjectPath": "/v1/AUTH_test/Replicated3Way_1/0000000000000076", - "Offset": 0 - }, - { - "Length": 19, - "ObjectPath": "/v1/AUTH_test/Replicated3Way_1/0000000000000078", - "Offset": 0 - }, - { - "Length": 89, - "ObjectPath": "/v1/AUTH_test/Replicated3Way_1/000000000000007A", - "Offset": 0 - } - ] - - Example return value: - - [ - ("/v1/AUTH_test/Replicated3Way_1/0000000000000074", None, None, 0, 3), - ("/v1/AUTH_test/Replicated3Way_1/0000000000000076", None, None, 0, 16), - ("/v1/AUTH_test/Replicated3Way_1/0000000000000078", None, None, 0, 18), - ("/v1/AUTH_test/Replicated3Way_1/000000000000007A", None, None, 0, 88), - ] - """ - if read_plan is None: - # ProxyFS likes to send null values instead of empty lists. - read_plan = () - - # It's a little ugly that the GoCase field names escape from the - # RPC-response parser all the way to here, but it's inefficient, in both - # CPU cycles and programmer brainpower, to create some intermediate - # representation just to avoid GoCase. - return [(rpe["ObjectPath"] or ZERO_FILL_PATH, - None, # we don't know the segment's ETag - None, # we don't know the segment's length - rpe["Offset"], - rpe["Offset"] + rpe["Length"] - 1) - for rpe in read_plan] - - -def x_timestamp_from_epoch_ns(epoch_ns): - """ - Convert a ProxyFS-style Unix timestamp to a Swift X-Timestamp header. - - ProxyFS uses an integral number of nanoseconds since the epoch, while - Swift uses a floating-point number with centimillisecond (10^-5 second) - precision. - - :param epoch_ns: Unix time, expressed as an integral number of - nanoseconds since the epoch. Note that this is not the - usual Unix convention of a *real* number of *seconds* - since the epoch. - - :returns: ISO-8601 timestamp (like those found in Swift's container - listings), e.g. '2016-08-05T00:55:16.966920' - """ - return "{0:.5f}".format(float(epoch_ns) / 1000000000) - - -def iso_timestamp_from_epoch_ns(epoch_ns): - """ - Convert a Unix timestamp to an ISO-8601 timestamp. - - :param epoch_ns: Unix time, expressed as an integral number of - nanoseconds since the epoch. Note that this is not the - usual Unix convention of a *real* number of *seconds* - since the epoch. - - :returns: ISO-8601 timestamp (like those found in Swift's container - listings), e.g. '2016-08-05T00:55:16.966920' - """ - iso_timestamp = datetime.datetime.utcfromtimestamp( - epoch_ns / 1000000000.0).isoformat() - - # Convieniently (ha!), isoformat() method omits the - # fractional-seconds part if it's equal to 0. The Swift proxy server - # does not, so we match its output. - if iso_timestamp[-7] != ".": - iso_timestamp += ".000000" - return iso_timestamp - - -def last_modified_from_epoch_ns(epoch_ns): - """ - Convert a Unix timestamp to an IMF-Fixdate timestamp. - - :param epoch_ns: Unix time, expressed as an integral number of - nanoseconds since the epoch. Note that this is not the - usual Unix convention of a *real* number of *seconds* - since the epoch. - - :returns: Last-Modified header value in IMF-Fixdate format as specified - in RFC 7231 section 7.1.1.1. - """ - return time.strftime( - '%a, %d %b %Y %H:%M:%S GMT', - time.gmtime(math.ceil(epoch_ns / 1000000000.0))) - - -def guess_content_type(filename, is_dir): - if is_dir: - return DIRECTORY_CONTENT_TYPE - - content_type, _ = mimetypes.guess_type(filename) - if not content_type: - content_type = "application/octet-stream" - return content_type - - -def should_validate_etag(a_string): - if not a_string: - return False - return not a_string.strip('"').startswith('pfsv') - - -@contextlib.contextmanager -def pop_and_restore(hsh, key, default=None): - """ - Temporarily remove and yield a value from a hash. Restores that - key/value pair to its original state in the hash on exit. - """ - if key in hsh: - value = hsh.pop(key) - was_there = True - else: - value = default - was_there = False - - yield value - - if was_there: - hsh[key] = value - else: - hsh.pop(key, None) - - -def deserialize_metadata(raw_metadata): - """Deserialize JSON-encoded metadata to WSGI strings""" - if raw_metadata: - try: - metadata = json.loads(raw_metadata) - except ValueError: - metadata = {} - else: - metadata = {} - encoded_metadata = {} - for k, v in metadata.items(): - if six.PY2: - key = k.encode('utf8') if isinstance( - k, six.text_type) else str(k) - value = v.encode('utf8') if isinstance( - v, six.text_type) else str(v) - else: - key = swift_code.str_to_wsgi(k) if isinstance(k, str) else str(k) - value = swift_code.str_to_wsgi(v) if isinstance(v, str) else str(v) - encoded_metadata[key] = value - return encoded_metadata - - -def serialize_metadata(wsgi_metadata): - return json.dumps({ - swift_code.wsgi_to_str(key): ( - swift_code.wsgi_to_str(value) - if isinstance(value, six.string_types) else value) - for key, value in wsgi_metadata.items()}) - - -def merge_container_metadata(old, new): - merged = old.copy() - for k, v in new.items(): - merged[k] = v - return {k: v for k, v in merged.items() if v} - - -def merge_object_metadata(old, new): - ''' - Merge the existing metadata for an object with new metadata passed - in as a result of a POST operation. X-Object-Sysmeta- and similar - metadata cannot be changed by a POST. - ''' - - merged = new.copy() - - for header, value in merged.items(): - if (header.startswith("X-Object-Sysmeta-") or - header in STICKY_OBJECT_METADATA_HEADERS): - del merged[header] - - for header, value in old.items(): - if (header.startswith("X-Object-Sysmeta-") or - header in STICKY_OBJECT_METADATA_HEADERS): - merged[header] = value - - old_ct = old.get("Content-Type") - new_ct = new.get("Content-Type") - if old_ct is not None: - if not new_ct: - merged["Content-Type"] = old_ct - elif ';swift_bytes=' in old_ct: - merged["Content-Type"] = '%s;swift_bytes=%s' % ( - new_ct, old_ct.rsplit(';swift_bytes=', 1)[1]) - - return {k: v for k, v in merged.items() if v} - - -def extract_object_metadata_from_headers(headers): - """ - Find and return the key/value pairs containing object metadata. - - This tries to do the same thing as the Swift object server: save only - relevant headers. If the user sends in "X-Fungus-Amungus: shroomy" in - the PUT request's headers, we'll ignore it, just like plain old Swift - would. - - :param headers: request headers (a dictionary) - - :returns: dictionary containing object-metadata headers (and not a - swob.HeaderKeyDict or similar object) - """ - meta_headers = {} - for header, value in headers.items(): - header = header.title() - - if (header.startswith("X-Object-Meta-") or - header.startswith("X-Object-Sysmeta-") or - header in SPECIAL_OBJECT_METADATA_HEADERS): - - # do not let a client pass in ORIGINAL_MD5_HEADER - if header not in (ORIGINAL_MD5_HEADER, - ORIGINAL_MD5_HEADER.title()): - meta_headers[header] = value - - return meta_headers - - -def extract_container_metadata_from_headers(req): - """ - Find and return the key/value pairs containing container metadata. - - This tries to do the same thing as the Swift container server: save only - relevant headers. If the user sends in "X-Fungus-Amungus: shroomy" in - the PUT request's headers, we'll ignore it, just like plain old Swift - would. - - :param req: a swob Request - - :returns: dictionary containing container-metadata headers - """ - meta_headers = {} - for header, value in req.headers.items(): - header = header.title() - - if ((header.startswith("X-Container-Meta-") or - header.startswith("X-Container-Sysmeta-") or - header in SPECIAL_CONTAINER_METADATA_HEADERS) and - (req.environ.get('swift_owner', False) or - header not in SWIFT_OWNER_HEADERS)): - meta_headers[header] = value - - if header.startswith("X-Remove-"): - header = header.replace("-Remove", "", 1) - if ((header.startswith("X-Container-Meta-") or - header in SPECIAL_CONTAINER_METADATA_HEADERS) and - (req.environ.get('swift_owner', False) or - header not in SWIFT_OWNER_HEADERS)): - meta_headers[header] = "" - return meta_headers - - -def mung_etags(obj_metadata, etag, num_writes): - ''' - Mung the ETag headers that will be stored with an object. The - goal is to preserve ETag metadata passed down by other filters but - to do so in such a way that it will be invalidated if there is a - write to or truncate of the object via the ProxyFS file API. - - The mechanism is to prepend a counter to the ETag header values - that is incremented each time the object is modified is modified. - When the object is read, if the value for the counter has changed, - the ETag is assumed to be invalid. The counter is typically the - number of writes to the object. - - etag is either None or the value that should be returned as the - ETag for the object (in the absence of other considerations). - - This assumes that all headers have been converted to "titlecase", - except ORIGINAL_MD5_HEADER which is the random case string - "X-Object-Sysmeta-ProxyFS-Initial-MD5". - - This ignores SLO headers because it assumes they have already been - stripped. - ''' - if LISTING_ETAG_OVERRIDE_HEADER in obj_metadata: - obj_metadata[LISTING_ETAG_OVERRIDE_HEADER] = "%d:%s" % ( - num_writes, obj_metadata[LISTING_ETAG_OVERRIDE_HEADER]) - - if S3API_ETAG_HEADER in obj_metadata: - obj_metadata[S3API_ETAG_HEADER] = "%d:%s" % ( - num_writes, obj_metadata[S3API_ETAG_HEADER]) - - if etag is not None: - obj_metadata[ORIGINAL_MD5_HEADER] = "%d:%s" % (num_writes, etag) - - return - - -def unmung_etags(obj_metadata, num_writes): - ''' - Unmung the ETag headers associated with an object to return - them to the state they were in when passed to pfs_middleware. - - Delete them if the object has changed or the header value - does not parse correctly. - - This assumes that all headers have been converted to "titlecase", - which means, among other things, that "ETag" will show up as - "Etag". - ''' - - # if the header is invalid or stale it is not added back after the pop - if LISTING_ETAG_OVERRIDE_HEADER in obj_metadata: - val = obj_metadata.pop(LISTING_ETAG_OVERRIDE_HEADER) - try: - stored_num_writes, rest = val.split(':', 1) - if int(stored_num_writes) == num_writes: - obj_metadata[LISTING_ETAG_OVERRIDE_HEADER] = rest - except ValueError: - pass - - if S3API_ETAG_HEADER in obj_metadata: - val = obj_metadata.pop(S3API_ETAG_HEADER) - try: - stored_num_writes, rest = val.split(':', 1) - if int(stored_num_writes) == num_writes: - obj_metadata[S3API_ETAG_HEADER] = rest - except ValueError: - pass - - if ORIGINAL_MD5_HEADER in obj_metadata: - val = obj_metadata.pop(ORIGINAL_MD5_HEADER) - try: - stored_num_writes, rest = val.split(':', 1) - if int(stored_num_writes) == num_writes: - obj_metadata[ORIGINAL_MD5_HEADER] = rest - except ValueError: - pass - - -def best_possible_etag(obj_metadata, account_name, inum, num_writes, - is_dir=False, container_listing=False): - ''' - Return the ETag that is most likely to be correct for the query, - but leave other valid ETags values in the metadata, in case a - higher layer filter wants to use them to override the value - returned here. - - If the ETags in the metadata are invalid, construct and return a - new ProxyFS ETag based on the account name, inode number, and - number of writes. - - obj_metadata may be a Python dictionary, a swob.HeaderKeyDict, or a - swob.HeaderEnvironProxy. ORIGINAL_MD5_HEADER is random case, not - title case, but if obj_metadata is a Python dictionary it will - preserve the same random case. The other two types do case folding so - we don't need to map ORIGINAL_MD5_HEADER to title case. - ''' - if is_dir: - return EMPTY_OBJECT_ETAG - - if container_listing and LISTING_ETAG_OVERRIDE_HEADER in obj_metadata: - return obj_metadata[LISTING_ETAG_OVERRIDE_HEADER] - - if ORIGINAL_MD5_HEADER in obj_metadata: - return obj_metadata[ORIGINAL_MD5_HEADER] - - return construct_etag(account_name, inum, num_writes) - - -def construct_etag(account_name, inum, num_writes): - # We append -32 in an attempt to placate S3 clients. In S3, the ETag of - # a multipart object looks like "hash-N" where is the MD5 of the - # MD5s of the segments and is the number of segments. - # - # Since this etag is not an MD5 digest value, we append -32 here in - # hopes that some S3 clients will be able to download ProxyFS files via - # S3 API without complaining about checksums. - # - # 32 was chosen because it was the ticket number of the author's lunch - # order on the day this code was written. It has no significance. - return '"pfsv2/{}/{:08X}/{:08X}-32"'.format( - urllib_parse.quote(account_name), inum, num_writes) - - -def iterator_posthook(iterable, posthook, *posthook_args, **posthook_kwargs): - try: - for x in iterable: - yield x - finally: - posthook(*posthook_args, **posthook_kwargs) - - -class ZeroFiller(object): - """ - Internal middleware to handle the zero-fill portions of sparse files for - object GET responses. - """ - ZEROES = b"\x00" * 4096 - - def __init__(self, app): - self.app = app - - @swob.wsgify - def __call__(self, req): - if req.path == ZERO_FILL_PATH: - # We know we can do this since the creator of these requests is - # also in this library. - start, end = req.range.ranges[0] - nbytes = end - start + 1 - resp = swob.Response( - request=req, status=206, - headers={"Content-Length": nbytes, - "Content-Range": "%d-%d/%d" % (start, end, nbytes)}, - app_iter=self.yield_n_zeroes(nbytes)) - return resp - else: - return self.app - - def yield_n_zeroes(self, n): - # It's a little clunky, but it does avoid creating new strings of - # zeroes over and over again, and it uses only a small amount of - # memory. This becomes important if a file contains hundreds of - # megabytes or more of zeroes; allocating a single string of zeroes - # would do Bad Things(tm) to our memory usage. - zlen = len(self.ZEROES) - while n > zlen: - yield self.ZEROES - n -= zlen - if n > 0: - yield self.ZEROES[:n] - - -class SnoopingInput(object): - """ - Wrap WSGI input and call a provided callback every time data is read. - """ - def __init__(self, wsgi_input, callback): - self.wsgi_input = wsgi_input - self.callback = callback - - def read(self, *a, **kw): - chunk = self.wsgi_input.read(*a, **kw) - self.callback(chunk) - return chunk - - def readline(self, *a, **kw): - line = self.wsgi_input.readline(*a, **kw) - self.callback(line) - return line - - -class LimitedInput(object): - """ - Wrap WSGI input and limit the consumer to taking at most N bytes. - - Also count bytes read. This lets us tell ProxyFS how big an object is - after an object PUT completes. - """ - def __init__(self, wsgi_input, limit): - self._peeked_data = b"" - self.limit = self.orig_limit = limit - self.bytes_read = 0 - self.wsgi_input = wsgi_input - - def read(self, length=None, *args, **kwargs): - if length is None: - to_read = self.limit - else: - to_read = min(self.limit, length) - to_read -= len(self._peeked_data) - - chunk = self.wsgi_input.read(to_read, *args, **kwargs) - chunk = self._peeked_data + chunk - self._peeked_data = b"" - - self.bytes_read += len(chunk) - self.limit -= len(chunk) - return chunk - - def readline(self, size=None, *args, **kwargs): - if size is None: - to_read = self.limit - else: - to_read = min(self.limit, size) - to_read -= len(self._peeked_data) - - line = self.wsgi_input.readline(to_read, *args, **kwargs) - line = self._peeked_data + line - self._peeked_data = b"" - - self.bytes_read += len(line) - self.limit -= len(line) - return line - - @property - def has_more_to_read(self): - if not self._peeked_data: - self._peeked_data = self.wsgi_input.read(1) - return len(self._peeked_data) > 0 - - -class RequestContext(object): - """ - Stuff we need to service the current request. - - Basically a pile of data with a name. - """ - def __init__(self, req, proxyfsd_addrinfo, - account_name, container_name, object_name): - # swob.Request object - self.req = req - - # address info for proxyfsd, as returned from socket.getaddrinfo() - # - # NB: this is only used for Server.IsAccountBimodal requests; the - # return value there tells us where to go for this particular - # account. That may or may not be the same as - # self.proxyfsd_addrinfo. - self.proxyfsd_addrinfo = proxyfsd_addrinfo - - # account/container/object names - self.account_name = account_name - self.container_name = container_name - self.object_name = object_name - - -class PfsMiddleware(object): - def __init__(self, app, conf, logger=None): - self._cached_proxy_info = None - self.app = app - self.zero_filler_app = ZeroFiller(app) - self.conf = conf - - self.logger = logger or get_logger(conf, log_route='pfs') - - proxyfsd_hosts = [h.strip() for h - in conf.get('proxyfsd_host', '127.0.0.1').split(',')] - self.proxyfsd_port = int(conf.get('proxyfsd_port', '12345')) - - self.proxyfsd_addrinfos = [] - for host in proxyfsd_hosts: - try: - # If hostname resolution fails, we'll cause the proxy to - # fail to start. This is probably better than returning 500 - # to every single request, but maybe not. - # - # To ensure that proxy startup works, use IP addresses for - # proxyfsd_host. Then socket.getaddrinfo() will always work. - addrinfo = socket.getaddrinfo( - host, self.proxyfsd_port, - socket.AF_UNSPEC, socket.SOCK_STREAM)[0] - self.proxyfsd_addrinfos.append(addrinfo) - except socket.gaierror: - self.logger.error("Error resolving hostname %r", host) - raise - - self.proxyfsd_rpc_timeout = float(conf.get('rpc_timeout', - RPC_TIMEOUT_DEFAULT)) - self.bimodal_recheck_interval = float(conf.get( - 'bimodal_recheck_interval', '60.0')) - self.max_get_time = int(conf.get('max_get_time', '86400')) - self.max_log_segment_size = int(conf.get( - 'max_log_segment_size', '2147483648')) # 2 GiB - self.max_coalesce = int(conf.get('max_coalesce', '1000')) - - # Assume a max object length of the Swift default of 1024 bytes plus - # a few extra for JSON quotes, commas, et cetera. - self.max_coalesce_request_size = self.max_coalesce * 1100 - - self.bypass_mode = conf.get('bypass_mode', 'off') - if self.bypass_mode not in ('off', 'read-only', 'read-write'): - raise ValueError('Expected bypass_mode to be one of off, ' - 'read-only, or read-write') - - @swob.wsgify - def __call__(self, req): - vrs, acc, con, obj = utils.parse_path(req.path) - - # The only way to specify bypass mode: /proxyfs/AUTH_acct/... - proxyfs_path = False - if vrs == 'proxyfs': - proxyfs_path = True - req.path_info = req.path_info.replace('/proxyfs/', '/v1/', 1) - vrs = 'v1' - - if not acc or not constraints.valid_api_version(vrs) or ( - obj and not con): - # could be a GET /info request or something made up by some - # other middleware; get out of the way. - return self.app - if not constraints.check_utf8(req.path_info): - return swob.HTTPPreconditionFailed( - body='Invalid UTF8 or contains NULL') - - try: - # Check account to see if this is a bimodal-access account or - # not. ProxyFS is the sole source of truth on this matter. - is_bimodal, proxyfsd_addrinfo = self._unpack_owning_proxyfs(req) - if not is_bimodal and proxyfsd_addrinfo is None: - # This is a plain old Swift account, so we get out of the - # way. - return self.app - elif proxyfsd_addrinfo is None: - # This is a bimodal account, but there is currently no - # proxyfsd responsible for it. This can happen during a move - # of a ProxyFS volume from one proxyfsd to another, and - # should be cleared up quickly. Nevertheless, all we can do - # here is return an error to the client. - return swob.HTTPServiceUnavailable(request=req) - - if con in ('.', '..') or con and len(con) > NAME_MAX: - if req.method == 'PUT' and not obj: - return swob.HTTPBadRequest( - request=req, - body='Container name cannot be "." or "..", ' - 'or be more than 255 bytes long') - else: - return swob.HTTPNotFound(request=req) - elif obj and any(p in ('', '.', '..') or len(p) > NAME_MAX - for p in obj.split('/')): - if req.method == 'PUT': - return swob.HTTPBadRequest( - request=req, - body='No path component may be "", ".", "..", or ' - 'more than 255 bytes long') - else: - return swob.HTTPNotFound(request=req) - - ctx = RequestContext(req, proxyfsd_addrinfo, acc, con, obj) - is_bypass_request = ( - proxyfs_path and - self.bypass_mode in ('read-only', 'read-write') and - req.method != "PROXYFS") - - # For requests that we make to Swift, we have to ensure that any - # auth callback is not present in the WSGI environment. - # Authorization typically uses the object path as an input, and - # letting that loose on log-segment object paths is not likely to - # end well. - # - # We are careful to restore any auth to the environment when we - # exit, though, as this lets authentication work on the segments of - # Swift large objects. The SLO or DLO middleware is to the left of - # us in the pipeline, and it will make multiple subrequests, and - # auth is required for each one. - with pop_and_restore(req.environ, - 'swift.authorize') as auth_cb, \ - pop_and_restore(req.environ, 'swift.authorize_override', - False): - if auth_cb and req.environ.get('swift.source') != 'PFS': - if not is_bypass_request: - req.acl = self._fetch_appropriate_acl(ctx) - # else, user needs to be swift owner - denial_response = auth_cb(ctx.req) - if denial_response: - return denial_response - - # Authorization succeeded - method = req.method - - # Check whether we ought to bypass. Note that swift_owner - # won't be set until we call authorize - if is_bypass_request and req.environ.get('swift_owner'): - if self.bypass_mode == 'read-only' and method not in ( - 'GET', 'HEAD'): - return swob.HTTPMethodNotAllowed(request=req) - return self.app - - # Otherwise, dispatch to a helper method - if method == 'GET' and obj: - resp = self.get_object(ctx) - elif method == 'HEAD' and obj: - resp = self.head_object(ctx) - elif method == 'PUT' and obj: - resp = self.put_object(ctx) - elif method == 'POST' and obj: - resp = self.post_object(ctx) - elif method == 'DELETE' and obj: - resp = self.delete_object(ctx) - elif method == 'COALESCE' and obj: - resp = self.coalesce_object(ctx, auth_cb) - elif method == 'GET' and con: - resp = self.get_container(ctx) - elif method == 'HEAD' and con: - resp = self.head_container(ctx) - elif method == 'PUT' and con: - resp = self.put_container(ctx) - elif method == 'POST' and con: - resp = self.post_container(ctx) - elif method == 'DELETE' and con: - resp = self.delete_container(ctx) - - elif method == 'GET': - resp = self.get_account(ctx) - elif method == 'HEAD': - resp = self.head_account(ctx) - elif method == 'PROXYFS' and not con and not obj: - if not (req.environ.get('swift_owner') and - self.bypass_mode in ('read-only', 'read-write')): - return swob.HTTPMethodNotAllowed(request=req) - resp = self.proxy_rpc(ctx) - # account PUT, POST, and DELETE are just passed - # through to Swift - else: - return self.app - - if req.method in ('GET', 'HEAD'): - resp.headers["Accept-Ranges"] = "bytes" - if not req.environ.get('swift_owner', False): - for key in SWIFT_OWNER_HEADERS: - if key in resp.headers: - del resp.headers[key] - - return resp - - # Provide some top-level exception handling and logging for - # exceptional exceptions. Non-exceptional exceptions will be handled - # closer to where they were raised. - except utils.RpcTimeout as err: - self.logger.error("RPC timeout: %s", err) - return swob.HTTPInternalServerError( - request=req, - headers={"Content-Type": "text/plain"}, - body="RPC timeout: {0}".format(err)) - except utils.RpcError as err: - self.logger.error( - "RPC error: %s; consulting proxyfsd logs may be helpful", err) - return swob.HTTPInternalServerError( - request=req, - headers={"Content-Type": "text/plain"}, - body="RPC error: {0}".format(err)) - - def _fetch_appropriate_acl(self, ctx): - """ - Return the appropriate ACL to handle authorization for the given - request. This method may make a subrequest if necessary. - - The ACL will be one of three things: - - * for object/container GET/HEAD, it's the container's read ACL - (X-Container-Read). - - * for object PUT/POST/DELETE, it's the container's write ACL - (X-Container-Write). - - * for object COALESCE, it's the container's write ACL - (X-Container-Write). Separately, we *also* need to authorize - all the "segments" against both read *and* write ACLs - (see coalesce_object). - - * for all other requests, it's None - - Some authentication systems, of course, also have account-level - ACLs. However, in the fine tradition of having two possible courses - of action and choosing both, the loading of the account-level ACLs - is handled by the auth callback. - """ - - bimodal_checker = ctx.req.environ[utils.ENV_BIMODAL_CHECKER] - - if ctx.req.method in ('GET', 'HEAD') and ctx.container_name: - container_info = get_container_info( - ctx.req.environ, bimodal_checker, - swift_source="PFS") - return container_info['read_acl'] - elif ctx.object_name and ctx.req.method in ( - 'PUT', 'POST', 'DELETE', 'COALESCE'): - container_info = get_container_info( - ctx.req.environ, bimodal_checker, - swift_source="PFS") - return container_info['write_acl'] - else: - return None - - def proxy_rpc(self, ctx): - req = ctx.req - ct = req.headers.get('Content-Type') - if not ct or ct.split(';', 1)[0].strip() != 'application/json': - msg = 'RPC body must have Content-Type application/json' - if ct: - msg += ', not %s' % ct - return swob.Response(status=415, request=req, body=msg) - - cl = req.content_length - if cl is None: - if req.headers.get('Transfer-Encoding') != 'chunked': - return swob.HTTPLengthRequired(request=req) - json_payloads = req.body_file.read(MAX_RPC_BODY_SIZE).split(b'\n') - if req.body_file.read(1): - return swob.HTTPRequestEntityTooLarge(request=req) - else: - if cl > MAX_RPC_BODY_SIZE: - return swob.HTTPRequestEntityTooLarge(request=req) - json_payloads = req.body.split(b'\n') - - if self.bypass_mode == 'read-write': - allowed_methods = rpc.allow_read_write - else: - allowed_methods = rpc.allow_read_only - - payloads = [] - for i, json_payload in enumerate(x for x in json_payloads - if x.strip()): - try: - payload = json.loads(json_payload.decode('utf8')) - if payload['jsonrpc'] != '2.0': - raise ValueError( - 'expected JSONRPC 2.0, got %s' % payload['jsonrpc']) - if not isinstance(payload['method'], six.string_types): - raise ValueError( - 'expected string, got %s' % type(payload['method'])) - if payload['method'] not in allowed_methods: - raise ValueError( - 'method %s not allowed' % payload['method']) - if not (isinstance(payload['params'], list) and - len(payload['params']) == 1 and - isinstance(payload['params'][0], dict)): - raise ValueError - except (TypeError, KeyError, ValueError) as err: - return swob.HTTPBadRequest( - request=req, - body=(b'Could not parse/validate JSON payload #%d %s: %s' % - (i, json_payload, str(err).encode('utf8')))) - payloads.append(payload) - - # TODO: consider allowing more than one payload per request - if len(payloads) != 1: - return swob.HTTPBadRequest( - request=req, - body='Expected exactly one JSON payload') - - # Our basic validation is done; spin up a connection and send requests - client = utils.JsonRpcClient(ctx.proxyfsd_addrinfo) - payload = payloads[0] - try: - if 'id' not in payload: - payload['id'] = str(uuid.uuid4()) - payload['params'][0]['AccountName'] = ctx.account_name - response = client.call(payload, self.proxyfsd_rpc_timeout, - raise_on_rpc_error=False) - except utils.RpcTimeout as err: - self.logger.debug(str(err)) - return swob.HTTPBadGateway(request=req) - except socket.error as err: - self.logger.debug("Error communicating with %r: %s.", - ctx.proxyfsd_addrinfo, err) - return swob.HTTPBadGateway(request=req) - else: - return swob.HTTPOk( - request=req, body=json.dumps(response), - headers={'Content-Type': 'application/json'}) - - def get_account(self, ctx): - req = ctx.req - limit = self._get_listing_limit( - req, self._default_account_listing_limit()) - marker = req.params.get('marker', '') - end_marker = req.params.get('end_marker', '') - get_account_request = rpc.get_account_request( - urllib_parse.unquote(req.path), marker, end_marker, limit) - # If the account does not exist, then __call__ just falls through to - # self.app, so we never even get here. If we got here, then the - # account does exist, so we don't have to worry about not-found - # versus other-error here. Any error counts as "completely busted". - # - # We let the top-level RpcError handler catch this. - get_account_response = self.rpc_call(ctx, get_account_request) - account_mtime, account_entries = rpc.parse_get_account_response( - get_account_response) - - resp_content_type = swift_code.get_listing_content_type(req) - if resp_content_type == "text/plain": - body = self._plaintext_account_get_response(account_entries) - elif resp_content_type == "application/json": - body = self._json_account_get_response(account_entries) - elif resp_content_type.endswith("/xml"): - body = self._xml_account_get_response(account_entries, - ctx.account_name) - else: - raise Exception("unexpected content type %r" % - (resp_content_type,)) - - resp_class = swob.HTTPOk if body else swob.HTTPNoContent - resp = resp_class(content_type=resp_content_type, charset="utf-8", - request=req, body=body) - - # For accounts, the meta/sysmeta is stored in the account DB in - # Swift, not in ProxyFS. - account_info = get_account_info( - req.environ, req.environ[utils.ENV_BIMODAL_CHECKER], - swift_source="PFS") - for key, value in account_info["meta"].items(): - resp.headers["X-Account-Meta-" + key] = value - for key, value in account_info["sysmeta"].items(): - resp.headers["X-Account-Sysmeta-" + key] = value - acc_acl = resp.headers.get("X-Account-Sysmeta-Core-Access-Control") - parsed_acc_acl = parse_acl(version=2, data=acc_acl) - if parsed_acc_acl: - acc_acl = format_acl(version=2, acl_dict=parsed_acc_acl) - resp.headers["X-Account-Access-Control"] = acc_acl - - resp.headers["X-Timestamp"] = x_timestamp_from_epoch_ns(account_mtime) - - # Pretend the object counts are 0 and that all containers have the - # default storage policy. Until (a) containers have some support for - # the X-Storage-Policy header, and (b) we get container metadata - # back from Server.RpcGetAccount, this is the best we can do. - policy = self._default_storage_policy() - resp.headers["X-Account-Object-Count"] = "0" - resp.headers["X-Account-Bytes-Used"] = "0" - resp.headers["X-Account-Container-Count"] = str(len(account_entries)) - - resp.headers["X-Account-Storage-Policy-%s-Object-Count" % policy] = "0" - resp.headers["X-Account-Storage-Policy-%s-Bytes-Used" % policy] = "0" - k = "X-Account-Storage-Policy-%s-Container-Count" % policy - resp.headers[k] = str(len(account_entries)) - - return resp - - def _plaintext_account_get_response(self, account_entries): - chunks = [] - for entry in account_entries: - chunks.append(entry["Basename"].encode('utf-8')) - chunks.append(b"\n") - return b''.join(chunks) - - def _json_account_get_response(self, account_entries): - json_entries = [] - for entry in account_entries: - json_entry = { - "name": entry["Basename"], - # Older versions of proxyfsd only returned mtime, but ctime - # better reflects the semantics of X-Timestamp - "last_modified": iso_timestamp_from_epoch_ns(entry.get( - "AttrChangeTime", entry["ModificationTime"])), - # proxyfsd can't compute these without recursively walking - # the entire filesystem, so rather than have a built-in DoS - # attack, we just put out zeros here. - # - # These stats aren't particularly up-to-date in Swift - # anyway, so there aren't going to be working clients that - # rely on them for accuracy. - "count": 0, - "bytes": 0} - json_entries.append(json_entry) - return json.dumps(json_entries) - - def _xml_account_get_response(self, account_entries, account_name): - root_node = ET.Element('account', name=account_name) - - for entry in account_entries: - container_node = ET.Element('container') - - name_node = ET.Element('name') - name_node.text = entry["Basename"] - container_node.append(name_node) - - count_node = ET.Element('count') - count_node.text = '0' - container_node.append(count_node) - - bytes_node = ET.Element('bytes') - bytes_node.text = '0' - container_node.append(bytes_node) - - lm_node = ET.Element('last_modified') - # Older versions of proxyfsd only returned mtime, but ctime - # better reflects the semantics of X-Timestamp - lm_node.text = iso_timestamp_from_epoch_ns(entry.get( - "AttrChangeTime", entry["ModificationTime"])) - container_node.append(lm_node) - - root_node.append(container_node) - - buf = BytesIO() - ET.ElementTree(root_node).write( - buf, encoding="utf-8", xml_declaration=True) - return buf.getvalue() - - def head_account(self, ctx): - req = ctx.req - resp = req.get_response(self.app) - resp.headers["ProxyFS-Enabled"] = "yes" - return resp - - def head_container(self, ctx): - head_request = rpc.head_request(urllib_parse.unquote(ctx.req.path)) - try: - head_response = self.rpc_call(ctx, head_request) - except utils.RpcError as err: - if err.errno == pfs_errno.NotFoundError: - return swob.HTTPNotFound(request=ctx.req) - else: - raise - - raw_metadata, mtime_ns, _, _, _, _ = rpc.parse_head_response( - head_response) - metadata = deserialize_metadata(raw_metadata) - resp = swob.HTTPNoContent(request=ctx.req, headers=metadata) - resp.headers["X-Timestamp"] = x_timestamp_from_epoch_ns(mtime_ns) - resp.headers["Last-Modified"] = last_modified_from_epoch_ns(mtime_ns) - resp.headers["Content-Type"] = swift_code.get_listing_content_type( - ctx.req) - resp.charset = "utf-8" - self._add_required_container_headers(resp) - return resp - - def put_container(self, ctx): - req = ctx.req - container_path = urllib_parse.unquote(req.path) - err = constraints.check_metadata(req, 'container') - if err: - return err - err = swift_code.clean_acls(req) - if err: - return err - new_metadata = extract_container_metadata_from_headers(req) - - # Check name's length. The account name is checked separately (by - # Swift, not by this middleware) and has its own limit; we are - # concerned only with the container portion of the path. - _, _, container_name, _ = utils.parse_path(req.path) - maxlen = self._max_container_name_length() - if len(container_name) > maxlen: - return swob.HTTPBadRequest( - request=req, - body=('Container name length of %d longer than %d' % - (len(container_name), maxlen))) - - try: - head_response = self.rpc_call( - ctx, rpc.head_request(container_path)) - raw_old_metadata, _, _, _, _, _ = rpc.parse_head_response( - head_response) - except utils.RpcError as err: - if err.errno == pfs_errno.NotFoundError: - clear_info_cache(None, ctx.req.environ, ctx.account_name, - container=ctx.container_name) - self.rpc_call(ctx, rpc.put_container_request( - container_path, - "", - serialize_metadata({ - k: v for k, v in new_metadata.items() if v}))) - return swob.HTTPCreated(request=req) - else: - raise - - old_metadata = deserialize_metadata(raw_old_metadata) - merged_metadata = merge_container_metadata( - old_metadata, new_metadata) - raw_merged_metadata = serialize_metadata(merged_metadata) - - clear_info_cache(None, ctx.req.environ, ctx.account_name, - container=ctx.container_name) - self.rpc_call(ctx, rpc.put_container_request( - container_path, raw_old_metadata, raw_merged_metadata)) - - return swob.HTTPAccepted(request=req) - - def post_container(self, ctx): - req = ctx.req - container_path = urllib_parse.unquote(req.path) - err = constraints.check_metadata(req, 'container') - if err: - return err - err = swift_code.clean_acls(req) - if err: - return err - new_metadata = extract_container_metadata_from_headers(req) - - try: - head_response = self.rpc_call( - ctx, rpc.head_request(container_path)) - raw_old_metadata, _, _, _, _, _ = rpc.parse_head_response( - head_response) - except utils.RpcError as err: - if err.errno == pfs_errno.NotFoundError: - return swob.HTTPNotFound(request=req) - else: - raise - - old_metadata = deserialize_metadata(raw_old_metadata) - merged_metadata = merge_container_metadata( - old_metadata, new_metadata) - raw_merged_metadata = serialize_metadata(merged_metadata) - - # Check that we're still within overall limits - req.headers.clear() - req.headers.update(merged_metadata) - err = constraints.check_metadata(req, 'container') - if err: - return err - # reset it... - req.headers.clear() - req.headers.update(new_metadata) - - clear_info_cache(None, req.environ, ctx.account_name, - container=ctx.container_name) - self.rpc_call(ctx, rpc.post_request( - container_path, raw_old_metadata, raw_merged_metadata)) - - return swob.HTTPNoContent(request=req) - - def delete_container(self, ctx): - # Turns out these are the same RPC with the same error handling, so - # why not? - clear_info_cache(None, ctx.req.environ, ctx.account_name, - container=ctx.container_name) - return self.delete_object(ctx) - - def _get_listing_limit(self, req, default_limit): - raw_limit = req.params.get('limit') - - if raw_limit is not None: - try: - limit = int(raw_limit) - except ValueError: - limit = default_limit - - if limit > default_limit: - err = "Maximum limit is %d" % default_limit - raise swob.HTTPPreconditionFailed(request=req, body=err) - elif limit < 0: - limit = default_limit - else: - limit = default_limit - - return limit - - def _max_container_name_length(self): - proxy_info = self._proxy_info() - swift_max = proxy_info["swift"]["max_container_name_length"] - return min(swift_max, NAME_MAX) - - def _default_account_listing_limit(self): - proxy_info = self._proxy_info() - return proxy_info["swift"]["account_listing_limit"] - - def _default_container_listing_limit(self): - proxy_info = self._proxy_info() - return proxy_info["swift"]["container_listing_limit"] - - def _default_storage_policy(self): - proxy_info = self._proxy_info() - # Swift guarantees that exactly one default storage policy exists. - return [pol["name"] - for pol in proxy_info["swift"]["policies"] - if pol.get("default", False)][0] - - def _proxy_info(self): - if self._cached_proxy_info is None: - req = swob.Request.blank("/info") - resp = req.get_response(self.app) - self._cached_proxy_info = json.loads(resp.body) - return self._cached_proxy_info - - def _add_required_container_headers(self, resp): - resp.headers.update(CONTAINER_HEADERS_WE_LIE_ABOUT) - resp.headers["X-Storage-Policy"] = self._default_storage_policy() - - def get_container(self, ctx): - req = ctx.req - if req.environ.get('swift.source') in ('DLO', 'SW', 'VW'): - # Middlewares typically want json, but most *assume* it following - # https://github.com/openstack/swift/commit/4806434 - # TODO: maybe replace with `if req.environ.get('swift.source'):` ?? - params = req.params - params['format'] = 'json' - req.params = params - - limit = self._get_listing_limit( - req, self._default_container_listing_limit()) - marker = req.params.get('marker', '') - end_marker = req.params.get('end_marker', '') - prefix = req.params.get('prefix', '') - delimiter = req.params.get('delimiter', '') - # For now, we only support "/" as a delimiter - if delimiter not in ("", "/"): - return swob.HTTPBadRequest(request=req) - get_container_request = rpc.get_container_request( - urllib_parse.unquote(req.path), - marker, end_marker, limit, prefix, delimiter) - try: - get_container_response = self.rpc_call(ctx, get_container_request) - except utils.RpcError as err: - if err.errno == pfs_errno.NotFoundError: - return swob.HTTPNotFound(request=req) - else: - raise - - container_ents, raw_metadata, mtime_ns = \ - rpc.parse_get_container_response(get_container_response) - - resp_content_type = swift_code.get_listing_content_type(req) - resp = swob.HTTPOk(content_type=resp_content_type, charset="utf-8", - request=req) - if resp_content_type == "text/plain": - resp.body = self._plaintext_container_get_response( - container_ents) - elif resp_content_type == "application/json": - resp.body = self._json_container_get_response( - container_ents, ctx.account_name, delimiter) - elif resp_content_type.endswith("/xml"): - resp.body = self._xml_container_get_response( - container_ents, ctx.account_name, ctx.container_name) - else: - raise Exception("unexpected content type %r" % - (resp_content_type,)) - - metadata = deserialize_metadata(raw_metadata) - resp.headers.update(metadata) - self._add_required_container_headers(resp) - resp.headers["X-Timestamp"] = x_timestamp_from_epoch_ns(mtime_ns) - resp.headers["Last-Modified"] = last_modified_from_epoch_ns(mtime_ns) - - return resp - - def _plaintext_container_get_response(self, container_entries): - chunks = [] - for ent in container_entries: - chunks.append(ent["Basename"].encode('utf-8')) - chunks.append(b"\n") - return b''.join(chunks) - - def _json_container_get_response(self, container_entries, account_name, - delimiter): - json_entries = [] - for ent in container_entries: - name = ent["Basename"] - size = ent["FileSize"] - # Older versions of proxyfsd only returned mtime, but ctime - # better reflects the semantics of X-Timestamp - last_modified = iso_timestamp_from_epoch_ns(ent.get( - "AttrChangeTime", ent["ModificationTime"])) - obj_metadata = deserialize_metadata(ent["Metadata"]) - unmung_etags(obj_metadata, ent["NumWrites"]) - - content_type = swift_code.wsgi_to_str( - obj_metadata.get("Content-Type")) - if content_type is None: - content_type = guess_content_type(ent["Basename"], - ent["IsDir"]) - content_type, swift_bytes = content_type.partition( - ';swift_bytes=')[::2] - - etag = best_possible_etag( - obj_metadata, account_name, - ent["InodeNumber"], ent["NumWrites"], is_dir=ent["IsDir"], - container_listing=True) - json_entry = { - "name": name, - "bytes": int(swift_bytes or size), - "content_type": content_type, - "hash": etag, - "last_modified": last_modified} - json_entries.append(json_entry) - - if delimiter != "" and "IsDir" in ent and ent["IsDir"]: - json_entries.append({"subdir": ent["Basename"] + delimiter}) - - return json.dumps(json_entries).encode('ascii') - - # TODO: This method is usually non reachable, because at some point in the - # pipeline, we convert JSON to XML. We should either remove this or update - # it to support delimiters in case it's really needed. - # Same thing probably applies to plain text responses. - def _xml_container_get_response(self, container_entries, account_name, - container_name): - root_node = ET.Element('container', name=container_name) - - for container_entry in container_entries: - obj_name = container_entry['Basename'] - obj_metadata = deserialize_metadata(container_entry["Metadata"]) - unmung_etags(obj_metadata, container_entry["NumWrites"]) - - content_type = swift_code.wsgi_to_str( - obj_metadata.get("Content-Type")) - if content_type is None: - content_type = guess_content_type( - container_entry["Basename"], container_entry["IsDir"]) - content_type, swift_bytes = content_type.partition( - ';swift_bytes=')[::2] - - etag = best_possible_etag( - obj_metadata, account_name, - container_entry["InodeNumber"], - container_entry["NumWrites"], - is_dir=container_entry["IsDir"]) - container_node = ET.Element('object') - - name_node = ET.Element('name') - name_node.text = obj_name - container_node.append(name_node) - - hash_node = ET.Element('hash') - hash_node.text = etag - container_node.append(hash_node) - - bytes_node = ET.Element('bytes') - bytes_node.text = swift_bytes or str(container_entry["FileSize"]) - container_node.append(bytes_node) - - ct_node = ET.Element('content_type') - if six.PY2: - ct_node.text = content_type.decode('utf-8') - else: - ct_node.text = content_type - container_node.append(ct_node) - - lm_node = ET.Element('last_modified') - # Older versions of proxyfsd only returned mtime, but ctime - # better reflects the semantics of X-Timestamp - lm_node.text = iso_timestamp_from_epoch_ns(container_entry.get( - "AttrChangeTime", container_entry["ModificationTime"])) - container_node.append(lm_node) - - root_node.append(container_node) - - buf = BytesIO() - ET.ElementTree(root_node).write( - buf, encoding="utf-8", xml_declaration=True) - return buf.getvalue() - - def put_object(self, ctx): - req = ctx.req - # Make sure the (virtual) container exists - # - # We have to dig out an earlier-in-the-chain middleware here because - # Swift's get_container_info() function has an internal whitelist of - # environ keys that it'll keep, and our is-bimodal stuff isn't - # included. To work around this, we pass in the middleware chain - # starting with the bimodal checker so it can repopulate our environ - # keys. At least the RpcIsBimodal response is cached, so this - # shouldn't be too slow. - container_info = get_container_info( - req.environ, req.environ[utils.ENV_BIMODAL_CHECKER], - swift_source="PFS") - if not 200 <= container_info["status"] < 300: - return swob.HTTPNotFound(request=req) - - if 'x-timestamp' in req.headers: - try: - req_timestamp = Timestamp(req.headers['X-Timestamp']) - except ValueError: - return swob.HTTPBadRequest( - request=req, content_type='text/plain', - body='X-Timestamp should be a UNIX timestamp float value; ' - 'was %r' % req.headers['x-timestamp']) - req.headers['X-Timestamp'] = req_timestamp.internal - else: - req.headers['X-Timestamp'] = Timestamp(time.time()).internal - - if not req.headers.get('Content-Type'): - req.headers['Content-Type'] = guess_content_type( - req.path, is_dir=ctx.object_name.endswith('/')) - err = constraints.check_object_creation(req, ctx.object_name) - if err: - return err - - if (req.headers['Content-Type'] == DIRECTORY_CONTENT_TYPE and - req.headers.get('Content-Length') == '0'): - return self.put_object_as_directory(ctx) - else: - return self.put_object_as_file(ctx) - - def put_object_as_directory(self, ctx): - """ - Create or update an object as a directory. - """ - req = ctx.req - - request_etag = req.headers.get("ETag", "") - if should_validate_etag(request_etag) and \ - request_etag != EMPTY_OBJECT_ETAG: - return swob.HTTPUnprocessableEntity(request=req) - - path = urllib_parse.unquote(req.path) - obj_metadata = extract_object_metadata_from_headers(req.headers) - - # mung the passed etags, if any (NumWrites for a directory is - # always zero) - mung_etags(obj_metadata, request_etag, 0) - - rpc_req = rpc.middleware_mkdir_request( - path, serialize_metadata(obj_metadata)) - rpc_resp = self.rpc_call(ctx, rpc_req) - mtime_ns, inode, num_writes = rpc.parse_middleware_mkdir_response( - rpc_resp) - - # currently best_possible_etag() returns EMPTY_OBJECT_ETAG for - # all directories, but that might change in the future. - # unmung the obj_metadata so best_possible_etag() can use it - # if its behavior changes (note that num_writes is forced to 0). - unmung_etags(obj_metadata, 0) - resp_headers = { - "Content-Type": DIRECTORY_CONTENT_TYPE, - "Last-Modified": last_modified_from_epoch_ns(mtime_ns)} - resp_headers["ETag"] = best_possible_etag( - obj_metadata, ctx.account_name, inode, 0, is_dir=True) - - return swob.HTTPCreated(request=req, headers=resp_headers, body="") - - def put_object_as_file(self, ctx): - """ - ProxyFS has the concepts of "virtual" and "physical" path. The - virtual path is the one that the user sees, i.e. /v1/acc/con/obj. - A physical path is the location of an underlying log-segment - object, e.g. /v1/acc/ContainerPoolName_1/00000000501B7321. - - An object in ProxyFS is backed by one or more physical objects. In - the case of an object PUT, we ask proxyfsd for one or more suitable - physical-object names for the object, write the data there - ourselves, then tell proxyfsd what we've done. - """ - req = ctx.req - - virtual_path = urllib_parse.unquote(req.path) - put_location_req = rpc.put_location_request(virtual_path) - - request_etag = req.headers.get("ETag", "") - hasher = hashlib.md5() - wsgi_input = SnoopingInput(req.environ["wsgi.input"], hasher.update) - - # TODO: when the upload size is known (i.e. Content-Length is set), - # ask for enough locations up front that we can consume the whole - # request with only one call to RpcPutLocation(s). - - # TODO: ask to validate the path a bit better; if we are putting an - # object at /v1/a/c/kitten.png/whoops.txt (where kitten.png is a - # file), we should probably catch that before reading any input so - # that, if the client sent "Expect: 100-continue", we can give them - # an error early. - - physical_path_gen = ( - rpc.parse_put_location_response( - self.rpc_call(ctx, put_location_req)) - for _ in itertools.repeat(None)) - - error_response = swift_code.check_object_creation(req) - if error_response: - return error_response - - # Since this upload can be arbitrarily large, we split it across - # multiple log segments. - log_segments = [] - i = 0 - while True: - # First, make sure there's more data to read from the client. No - # sense allocating log segments and whatnot if we're not going - # to use them. - subinput = LimitedInput(wsgi_input, self.max_log_segment_size) - if not subinput.has_more_to_read: - break - - # Ask ProxyFS for the next log segment we can use - phys_path = next(physical_path_gen) - - # Set up the subrequest with the bare minimum of useful headers. - # This lets us avoid headers that will break the PUT immediately - # (ETag), headers that may complicate GETs of this object - # (X-Static-Large-Object, X-Object-Manifest), things that will - # break the GET some time in the future (X-Delete-At, - # X-Delete-After), and things that take up xattr space for no - # real gain (user metadata). - subreq = swob.Request.blank(phys_path) - subreq.method = 'PUT' - subreq.environ['wsgi.input'] = subinput - subreq.headers["Transfer-Encoding"] = "chunked" - - # This ensures that (a) every subrequest has its own unique - # txid, and (b) a log search for the txid in the response finds - # all of the subrequests. - trans_id = req.headers.get('X-Trans-Id') - if trans_id: - subreq.headers['X-Trans-Id'] = trans_id + ("-%03x" % i) - - # Actually put one chunk of the data into Swift - subresp = subreq.get_response(self.app) - if not 200 <= subresp.status_int < 299: - # Something went wrong; may as well bail out now - return subresp - - log_segments.append((phys_path, subinput.bytes_read)) - i += 1 - - if should_validate_etag(request_etag) and \ - hasher.hexdigest() != request_etag: - return swob.HTTPUnprocessableEntity(request=req) - - # All the data is now in Swift; we just have to tell proxyfsd - # about it. Mung any passed ETags values to include the - # number of writes to the file (basically, the object's update - # count) and supply the MD5 hash computed here which becomes - # object's future ETag value until the object updated. - obj_metadata = extract_object_metadata_from_headers(req.headers) - mung_etags(obj_metadata, hasher.hexdigest(), len(log_segments)) - - put_complete_req = rpc.put_complete_request( - virtual_path, log_segments, serialize_metadata(obj_metadata)) - try: - mtime_ns, inode, __writes = rpc.parse_put_complete_response( - self.rpc_call(ctx, put_complete_req)) - except utils.RpcError as err: - # We deliberately don't try to clean up our log segments on - # failure. ProxyFS is responsible for cleaning up unreferenced - # log segments. - if err.errno == pfs_errno.NotEmptyError: - return swob.HTTPConflict( - request=req, - headers={"Content-Type": "text/plain"}, - body="This is a non-empty directory") - elif err.errno == pfs_errno.NotDirError: - return swob.HTTPConflict( - request=req, - headers={"Content-Type": "text/plain"}, - body="Path element is a file, not a directory") - else: - # punt to top-level error handler - raise - - # For reference, an object PUT response to plain Swift looks like: - # HTTP/1.1 201 Created - # Last-Modified: Thu, 08 Dec 2016 22:51:13 GMT - # Content-Length: 0 - # Etag: 9303a8d23189779e71f347032d633327 - # Content-Type: text/html; charset=UTF-8 - # X-Trans-Id: tx7b3e2b88df2f4975a5476-005849e3e0dfw1 - # Date: Thu, 08 Dec 2016 22:51:12 GMT - # - # We get Content-Length, X-Trans-Id, and Date for free, but we need - # to fill in the rest. - resp_headers = { - "Etag": hasher.hexdigest(), - "Content-Type": guess_content_type(req.path, False), - "Last-Modified": last_modified_from_epoch_ns(mtime_ns)} - return swob.HTTPCreated(request=req, headers=resp_headers, body="") - - def post_object(self, ctx): - req = ctx.req - err = constraints.check_metadata(req, 'object') - if err: - return err - path = urllib_parse.unquote(req.path) - new_metadata = extract_object_metadata_from_headers(req.headers) - - try: - head_response = self.rpc_call(ctx, rpc.head_request(path)) - raw_old_metadata, mtime, _, _, inode_number, _ = \ - rpc.parse_head_response(head_response) - except utils.RpcError as err: - if err.errno in (pfs_errno.NotFoundError, pfs_errno.NotDirError): - return swob.HTTPNotFound(request=req) - else: - raise - - # There is no need to call unmung_etags() before the merge and - # mung_etags() after because the merge cannot change the several - # possible ETag headers. - # - # This might be an opportunity to drop an ETAG header that has - # become stale due to num_writes changing, but that does not - # seem important to address. - old_metadata = deserialize_metadata(raw_old_metadata) - merged_metadata = merge_object_metadata(old_metadata, new_metadata) - raw_merged_metadata = serialize_metadata(merged_metadata) - - self.rpc_call(ctx, rpc.post_request( - path, raw_old_metadata, raw_merged_metadata)) - - resp = swob.HTTPAccepted(request=req, body="") - return resp - - def get_object(self, ctx): - req = ctx.req - byteranges = req.range.ranges if req.range else () - - try: - object_response = self.rpc_call(ctx, rpc.get_object_request( - urllib_parse.unquote(req.path), byteranges)) - except utils.RpcError as err: - if err.errno in (pfs_errno.NotFoundError, pfs_errno.NotDirError): - return swob.HTTPNotFound(request=req) - elif err.errno == pfs_errno.IsDirError: - return swob.HTTPOk( - request=req, body="", - headers={"Content-Type": DIRECTORY_CONTENT_TYPE, - "ETag": EMPTY_OBJECT_ETAG}) - - else: - # punt to top-level exception handler - raise - - (read_plan, raw_metadata, size, mtime_ns, - is_dir, ino, num_writes, lease_id) = \ - rpc.parse_get_object_response(object_response) - - metadata = deserialize_metadata(raw_metadata) - unmung_etags(metadata, num_writes) - headers = swob.HeaderKeyDict(metadata) - - if "Content-Type" not in headers: - headers["Content-Type"] = guess_content_type(req.path, is_dir) - else: - headers['Content-Type'] = headers['Content-Type'].split( - ';swift_bytes=')[0] - - headers["Accept-Ranges"] = "bytes" - headers["Last-Modified"] = last_modified_from_epoch_ns( - mtime_ns) - headers["X-Timestamp"] = x_timestamp_from_epoch_ns( - mtime_ns) - headers["Etag"] = best_possible_etag( - headers, ctx.account_name, ino, num_writes, is_dir=is_dir) - - get_read_plan = req.params.get("get-read-plan", "no") - if get_read_plan == "": - get_read_plan = "yes" - if self.bypass_mode != 'off' and req.environ.get('swift_owner') and \ - config_true_value(get_read_plan): - headers.update({ - # Flag that pfs_middleware correctly interpretted this request - "X-ProxyFS-Read-Plan": "True", - # Stash the "real" content type... - "X-Object-Content-Type": headers["Content-Type"], - # ... so we can indicate that *this* data is coming out JSON - "Content-Type": "application/json", - # Also include the total object size - # (since the read plan respects Range requests) - "X-Object-Content-Length": size, - }) - return swob.HTTPOk(request=req, body=json.dumps(read_plan), - headers=headers) - - if size > 0 and read_plan is None: - headers["Content-Range"] = "bytes */%d" % size - return swob.HTTPRequestedRangeNotSatisfiable( - request=req, headers=headers) - - # NB: this is a size-0 queue, so it acts as a channel: a put() - # blocks until another greenthread does a get(). This lets us use it - # for (very limited) bidirectional communication. - channel = eventlet.queue.Queue(0) - eventlet.spawn_n(self._keep_lease_alive, ctx, channel, lease_id) - - listing_iter = listing_iter_from_read_plan(read_plan) - # Make sure that nobody (like our __call__ method) messes with this - # environment once we've started. Otherwise, the auth callback may - # reappear, causing log-segment GET requests to fail. This may be - # seen with container ACLs; since the underlying container name - # differs from the user-presented one, without copying the - # environment, all object GET requests for objects in a public - # container would fail. - copied_req = swob.Request(req.environ.copy()) - - # Ideally we'd wrap seg_iter instead, but swob.Response relies on - # its app_iter supporting certain methods for conditional responses - # to work, and forwarding all those methods through the wrapper is - # prone to failure whenever a new method is added. - # - # Wrapping the listing iterator is just as good. After - # SegmentedIterable exhausts it, we can safely release the lease. - def done_with_object_get(): - channel.put("you can stop now") - # It's not technically necessary for us to wait for the other - # greenthread here; we could use one-way notification. However, - # doing things this way lets us ensure that, once this function - # returns, there are no more background actions taken by the - # greenthread. This makes testing a lot easier; we can call the - # middleware, let it return, and then assert things. Were we to - # use a fire-and-forget style, we'd never be sure when all the - # RPCs had been called, and the tests would end up flaky. - channel.get() - - wrapped_listing_iter = iterator_posthook( - listing_iter, done_with_object_get) - - seg_iter = swift_code.SegmentedIterable( - copied_req, self.zero_filler_app, wrapped_listing_iter, - self.max_get_time, - self.logger, 'PFS', 'PFS', - name=req.path) - - resp = swob.HTTPOk(app_iter=seg_iter, conditional_response=True, - request=req, - headers=headers, - content_length=size) - - # Support conditional if-match/if-none-match requests for SLOs - resp._conditional_etag = swift_code.resolve_etag_is_at_header( - req, resp.headers) - return resp - - def _keep_lease_alive(self, ctx, channel, lease_id): - keep_going = [True] - lease_error = [False] - - def renew(): - if lease_error[0]: - return - - try: - self.rpc_call(ctx, rpc.renew_lease_request(lease_id)) - except (utils.RpcError, utils.RpcTimeout): - # If there's an error renewing the lease, stop pestering - # proxyfsd about it. We'll keep serving the object - # anyway, and we'll just hope no log segments vanish - # while we do it. - keep_going[0] = False - lease_error[0] = True - - # It could have been a while since this greenthread was created. - # Let's renew first just to be sure. - renew() - - while keep_going[0]: - try: - channel.get(block=True, timeout=LEASE_RENEWAL_INTERVAL) - # When we get a message here, we should stop. - keep_going[0] = False - except eventlet.queue.Empty: - # Nobody told us we're done, so renew the lease and loop - # around again. - renew() - - if not lease_error[0]: - # Tell proxyfsd that we're done with the lease, but only if - # there were no errors keeping it renewed. - self.rpc_call(ctx, rpc.release_lease_request(lease_id)) - - channel.put("alright, it's done") - - def delete_object(self, ctx): - try: - self.rpc_call(ctx, rpc.delete_request( - urllib_parse.unquote(ctx.req.path))) - except utils.RpcError as err: - if err.errno in (pfs_errno.NotFoundError, pfs_errno.NotDirError): - return swob.HTTPNotFound(request=ctx.req) - elif err.errno == pfs_errno.NotEmptyError: - return swob.HTTPConflict(request=ctx.req) - else: - raise - return swob.HTTPNoContent(request=ctx.req) - - def head_object(self, ctx): - req = ctx.req - head_request = rpc.head_request(urllib_parse.unquote(req.path)) - try: - head_response = self.rpc_call(ctx, head_request) - except utils.RpcError as err: - if err.errno in (pfs_errno.NotFoundError, pfs_errno.NotDirError): - return swob.HTTPNotFound(request=req) - else: - raise - - raw_md, last_modified_ns, file_size, is_dir, ino, num_writes = \ - rpc.parse_head_response(head_response) - - metadata = deserialize_metadata(raw_md) - unmung_etags(metadata, num_writes) - headers = swob.HeaderKeyDict(metadata) - - if "Content-Type" not in headers: - headers["Content-Type"] = guess_content_type(req.path, is_dir) - else: - headers['Content-Type'] = headers['Content-Type'].split( - ';swift_bytes=')[0] - - headers["Content-Length"] = file_size - headers["ETag"] = best_possible_etag( - headers, ctx.account_name, ino, num_writes, is_dir=is_dir) - headers["Last-Modified"] = last_modified_from_epoch_ns( - last_modified_ns) - headers["X-Timestamp"] = x_timestamp_from_epoch_ns( - last_modified_ns) - - resp = swob.HTTPOk(request=req, headers=headers, - conditional_response=True) - - # Support conditional if-match/if-none-match requests for SLOs - resp._conditional_etag = swift_code.resolve_etag_is_at_header( - req, resp.headers) - return resp - - def coalesce_object(self, ctx, auth_cb): - - # extract and verify the object list for the new object - req = ctx.req - object_path = urllib_parse.unquote(req.path) - - probably_json = req.environ['wsgi.input'].read( - self.max_coalesce_request_size + 1) - - if len(probably_json) > self.max_coalesce_request_size: - return swob.HTTPRequestEntityTooLarge(request=req) - - try: - decoded_json = json.loads(probably_json) - except ValueError: - return swob.HTTPBadRequest(request=req, body="Malformed JSON") - - if "elements" not in decoded_json: - return swob.HTTPBadRequest(request=req, body="Malformed JSON") - if not isinstance(decoded_json, dict): - return swob.HTTPBadRequest(request=req, body="Malformed JSON") - if not isinstance(decoded_json["elements"], list): - return swob.HTTPBadRequest(request=req, body="Malformed JSON") - if len(decoded_json["elements"]) > self.max_coalesce: - return swob.HTTPRequestEntityTooLarge(request=req) - authed_containers = set() - ctx.req.environ.setdefault('swift.infocache', {}) - for elem in decoded_json["elements"]: - if not isinstance(elem, six.string_types): - return swob.HTTPBadRequest(request=req, body="Malformed JSON") - - normalized_elem = elem - if normalized_elem.startswith('/'): - normalized_elem = normalized_elem[1:] - if any(p in ('', '.', '..') for p in normalized_elem.split('/')): - return swob.HTTPBadRequest(request=req, - body="Bad element path: %s" % elem) - elem_container = normalized_elem.split('/', 1)[0] - elem_container_path = '/v1/%s/%s' % ( - ctx.account_name, elem_container) - if auth_cb and elem_container_path not in authed_containers: - # Gotta check auth for all of the segments, too - bimodal_checker = ctx.req.environ[utils.ENV_BIMODAL_CHECKER] - acl_env = ctx.req.environ.copy() - acl_env['PATH_INFO'] = swift_code.text_to_wsgi( - elem_container_path) - container_info = get_container_info( - acl_env, bimodal_checker, - swift_source="PFS") - for acl in ('read_acl', 'write_acl'): - req.acl = container_info[acl] - denial_response = auth_cb(ctx.req) - if denial_response: - return denial_response - authed_containers.add(elem_container_path) - - # proxyfs treats the number of objects as the number of writes - num_writes = len(decoded_json["elements"]) - - # validate the metadata for the new object (further munging - # of ETags will be done later) - err = constraints.check_metadata(req, 'object') - if err: - return err - - # retrieve the ETag value in the request, or None - req_etag = req.headers.get('ETag') - - # strip out user supplied and other unwanted headers - obj_metadata = extract_object_metadata_from_headers(req.headers) - - # strip out headers that apply only to SLO objects - unwanted_headers = ['X-Static-Large-Object'] - for header in obj_metadata.keys(): - if header.startswith("X-Object-Sysmeta-Slo-"): - unwanted_headers.append(header) - for header in unwanted_headers: - if header in obj_metadata: - del obj_metadata[header] - - # Now that we know the number of writes (really number of objects) we - # can mung the sundry ETag headers. - mung_etags(obj_metadata, req_etag, num_writes) - - raw_obj_metadata = serialize_metadata(obj_metadata) - - # now get proxyfs to coalesce the objects and set initial headers - try: - coalesce_response = self.rpc_call( - ctx, rpc.coalesce_object_request( - object_path, decoded_json["elements"], raw_obj_metadata)) - - except utils.RpcError as err: - if err.errno == pfs_errno.NotFoundError: - return swob.HTTPNotFound( - request=req, - headers={"Content-Type": "text/plain"}, - body="One or more path elements not found") - elif err.errno in (pfs_errno.NotDirError, pfs_errno.IsDirError): - return swob.HTTPConflict( - request=req, - headers={"Content-Type": "text/plain"}, - body="Elements must be plain files, not directories") - elif err.errno == pfs_errno.TooManyLinksError: - return swob.HTTPConflict( - request=req, - headers={"Content-Type": "text/plain"}, - body=("One or more path elements has multiple links; " - "only singly-linked files can be combined")) - else: - raise - - last_modified_ns, inum, num_writes = \ - rpc.parse_coalesce_object_response(coalesce_response) - - unmung_etags(obj_metadata, num_writes) - headers = {} - headers["Etag"] = best_possible_etag( - obj_metadata, ctx.account_name, inum, num_writes) - headers["Last-Modified"] = last_modified_from_epoch_ns( - last_modified_ns) - headers["X-Timestamp"] = x_timestamp_from_epoch_ns( - last_modified_ns) - - return swob.HTTPCreated(request=req, headers=headers) - - def _unpack_owning_proxyfs(self, req): - """ - Checks to see if an account is bimodal or not, and if so, which proxyfs - daemon is responsible for it. - - This is done by looking in the request environment; there's another - middleware (BimodalChecker) that populates these fields. - - :returns: 2-tuple (is-bimodal, proxyfsd-addrinfo). - """ - - return (req.environ.get(utils.ENV_IS_BIMODAL), - req.environ.get(utils.ENV_OWNING_PROXYFS)) - - def rpc_call(self, ctx, rpc_request): - """ - Call a remote procedure in proxyfsd. - - :param ctx: context for the current HTTP request - - :param rpc_request: Python dictionary containing the request - (method, args, etc.) in JSON-RPC format. - - :returns: the result of the RPC, whatever that looks like - - :raises: utils.RpcTimeout if the RPC takes too long - - :raises: utils.RpcError if the RPC returns an error. Inspecting this - exception's "errno" attribute may be useful. However, errno may - not always be set; if the error returned from proxyfsd does not - have an errno in it, then the exception's errno attribute will - be None. - """ - rpc_method = rpc_request['method'] - start_time = time.time() - try: - return self._rpc_call([ctx.proxyfsd_addrinfo], rpc_request) - finally: - duration = time.time() - start_time - self.logger.debug("RPC %s took %.6fs", rpc_method, duration) - - def _rpc_call(self, addrinfos, rpc_request): - addrinfos = set(addrinfos) - - # We can get fast errors or slow errors here; we retry across all - # hosts on fast errors, but immediately raise a slow error. HTTP - # clients won't wait forever for a response, so we can't retry slow - # errors across all hosts. - # - # Fast errors are things like "connection refused" or "no route to - # host". Slow errors are timeouts. - result = None - while addrinfos: - addrinfo = addrinfos.pop() - rpc_client = utils.JsonRpcClient(addrinfo) - try: - result = rpc_client.call(rpc_request, - self.proxyfsd_rpc_timeout) - except socket.error as err: - if addrinfos: - self.logger.debug("Error communicating with %r: %s. " - "Trying again with another host.", - addrinfo, err) - continue - else: - raise - - errstr = result.get("error") - if errstr: - errno = utils.extract_errno(errstr) - raise utils.RpcError(errno, errstr) - - return result["result"] diff --git a/pfs_middleware/pfs_middleware/pfs_errno.py b/pfs_middleware/pfs_middleware/pfs_errno.py deleted file mode 100644 index 8dc7959e..00000000 --- a/pfs_middleware/pfs_middleware/pfs_errno.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -""" -Error numbers from ProxyFS. Not guaranteed complete; these are just the -ones that the middleware cares about. - - -The interface mimics errno's interface, but the error names and values are -different. -""" - -errorcode = { - 2: "NotFoundError", - 17: "FileExistsError", - 20: "NotDirError", - 21: "IsDirError", - 22: "InvalidArgError", - 31: "TooManyLinksError", - 39: "NotEmptyError", -} - -g = globals() -for errval, errstr in errorcode.items(): - g[errstr] = errval diff --git a/pfs_middleware/pfs_middleware/rpc.py b/pfs_middleware/pfs_middleware/rpc.py deleted file mode 100644 index cc97cdfc..00000000 --- a/pfs_middleware/pfs_middleware/rpc.py +++ /dev/null @@ -1,541 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -""" -This module contains functions for creating and parsing JSON-RPC -requests and replies for calling remote procedures in proxyfsd. - -These functions do not perform any network IO. -""" - -import base64 -import uuid - -allow_read_only = { - "Server.RpcAccess", - "Server.RpcFetchExtentMapChunk", - "Server.RpcGetAccount", - "Server.RpcGetContainer", - "Server.RpcGetObject", - "Server.RpcGetStat", - "Server.RpcGetXAttr", - "Server.RpcLease", - "Server.RpcListXAttr", - "Server.RpcLookup", - "Server.RpcLookupPlus", - "Server.RpcMountByAccountName", - "Server.RpcPing", - "Server.RpcReadSymlink", - "Server.RpcReaddirByLoc", - "Server.RpcReaddirPlusByLoc", - "Server.RpcReaddirPlus", - "Server.RpcReaddir", - "Server.RpcStatVFS", - "Server.RpcType", - "Server.RpcUnmount", -} - -allow_read_write = { - "Server.RpcAccess", - "Server.RpcChmod", - "Server.RpcChown", - "Server.RpcCreate", - "Server.RpcDelete", - "Server.RpcDestroy", - "Server.RpcFetchExtentMapChunk", - "Server.RpcFlock", - "Server.RpcFlush", - "Server.RpcGetAccount", - "Server.RpcGetContainer", - "Server.RpcGetObject", - "Server.RpcGetStat", - "Server.RpcGetXAttr", - "Server.RpcLease", - "Server.RpcLink", - "Server.RpcListXAttr", - "Server.RpcLog", - "Server.RpcLookup", - "Server.RpcLookupPlus", - "Server.RpcMiddlewareMkdir", - "Server.RpcMiddlewarePost", - "Server.RpcMkdir", - "Server.RpcMountByAccountName", - "Server.RpcMove", - "Server.RpcPing", - "Server.RpcProvisionObject", - "Server.RpcPutLocation", - "Server.RpcReadSymlink", - "Server.RpcReaddirByLoc", - "Server.RpcReaddirPlusByLoc", - "Server.RpcReaddirPlus", - "Server.RpcReaddir", - "Server.RpcReleaseLease", - "Server.RpcRemoveXAttr", - "Server.RpcRename", - "Server.RpcRenewLease", - "Server.RpcResize", - "Server.RpcRmdir", - "Server.RpcSetTime", - "Server.RpcSetXAttr", - "Server.RpcSnapShotCreate", - "Server.RpcSnapShotDelete", - "Server.RpcSnapShotListByID", - "Server.RpcSnapShotListByName", - "Server.RpcSnapShotListByTime", - "Server.RpcSnapShotLookupByName", - "Server.RpcStatVFS", - "Server.RpcSymlink", - "Server.RpcType", - "Server.RpcUnlink", - "Server.RpcUnmount", - "Server.RpcWrote", -} - - -def _encode_binary(data): - if data: - return base64.b64encode(data.encode('ascii')).decode('ascii') - else: - return "" - - -def _decode_binary(bindata): - if bindata: - return base64.b64decode(bindata).decode('ascii') - else: - # None (happens sometimes) or empty-string (also happens sometimes) - return "" - - -def _ctime_or_mtime(res): - ''' - Get the last-modified time from a response. - - Ideally, this will be the AttrChangeTime (ctime), but older versions of - proxyfsd only returned the user-settable ModificationTime (mtime). - ''' - return res.get("AttrChangeTime", res["ModificationTime"]) - - -def jsonrpc_request(method, params, call_id=None): - """ - Marshal an RPC request in JSON-RPC 2.0 format. - - See http://www.jsonrpc.org/specification for details. - - :param method: remote method to invoke - :param params: parameters to remote method - :param call_id: value of JSON-RPC 'id' parameter - - :returns: dictionary suitable for passing to json.dumps() to get a - serialized JSON-RPC request - """ - # JSON-RPC requires an 'id' parameter in order to receive a response, - # and that parameter has to be unique for every call made by that - # client. - if call_id is None: - call_id = str(uuid.uuid4()) - - return {'jsonrpc': '2.0', # magic value - 'method': method, - 'params': params, - 'id': call_id} - - -def is_account_bimodal_request(account_name): - """ - Return a JSON-RPC request to ask if an account is bimodal (proxyfs - middleware is responsible for it) or not (not). - - :param account_name: name of the account. Note that this is the bare - account name: no /v1, no container, and no object. - """ - return jsonrpc_request("Server.RpcIsAccountBimodal", - [{"AccountName": account_name}]) - - -def parse_is_account_bimodal_response(resp): - """ - Parse a response from RpcIsAccountBimodal. - - Returns (True, peer-addr) if the account is bimodal, (False, None) - otherwise. - """ - is_bimodal = resp["IsBimodal"] - ip = resp["ActivePeerPrivateIPAddr"] - - if ip.startswith("[") and ip.endswith("]"): - # IPv6 addresses come with brackets around them, IPv4 addresses do - # not - ip = ip[1:-1] - - return (is_bimodal, ip) - - -def get_object_request(path, http_ranges): - """ - Return a JSON-RPC request to get a read plan and other info for a - particular object. - - :param path: URL path component for the object, e.g. "/v1/acc/con/obj" - :param ranges: HTTP byte-ranges from the HTTP request's Range header - """ - # This RPC method takes one positional argument, which is a JSON object - # with two fields: the path and the ranges. - # - # The ranges are a bit weird: if both ends are specified, then we have - # to convert from HTTP-style start/end byte indices to offset/length. - # It's slightly annoying, but just arithmetic. - # - # However, if it's a half-open range (e.g. bytes=M- or bytes=-N), then - # we send the left endpoint in Offset or the right one in Len; the other - # is None. - rpc_ranges = [] - for start, end in http_ranges: - if start is None or end is None: - rpc_range = {"Offset": start, "Len": end} - else: - rpc_range = {"Offset": start, "Len": end - start + 1} - rpc_ranges.append(rpc_range) - - return jsonrpc_request("Server.RpcGetObject", - [{"VirtPath": path, - "ReadEntsIn": rpc_ranges}]) - - -def parse_get_object_response(read_plan_response): - """ - Parse a response from RpcGetObject. - - Returns (read_plan, metadata, file_size, modification time, - IsDir, inode number, number of writes, lease ID) - - The read plan is a list of dictionaries with keys "Length", "Offset", - and "ObjectPath". One gets the object data by reading bytes - from starting at byte . - - IsDir was added to GET response later so it may not be present in a rolling - upgrade scenario. But if its not present and the GET returned success then - its not a directory because GET used to fail on a directory object. - """ - is_dir = read_plan_response.get("IsDir") or False - return (read_plan_response["ReadEntsOut"], - _decode_binary(read_plan_response["Metadata"]), - read_plan_response["FileSize"], - _ctime_or_mtime(read_plan_response), - is_dir, - read_plan_response["InodeNumber"], - read_plan_response["NumWrites"], - read_plan_response["LeaseId"]) - - -def coalesce_object_request(destination, elements, new_metadata): - """ - Return a JSON-RPC request to get a read plan and other info for a - particular object. - - :param destination: full path for the destination object, e.g. /v1/a/c/o - - :param elements: list of account-relative paths for the files to - coalesce, e.g. ["c1/o1", "c2/o2"] - """ - return jsonrpc_request("Server.RpcCoalesce", - [{"VirtPath": destination, - "ElementAccountRelativePaths": elements, - "NewMetaData": _encode_binary(new_metadata)}]) - - -def parse_coalesce_object_response(coalesce_object_response): - """ - Parse a response from RpcGetObject. - - Returns (modification time, inode no., no. writes). - """ - return (_ctime_or_mtime(coalesce_object_response), - coalesce_object_response["InodeNumber"], - coalesce_object_response["NumWrites"]) - - -def put_location_request(path): - """ - Return a JSON-RPC request to get a segment path for an incoming object. - - This is used in handling an object PUT; the middleware asks proxyfsd - where to store the data (this request), then tells proxyfsd what it's - done (RpcPutComplete). - - :param path: URL path component for the object, e.g. "/v1/acc/con/obj" - """ - return jsonrpc_request("Server.RpcPutLocation", [{"VirtPath": path}]) - - -def parse_put_location_response(put_location_response): - """ - Parse a response from RpcPutLocation. - - :returns: physical path (an object path) - """ - return put_location_response["PhysPath"] - - -def put_complete_request(virtual_path, log_segments, obj_metadata): - """ - Return a JSON-RPC request to notify proxyfsd that an object PUT has - completed. - - This is used in handling an object PUT; the middleware asks proxyfsd - where to store the data (RpcPutLocation), then tells proxyfsd what it's - done (this request). - - :param virtual_path: user-visible path component for the object, e.g. - "/v1/acc/con/obj". This is what the user specified in their HTTP PUT - request. - - :param log_segments: the log segments containing the data and their - sizes. Comes as a list of 2-tuples (segment-name, segment-size). - - :param obj_metadata: serialized object metadata - """ - return jsonrpc_request("Server.RpcPutComplete", [{ - "VirtPath": virtual_path, - "PhysPaths": [ls[0] for ls in log_segments], - "PhysLengths": [ls[1] for ls in log_segments], - "Metadata": _encode_binary(obj_metadata)}]) - - -def parse_put_complete_response(put_complete_response): - """ - Parse a response from RpcPutComplete. - - :returns: 3-tuple. The first element is the mtime of the new file, in - nanoseconds since the epoch; the second is the file's inode number, and - the third is the number of writes the file has seen since its creation. - """ - return (_ctime_or_mtime(put_complete_response), - put_complete_response["InodeNumber"], - put_complete_response["NumWrites"]) - - -def get_account_request(path, marker, end_marker, limit): - """ - Return a JSON-RPC request to get a account listing for a given - account. - - :param path: URL path component for the account, e.g. "/v1/acc" - - :param marker: marker query param, e.g. "animals/fish/carp.png". Comes - from the ?marker=X part of the query string sent by the - client. - - :param end_marker: end_marker query param, e.g. "animals/zebra.png". Comes - from the ?end_marker=X part of the query string sent by - the client. - - :param limit: maximum number of entries to return - """ - # This RPC method takes one positional argument, which is a JSON object - # with two fields: the path and the ranges. - return jsonrpc_request("Server.RpcGetAccount", - [{"VirtPath": path, "Marker": marker, - "EndMarker": end_marker, "MaxEntries": limit}]) - - -def parse_get_account_response(get_account_response): - """ - Parse a response from RpcGetAccount. - - Returns: (account mtime, account entries). - - The account mtime is in nanoseconds since the epoch. - - The account entries are a list of dictionaries with keys: - - Basename: the container name - - ModificationTime: container mtime, in nanoseconds since the epoch - """ - mtime = _ctime_or_mtime(get_account_response) - account_entries = get_account_response["AccountEntries"] - if account_entries is None: - # seems to happen when it's an empty list - return (mtime, []) - else: - return (mtime, account_entries) - - -def get_container_request(path, marker, end_marker, limit, prefix, delimiter): - """ - Return a JSON-RPC request to get a container listing for a given - container. - - :param path: URL path component for the container, e.g. "/v1/acc/con" - - :param marker: marker query param, e.g. "animals/fish/carp.png". Comes - from the ?marker=X part of the query string sent by the - client. - - :param end_marker: end_marker query param, e.g. "animals/zebra.png". Comes - from the ?end_marker=X part of the query string sent by - the client. - - :param limit: maximum number of entries to return - - :param prefix: prefix of all returned entries' filenames - - :param delimiter: delimiter value, which returns the object names that are - nested in the container - """ - # This RPC method takes one positional argument, which is a JSON object - # with two fields: the path and the ranges. - return jsonrpc_request("Server.RpcGetContainer", - [{"VirtPath": path, "Marker": marker, - "EndMarker": end_marker, - "MaxEntries": limit, "Prefix": prefix, - "Delimiter": delimiter}]) - - -def parse_get_container_response(get_container_response): - """ - Parse a response from RpcGetContainer. - - Returns: (container entries, container metadata, container mtime). - - The container entries are a list of dictionaries with keys: - - Basename: the object name - - FileSize: the object's size in bytes - - ModificationTime: object mtime, in nanoseconds since the epoch - - IsDir: True if object is a directory, False otherwise - - InodeNumber: object's inode number - - Metadata: object's serialized metadata, if any - - The container's metadata is just a string. Presumably it's some - JSON-serialized dictionary that this middleware previously set, but it - could really be anything. - - The container's mtime is the modification time of the directory, in - nanoseconds since the epoch. - """ - res = get_container_response - - ents = [] - for ent in res["ContainerEntries"]: - ent = ent.copy() - ent["Metadata"] = _decode_binary(ent["Metadata"]) - ents.append(ent) - - container_meta = _decode_binary(res["Metadata"]) - container_mtime = _ctime_or_mtime(res) - return ents, container_meta, container_mtime - - -def middleware_mkdir_request(path, obj_metadata): - """ - :param path: URL path component for the dir, e.g. "/v1/acc/con/obj". - Must refer to an object, not a container (use - get_container_request() for containers). - - :param metadata: serialized object metadata - """ - # There's also an RpcMkdir, but it's not suitable for our use. - return jsonrpc_request("Server.RpcMiddlewareMkdir", [{ - "VirtPath": path, - "Metadata": _encode_binary(obj_metadata)}]) - - -def parse_middleware_mkdir_response(mkdir_resp): - """ - Parse a response from RpcMiddlewareMkdir. - - Returns (mtime in nanoseconds, inode number, number of writes) - """ - return (_ctime_or_mtime(mkdir_resp), - mkdir_resp["InodeNumber"], - mkdir_resp["NumWrites"]) - - -def head_request(path): - """ - Return a JSON-RPC request to HEAD a container or object - - :param path: URL path component for the container or object, - e.g. "/v1/acc/con" - """ - return jsonrpc_request("Server.RpcHead", [{"VirtPath": path}]) - - -def parse_head_response(head_response): - """ - Parse a response from RpcHead. - - Returns: 6-tuple (serialized metadata, modification time in nanoseconds, - file size, is-directory, inode number, number of writes) - - The metadata is whatever was sent to proxyfsd. Presumably it's - serialized somehow, but it's not guaranteed by proxyfsd. This will be - empty for a container or object created via Samba/FUSE and never - modified via HTTP. - """ - return (_decode_binary(head_response["Metadata"]), - _ctime_or_mtime(head_response), - head_response["FileSize"], head_response["IsDir"], - head_response["InodeNumber"], head_response["NumWrites"]) - - -def delete_request(path): - """ - Return a JSON-RPC request to delete a file or directory. - - :param path: path to the object or container, e.g. "/v1/a/c/o" - """ - return jsonrpc_request("Server.RpcDelete", [{"VirtPath": path}]) - - -# NB: there is no parse_delete_response since a successful response to -# RpcDelete contains no useful information. - - -def post_request(path, old_metadata, new_metadata): - return jsonrpc_request("Server.RpcPost", [{ - "VirtPath": path, - "OldMetaData": _encode_binary(old_metadata), - "NewMetaData": _encode_binary(new_metadata)}]) - - -def put_container_request(path, old_metadata, new_metadata): - return jsonrpc_request("Server.RpcPutContainer", [{ - "VirtPath": path, - "OldMetadata": _encode_binary(old_metadata), - "NewMetadata": _encode_binary(new_metadata)}]) - - -def renew_lease_request(lease_id): - """ - Return a JSON-RPC request to renew a lease. We do this while serving an - object GET response to make sure that the file's log segments don't get - deleted while we're still using them. - - :param lease: ID of the lease, probably returned as part of a response - from Server.RpcGetObject - """ - return jsonrpc_request("Server.RpcRenewLease", [{ - "LeaseId": lease_id}]) - - -def release_lease_request(lease_id): - """ - Return a JSON-RPC request to release a lease. We do this when we're done - serving an object GET response to unblock log-segment deletion for the - file we're serving. - - :param lease: ID of the lease, probably returned as part of a response - from Server.RpcGetObject - """ - return jsonrpc_request("Server.RpcReleaseLease", [{ - "LeaseId": lease_id}]) diff --git a/pfs_middleware/pfs_middleware/s3_compat.py b/pfs_middleware/pfs_middleware/s3_compat.py deleted file mode 100644 index f910e3ec..00000000 --- a/pfs_middleware/pfs_middleware/s3_compat.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -""" -Middleware to provide S3 compatibility functions. - -In conjunction with the Swift3 middleware, this allows clients using the S3 -API to access data in bimodal ProxyFS accounts. - -Features: - - * To handle an S3 Complete Multipart Upload request, swift3 PUTs a SLO - manifest. This middleware converts that manifest to a COALESCE request - so that the resulting object works as expected for both HTTP and - filesystem access. -""" - -import json - -from . import utils - -from swift.common import swob -from swift.common.utils import get_logger - - -def convert_slo_to_coalesce(slo_manifest): - return {"elements": [e["name"].lstrip("/") for e in slo_manifest]} - - -class S3Compat(object): - def __init__(self, app, conf, logger=None): - self.logger = logger or get_logger(conf, log_route='pfs_s3_compat') - self.app = app - - @swob.wsgify - def __call__(self, req): - if (self._is_s3_slo_manifest_put(req) and - req.environ.get(utils.ENV_IS_BIMODAL)): - # This was generated by swift3 and mangled by slo, so there's no - # worry about size here. Either it fits in memory and we didn't - # crash before getting here, or it doesn't fit in memory and - # this function is never reached. - slo_manifest = json.loads(req.environ['wsgi.input'].read()) - coalesce_input = json.dumps( - convert_slo_to_coalesce(slo_manifest)).encode('utf-8') - req.environ['wsgi.input'] = swob.WsgiBytesIO(coalesce_input) - req.headers['Content-Length'] = str(len(coalesce_input)) - req.method = 'COALESCE' - req.params.pop('multipart-manifest', None) - return self.app - - def _is_s3_slo_manifest_put(self, req): - ver, acc, con, obj = utils.parse_path(req.path) - return ( - obj and - req.method == 'PUT' and - req.params.get('multipart-manifest', '').lower() == 'put' and - # This is a somewhat brittle way of detecting a Swift3 request, - # but it's all we've got. - req.environ.get('swift.source') == 'S3') diff --git a/pfs_middleware/pfs_middleware/swift_code.py b/pfs_middleware/pfs_middleware/swift_code.py deleted file mode 100644 index 3b207ca2..00000000 --- a/pfs_middleware/pfs_middleware/swift_code.py +++ /dev/null @@ -1,1081 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -""" -Code copied from OpenStack Swift. - -We import some things from Swift if we think they're in stable-enough -locations to depend on. When it turns out we're wrong, the things get copied -into here. - -This is all Apache-licensed code, so copying it into here is fine. - -The copyright header in this file was taken from -swift/common/request_helpers.py in the OpenStack Swift source distribution. -""" - -from distutils.version import LooseVersion -import email.parser -import hashlib -import itertools -import re -import six -import sys -import time -from six import BytesIO -from six.moves.urllib.parse import unquote -from swift import __version__ as swift_version -from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable, \ - HTTPNotImplemented, HTTPLengthRequired, Request, Range, \ - multi_range_iterator, HeaderKeyDict - - -# Taken from swift/common/constraints.py, commit d2e32b3 -#: Query string format= values to their corresponding content-type values -FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json', - 'xml': 'application/xml'} - - -# On new enough swift, we should always send JSON listings; -# see https://github.com/openstack/swift/commit/4806434 -LISTING_FORMATS_SWIFT = (LooseVersion(swift_version) >= LooseVersion('2.16.0')) - - -# Taken from swift/common/request_helpers.py, commit d2e32b3 -def get_param(req, name, default=None): - """ - Get parameters from an HTTP request ensuring proper handling UTF-8 - encoding. - - :param req: request object - :param name: parameter name - :param default: result to return if the parameter is not found - :returns: HTTP request parameter value - (as UTF-8 encoded str, not unicode object) - :raises HTTPBadRequest: if param not valid UTF-8 byte sequence - """ - value = req.params.get(name, default) - if value and not isinstance(value, six.text_type): - try: - value.decode('utf8') # Ensure UTF8ness - except UnicodeDecodeError: - raise HTTPBadRequest( - request=req, content_type='text/plain', - body='"%s" parameter not valid UTF-8' % name) - return value - - -# Taken from swift/common/request_helpers.py, commit d2e32b3 -def get_listing_content_type(req): - """ - Determine the content type to use for an account or container listing - response. - - :param req: request object - :returns: content type as a string (e.g. text/plain, application/json) - :raises HTTPNotAcceptable: if the requested content type is not acceptable - :raises HTTPBadRequest: if the 'format' query param is provided and - not valid UTF-8 - """ - if LISTING_FORMATS_SWIFT: - return 'application/json' - query_format = get_param(req, 'format') - if query_format: - req.accept = FORMAT2CONTENT_TYPE.get( - query_format.lower(), FORMAT2CONTENT_TYPE['plain']) - out_content_type = req.accept.best_match( - ['text/plain', 'application/json', 'application/xml', 'text/xml']) - if not out_content_type: - raise HTTPNotAcceptable(request=req) - return out_content_type - - -# Taken from swift/common/utils.py, commit d2e32b3 -# Used when reading config values -TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) - - -def config_true_value(value): - """ - Returns True if the value is either True or a string in TRUE_VALUES. - Returns False otherwise. - """ - return value is True or \ - (isinstance(value, six.string_types) and value.lower() in TRUE_VALUES) - - -# Taken from swift/common/constraints.py, commit 1962b18 -# -# Modified to not check maximum object size; ProxyFS doesn't enforce one. -def check_object_creation(req): - """ - Check to ensure that everything is alright about an object to be created. - :param req: HTTP request object - :returns: HTTPLengthRequired -- missing content-length header and not - a chunked request - :returns: HTTPBadRequest -- missing or bad content-type header, or - bad metadata - :returns: HTTPNotImplemented -- unsupported transfer-encoding header value - """ - try: - req.message_length() - except ValueError as e: - return HTTPBadRequest(request=req, content_type='text/plain', - body=str(e)) - except AttributeError as e: - return HTTPNotImplemented(request=req, content_type='text/plain', - body=str(e)) - if req.content_length is None and \ - req.headers.get('transfer-encoding') != 'chunked': - return HTTPLengthRequired(body='Missing Content-Length header.', - request=req, - content_type='text/plain') - - if 'Content-Type' not in req.headers: - return HTTPBadRequest(request=req, content_type='text/plain', - body='No content type') - return None - - -# Made up for SegmentedIterable -class ListingIterError(Exception): - pass - - -# Made up for SegmentedIterable -class SegmentError(Exception): - pass - - -# Made up for iter_multipart_mime_documents -class ChunkReadError(Exception): - pass - - -# Made up for iter_multipart_mime_documents -class MimeInvalid(Exception): - pass - - -# Taken from swift/common/utils.py, commit 1962b18 -def close_if_possible(maybe_closable): - close_method = getattr(maybe_closable, 'close', None) - if callable(close_method): - return close_method() - - -# Taken from swift/common/utils.py, commit 1962b18 -# -# Modified to use different exception classes. -class _MultipartMimeFileLikeObject(object): - - def __init__(self, wsgi_input, boundary, input_buffer, read_chunk_size): - self.no_more_data_for_this_file = False - self.no_more_files = False - self.wsgi_input = wsgi_input - self.boundary = boundary - self.input_buffer = input_buffer - self.read_chunk_size = read_chunk_size - - def read(self, length=None): - if not length: - length = self.read_chunk_size - if self.no_more_data_for_this_file: - return b'' - - # read enough data to know whether we're going to run - # into a boundary in next [length] bytes - if len(self.input_buffer) < length + len(self.boundary) + 2: - to_read = length + len(self.boundary) + 2 - while to_read > 0: - try: - chunk = self.wsgi_input.read(to_read) - except (IOError, ValueError) as e: - raise ChunkReadError(str(e)) - to_read -= len(chunk) - self.input_buffer += chunk - if not chunk: - self.no_more_files = True - break - - boundary_pos = self.input_buffer.find(self.boundary) - - # boundary does not exist in the next (length) bytes - if boundary_pos == -1 or boundary_pos > length: - ret = self.input_buffer[:length] - self.input_buffer = self.input_buffer[length:] - # if it does, just return data up to the boundary - else: - ret, self.input_buffer = self.input_buffer.split(self.boundary, 1) - self.no_more_files = self.input_buffer.startswith(b'--') - self.no_more_data_for_this_file = True - self.input_buffer = self.input_buffer[2:] - return ret - - def readline(self): - if self.no_more_data_for_this_file: - return b'' - boundary_pos = newline_pos = -1 - while newline_pos < 0 and boundary_pos < 0: - try: - chunk = self.wsgi_input.read(self.read_chunk_size) - except (IOError, ValueError) as e: - raise ChunkReadError(str(e)) - self.input_buffer += chunk - newline_pos = self.input_buffer.find(b'\r\n') - boundary_pos = self.input_buffer.find(self.boundary) - if not chunk: - self.no_more_files = True - break - # found a newline - if newline_pos >= 0 and \ - (boundary_pos < 0 or newline_pos < boundary_pos): - # Use self.read to ensure any logic there happens... - ret = b'' - to_read = newline_pos + 2 - while to_read > 0: - chunk = self.read(to_read) - # Should never happen since we're reading from input_buffer, - # but just for completeness... - if not chunk: - break - to_read -= len(chunk) - ret += chunk - return ret - else: # no newlines, just return up to next boundary - return self.read(len(self.input_buffer)) - - -# Taken from swift/common/utils.py, commit 1962b18 -class Spliterator(object): - """ - Takes an iterator yielding sliceable things (e.g. strings or lists) and - yields subiterators, each yielding up to the requested number of items - from the source. - - >>> si = Spliterator(["abcde", "fg", "hijkl"]) - >>> ''.join(si.take(4)) - "abcd" - >>> ''.join(si.take(3)) - "efg" - >>> ''.join(si.take(1)) - "h" - >>> ''.join(si.take(3)) - "ijk" - >>> ''.join(si.take(3)) - "l" # shorter than requested; this can happen with the last iterator - - """ - def __init__(self, source_iterable): - self.input_iterator = iter(source_iterable) - self.leftovers = None - self.leftovers_index = 0 - self._iterator_in_progress = False - - def take(self, n): - if self._iterator_in_progress: - raise ValueError( - "cannot call take() again until the first iterator is" - " exhausted (has raised StopIteration)") - self._iterator_in_progress = True - - try: - if self.leftovers: - # All this string slicing is a little awkward, but it's for - # a good reason. Consider a length N string that someone is - # taking k bytes at a time. - # - # With this implementation, we create one new string of - # length k (copying the bytes) on each call to take(). Once - # the whole input has been consumed, each byte has been - # copied exactly once, giving O(N) bytes copied. - # - # If, instead of this, we were to set leftovers = - # leftovers[k:] and omit leftovers_index, then each call to - # take() would copy k bytes to create the desired substring, - # then copy all the remaining bytes to reset leftovers, - # resulting in an overall O(N^2) bytes copied. - llen = len(self.leftovers) - self.leftovers_index - if llen <= n: - n -= llen - to_yield = self.leftovers[self.leftovers_index:] - self.leftovers = None - self.leftovers_index = 0 - yield to_yield - else: - to_yield = self.leftovers[ - self.leftovers_index:(self.leftovers_index + n)] - self.leftovers_index += n - n = 0 - yield to_yield - - while n > 0: - chunk = next(self.input_iterator) - cl = len(chunk) - if cl <= n: - n -= cl - yield chunk - else: - self.leftovers = chunk - self.leftovers_index = n - yield chunk[:n] - n = 0 - finally: - self._iterator_in_progress = False - - -# Taken from swift/common/request_helpers.py, commit 1962b18 -def make_env(env, method=None, path=None, agent='Swift', query_string=None, - swift_source=None): - """ - Returns a new fresh WSGI environment. - - :param env: The WSGI environment to base the new environment on. - :param method: The new REQUEST_METHOD or None to use the - original. - :param path: The new path_info or none to use the original. path - should NOT be quoted. When building a url, a Webob - Request (in accordance with wsgi spec) will quote - env['PATH_INFO']. url += quote(environ['PATH_INFO']) - :param query_string: The new query_string or none to use the original. - When building a url, a Webob Request will append - the query string directly to the url. - url += '?' + env['QUERY_STRING'] - :param agent: The HTTP user agent to use; default 'Swift'. You - can put %(orig)s in the agent to have it replaced - with the original env's HTTP_USER_AGENT, such as - '%(orig)s StaticWeb'. You also set agent to None to - use the original env's HTTP_USER_AGENT or '' to - have no HTTP_USER_AGENT. - :param swift_source: Used to mark the request as originating out of - middleware. Will be logged in proxy logs. - :returns: Fresh WSGI environment. - """ - newenv = {} - for name in ('HTTP_USER_AGENT', 'HTTP_HOST', 'PATH_INFO', - 'QUERY_STRING', 'REMOTE_USER', 'REQUEST_METHOD', - 'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT', - 'HTTP_ORIGIN', 'HTTP_ACCESS_CONTROL_REQUEST_METHOD', - 'SERVER_PROTOCOL', 'swift.cache', 'swift.source', - 'swift.trans_id', 'swift.authorize_override', - 'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID', - 'HTTP_REFERER', 'swift.infocache'): - if name in env: - newenv[name] = env[name] - if method: - newenv['REQUEST_METHOD'] = method - if path: - newenv['PATH_INFO'] = path - newenv['SCRIPT_NAME'] = '' - if query_string is not None: - newenv['QUERY_STRING'] = query_string - if agent: - newenv['HTTP_USER_AGENT'] = ( - agent % {'orig': env.get('HTTP_USER_AGENT', '')}).strip() - elif agent == '' and 'HTTP_USER_AGENT' in newenv: - del newenv['HTTP_USER_AGENT'] - if swift_source: - newenv['swift.source'] = swift_source - newenv['wsgi.input'] = BytesIO() - if 'SCRIPT_NAME' not in newenv: - newenv['SCRIPT_NAME'] = '' - return newenv - - -# Taken from swift/common/request_helpers.py, commit 1962b18 -def make_subrequest(env, method=None, path=None, body=None, headers=None, - agent='Swift', swift_source=None, make_env=make_env): - """ - Makes a new swob.Request based on the current env but with the - parameters specified. - - :param env: The WSGI environment to base the new request on. - :param method: HTTP method of new request; default is from - the original env. - :param path: HTTP path of new request; default is from the - original env. path should be compatible with what you - would send to Request.blank. path should be quoted and it - can include a query string. for example: - '/a%20space?unicode_str%E8%AA%9E=y%20es' - :param body: HTTP body of new request; empty by default. - :param headers: Extra HTTP headers of new request; None by - default. - :param agent: The HTTP user agent to use; default 'Swift'. You - can put %(orig)s in the agent to have it replaced - with the original env's HTTP_USER_AGENT, such as - '%(orig)s StaticWeb'. You also set agent to None to - use the original env's HTTP_USER_AGENT or '' to - have no HTTP_USER_AGENT. - :param swift_source: Used to mark the request as originating out of - middleware. Will be logged in proxy logs. - :param make_env: make_subrequest calls this make_env to help build the - swob.Request. - :returns: Fresh swob.Request object. - """ - query_string = None - path = path or '' - if path and '?' in path: - path, query_string = path.split('?', 1) - newenv = make_env(env, method, path=text_to_wsgi(unquote(path)), - agent=agent, query_string=query_string, - swift_source=swift_source) - if not headers: - headers = {} - if body: - return Request.blank(path, environ=newenv, body=body, headers=headers) - else: - return Request.blank(path, environ=newenv, headers=headers) - - -# Taken from swift/common/http.py, commit 1962b18 -def is_success(status): - """ - Check if HTTP status code is successful. - - :param status: http status code - :returns: True if status is successful, else False - """ - return 200 <= status <= 299 - - -# Taken from swift/common/utils.py, commit 1962b18 -_rfc_token = r'[^()<>@,;:\"/\[\]?={}\x00-\x20\x7f]+' -_rfc_extension_pattern = re.compile( - r'(?:\s*;\s*(' + _rfc_token + r")\s*(?:=\s*(" + _rfc_token + - r'|"(?:[^"\\]|\\.)*"))?)') - - -# Taken from swift/common/utils.py, commit 1962b18 -def parse_content_type(content_type): - """ - Parse a content-type and its parameters into values. - RFC 2616 sec 14.17 and 3.7 are pertinent. - - **Examples**:: - - 'text/plain; charset=UTF-8' -> ('text/plain', [('charset, 'UTF-8')]) - 'text/plain; charset=UTF-8; level=1' -> - ('text/plain', [('charset, 'UTF-8'), ('level', '1')]) - - :param content_type: content_type to parse - :returns: a tuple containing (content type, list of k, v parameter tuples) - """ - parm_list = [] - if ';' in content_type: - content_type, parms = content_type.split(';', 1) - parms = ';' + parms - for m in _rfc_extension_pattern.findall(parms): - key = m[0].strip() - value = m[1].strip() - parm_list.append((key, value)) - return content_type, parm_list - - -# Taken from swift/common/utils.py, commit 1962b18 -def parse_mime_headers(doc_file): - """ - Takes a file-like object containing a MIME document and returns a - HeaderKeyDict containing the headers. The body of the message is not - consumed: the position in doc_file is left at the beginning of the body. - - This function was inspired by the Python standard library's - http.client.parse_headers. - - :param doc_file: binary file-like object containing a MIME document - :returns: a swift.common.swob.HeaderKeyDict containing the headers - """ - headers = [] - while True: - line = doc_file.readline() - done = line in (b'\r\n', b'\n', b'') - if six.PY3: - try: - line = line.decode('utf-8') - except UnicodeDecodeError: - line = line.decode('latin1') - headers.append(line) - if done: - break - if six.PY3: - header_string = ''.join(headers) - else: - header_string = b''.join(headers) - headers = email.parser.Parser().parsestr(header_string) - return HeaderKeyDict(headers) - - -# Taken from swift/common/utils.py, commit 1962b18 -def maybe_multipart_byteranges_to_document_iters(app_iter, content_type): - """ - Takes an iterator that may or may not contain a multipart MIME document - as well as content type and returns an iterator of body iterators. - - :param app_iter: iterator that may contain a multipart MIME document - :param content_type: content type of the app_iter, used to determine - whether it conains a multipart document and, if - so, what the boundary is between documents - """ - content_type, params_list = parse_content_type(content_type) - if content_type != 'multipart/byteranges': - yield app_iter - return - - body_file = FileLikeIter(app_iter) - boundary = dict(params_list)['boundary'] - for _headers, body in mime_to_document_iters(body_file, boundary): - yield (chunk for chunk in iter(lambda: body.read(65536), '')) - - -# Taken from swift/common/utils.py, commit 1962b18 -def mime_to_document_iters(input_file, boundary, read_chunk_size=4096): - """ - Takes a file-like object containing a multipart MIME document and - returns an iterator of (headers, body-file) tuples. - - :param input_file: file-like object with the MIME doc in it - :param boundary: MIME boundary, sans dashes - (e.g. "divider", not "--divider") - :param read_chunk_size: size of strings read via input_file.read() - """ - doc_files = iter_multipart_mime_documents(input_file, boundary, - read_chunk_size) - for i, doc_file in enumerate(doc_files): - # this consumes the headers and leaves just the body in doc_file - headers = parse_mime_headers(doc_file) - yield (headers, doc_file) - - -# Taken from swift/common/utils.py, commit 1962b18 -# -# Modified to use different exception classes. -def iter_multipart_mime_documents(wsgi_input, boundary, read_chunk_size=4096): - """ - Given a multi-part-mime-encoded input file object and boundary, - yield file-like objects for each part. Note that this does not - split each part into headers and body; the caller is responsible - for doing that if necessary. - - :param wsgi_input: The file-like object to read from. - :param boundary: The mime boundary to separate new file-like - objects on. - :returns: A generator of file-like objects for each part. - :raises MimeInvalid: if the document is malformed - """ - boundary = '--' + boundary - blen = len(boundary) + 2 # \r\n - try: - got = wsgi_input.readline(blen) - while got == '\r\n': - got = wsgi_input.readline(blen) - except (IOError, ValueError) as e: - raise ChunkReadError(str(e)) - - if got.strip() != boundary: - raise MimeInvalid( - 'invalid starting boundary: wanted %r, got %r', (boundary, got)) - boundary = '\r\n' + boundary - input_buffer = '' - done = False - while not done: - it = _MultipartMimeFileLikeObject(wsgi_input, boundary, input_buffer, - read_chunk_size) - yield it - done = it.no_more_files - input_buffer = it.input_buffer - - -# Taken from swift/common/utils.py, commit 1962b18 -class FileLikeIter(object): - - def __init__(self, iterable): - """ - Wraps an iterable to behave as a file-like object. - - The iterable must yield bytes strings. - """ - self.iterator = iter(iterable) - self.buf = None - self.closed = False - - def __iter__(self): - return self - - def next(self): - """ - next(x) -> the next value, or raise StopIteration - """ - if self.closed: - raise ValueError('I/O operation on closed file') - if self.buf: - rv = self.buf - self.buf = None - return rv - else: - return next(self.iterator) - __next__ = next - - def read(self, size=-1): - """ - read([size]) -> read at most size bytes, returned as a bytes string. - - If the size argument is negative or omitted, read until EOF is reached. - Notice that when in non-blocking mode, less data than what was - requested may be returned, even if no size parameter was given. - """ - if self.closed: - raise ValueError('I/O operation on closed file') - if size < 0: - return b''.join(self) - elif not size: - chunk = b'' - elif self.buf: - chunk = self.buf - self.buf = None - else: - try: - chunk = next(self.iterator) - except StopIteration: - return b'' - if len(chunk) > size: - self.buf = chunk[size:] - chunk = chunk[:size] - return chunk - - def readline(self, size=-1): - """ - readline([size]) -> next line from the file, as a bytes string. - - Retain newline. A non-negative size argument limits the maximum - number of bytes to return (an incomplete line may be returned then). - Return an empty string at EOF. - """ - if self.closed: - raise ValueError('I/O operation on closed file') - data = b'' - while b'\n' not in data and (size < 0 or len(data) < size): - if size < 0: - chunk = self.read(1024) - else: - chunk = self.read(size - len(data)) - if not chunk: - break - data += chunk - if b'\n' in data: - data, sep, rest = data.partition(b'\n') - data += sep - if self.buf: - self.buf = rest + self.buf - else: - self.buf = rest - return data - - def readlines(self, sizehint=-1): - """ - readlines([size]) -> list of bytes strings, each a line from the file. - - Call readline() repeatedly and return a list of the lines so read. - The optional size argument, if given, is an approximate bound on the - total number of bytes in the lines returned. - """ - if self.closed: - raise ValueError('I/O operation on closed file') - lines = [] - while True: - line = self.readline(sizehint) - if not line: - break - lines.append(line) - if sizehint >= 0: - sizehint -= len(line) - if sizehint <= 0: - break - return lines - - def close(self): - """ - close() -> None or (perhaps) an integer. Close the file. - - Sets data attribute .closed to True. A closed file cannot be used for - further I/O operations. close() may be called more than once without - error. Some kinds of file objects (for example, opened by popen()) - may return an exit status upon closing. - """ - self.iterator = None - self.closed = True - - -# Taken from swift/common/request_helpers.py, commit 1962b18 -# -# Modified to use different exception classes. -class SegmentedIterable(object): - """ - Iterable that returns the object contents for a large object. - - :param req: original request object - :param app: WSGI application from which segments will come - - :param listing_iter: iterable yielding the object segments to fetch, - along with the byte subranges to fetch, in the form of a 5-tuple - (object-path, object-etag, object-size, first-byte, last-byte). - - If object-etag is None, no MD5 verification will be done. - - If object-size is None, no length verification will be done. - - If first-byte and last-byte are None, then the entire object will be - fetched. - - :param max_get_time: maximum permitted duration of a GET request (seconds) - :param logger: logger object - :param swift_source: value of swift.source in subrequest environ - (just for logging) - :param ua_suffix: string to append to user-agent. - :param name: name of manifest (used in logging only) - :param response_body_length: optional response body length for - the response being sent to the client. - """ - - def __init__(self, req, app, listing_iter, max_get_time, - logger, ua_suffix, swift_source, - name='', response_body_length=None): - self.req = req - self.app = app - self.listing_iter = listing_iter - self.max_get_time = max_get_time - self.logger = logger - self.ua_suffix = " " + ua_suffix - self.swift_source = swift_source - self.name = name - self.response_body_length = response_body_length - self.peeked_chunk = None - self.app_iter = self._internal_iter() - self.validated_first_segment = False - self.current_resp = None - - def _coalesce_requests(self): - start_time = time.time() - pending_req = None - pending_etag = None - pending_size = None - try: - for seg_path, seg_etag, seg_size, first_byte, last_byte \ - in self.listing_iter: - first_byte = first_byte or 0 - go_to_end = last_byte is None or ( - seg_size is not None and last_byte == seg_size - 1) - if time.time() - start_time > self.max_get_time: - raise SegmentError( - 'While processing manifest %s, ' - 'max LO GET time of %ds exceeded' % - (self.name, self.max_get_time)) - # The "multipart-manifest=get" query param ensures that the - # segment is a plain old object, not some flavor of large - # object; therefore, its etag is its MD5sum and hence we can - # check it. - path = seg_path + '?multipart-manifest=get' - seg_req = make_subrequest( - self.req.environ, path=path, method='GET', - headers={'x-auth-token': self.req.headers.get( - 'x-auth-token')}, - agent=('%(orig)s ' + self.ua_suffix), - swift_source=self.swift_source) - - seg_req_rangeval = None - if first_byte != 0 or not go_to_end: - seg_req_rangeval = "%s-%s" % ( - first_byte, '' if go_to_end else last_byte) - seg_req.headers['Range'] = "bytes=" + seg_req_rangeval - - # We can only coalesce if paths match and we know the segment - # size (so we can check that the ranges will be allowed) - if pending_req and pending_req.path == seg_req.path and \ - seg_size is not None: - - # Make a new Range object so that we don't goof up the - # existing one in case of invalid ranges. Note that a - # range set with too many individual byteranges is - # invalid, so we can combine N valid byteranges and 1 - # valid byterange and get an invalid range set. - if pending_req.range: - new_range_str = str(pending_req.range) - else: - new_range_str = "bytes=0-%d" % (seg_size - 1) - - if seg_req.range: - new_range_str += "," + seg_req_rangeval - else: - new_range_str += ",0-%d" % (seg_size - 1) - - if Range(new_range_str).ranges_for_length(seg_size): - # Good news! We can coalesce the requests - pending_req.headers['Range'] = new_range_str - continue - # else, Too many ranges, or too much backtracking, or ... - - if pending_req: - yield pending_req, pending_etag, pending_size - pending_req = seg_req - pending_etag = seg_etag - pending_size = seg_size - - except ListingIterError: - e_type, e_value, e_traceback = sys.exc_info() - if time.time() - start_time > self.max_get_time: - raise SegmentError( - 'While processing manifest %s, ' - 'max LO GET time of %ds exceeded' % - (self.name, self.max_get_time)) - if pending_req: - yield pending_req, pending_etag, pending_size - six.reraise(e_type, e_value, e_traceback) - - if time.time() - start_time > self.max_get_time: - raise SegmentError( - 'While processing manifest %s, ' - 'max LO GET time of %ds exceeded' % - (self.name, self.max_get_time)) - if pending_req: - yield pending_req, pending_etag, pending_size - - def _internal_iter(self): - bytes_left = self.response_body_length - - try: - for seg_req, seg_etag, seg_size in self._coalesce_requests(): - seg_resp = seg_req.get_response(self.app) - if not is_success(seg_resp.status_int): - close_if_possible(seg_resp.app_iter) - raise SegmentError( - 'While processing manifest %s, ' - 'got %d while retrieving %s' % - (self.name, seg_resp.status_int, seg_req.path)) - - elif ((seg_etag and (seg_resp.etag != seg_etag)) or - (seg_size and (seg_resp.content_length != seg_size) and - not seg_req.range)): - # The content-length check is for security reasons. Seems - # possible that an attacker could upload a >1mb object and - # then replace it with a much smaller object with same - # etag. Then create a big nested SLO that calls that - # object many times which would hammer our obj servers. If - # this is a range request, don't check content-length - # because it won't match. - close_if_possible(seg_resp.app_iter) - raise SegmentError( - 'Object segment no longer valid: ' - '%(path)s etag: %(r_etag)s != %(s_etag)s or ' - '%(r_size)s != %(s_size)s.' % - {'path': seg_req.path, 'r_etag': seg_resp.etag, - 'r_size': seg_resp.content_length, - 's_etag': seg_etag, - 's_size': seg_size}) - else: - self.current_resp = seg_resp - - seg_hash = None - if seg_resp.etag and not seg_req.headers.get('Range'): - # Only calculate the MD5 if it we can use it to validate - seg_hash = hashlib.md5() - - document_iters = maybe_multipart_byteranges_to_document_iters( - seg_resp.app_iter, - seg_resp.headers['Content-Type']) - - for chunk in itertools.chain.from_iterable(document_iters): - if seg_hash: - seg_hash.update(chunk) - - if bytes_left is None: - yield chunk - elif bytes_left >= len(chunk): - yield chunk - bytes_left -= len(chunk) - else: - yield chunk[:bytes_left] - bytes_left -= len(chunk) - close_if_possible(seg_resp.app_iter) - raise SegmentError( - 'Too many bytes for %(name)s; truncating in ' - '%(seg)s with %(left)d bytes left' % - {'name': self.name, 'seg': seg_req.path, - 'left': bytes_left}) - close_if_possible(seg_resp.app_iter) - - if seg_hash and seg_hash.hexdigest() != seg_resp.etag: - raise SegmentError( - "Bad MD5 checksum in %(name)s for %(seg)s: headers had" - " %(etag)s, but object MD5 was actually %(actual)s" % - {'seg': seg_req.path, 'etag': seg_resp.etag, - 'name': self.name, 'actual': seg_hash.hexdigest()}) - - if bytes_left: - raise SegmentError( - 'Not enough bytes for %s; closing connection' % self.name) - except (ListingIterError, SegmentError) as err: - self.logger.error(err) - if not self.validated_first_segment: - raise - finally: - if self.current_resp: - close_if_possible(self.current_resp.app_iter) - - def app_iter_range(self, *a, **kw): - """ - swob.Response will only respond with a 206 status in certain cases; one - of those is if the body iterator responds to .app_iter_range(). - - However, this object (or really, its listing iter) is smart enough to - handle the range stuff internally, so we just no-op this out for swob. - """ - return self - - def app_iter_ranges(self, ranges, content_type, boundary, content_size): - """ - This method assumes that iter(self) yields all the data bytes that - go into the response, but none of the MIME stuff. For example, if - the response will contain three MIME docs with data "abcd", "efgh", - and "ijkl", then iter(self) will give out the bytes "abcdefghijkl". - - This method inserts the MIME stuff around the data bytes. - """ - si = Spliterator(self) - mri = multi_range_iterator( - ranges, content_type, boundary, content_size, - lambda start, end_plus_one: si.take(end_plus_one - start)) - try: - for x in mri: - yield x - finally: - self.close() - - def validate_first_segment(self): - """ - Start fetching object data to ensure that the first segment (if any) is - valid. This is to catch cases like "first segment is missing" or - "first segment's etag doesn't match manifest". - - Note: this does not validate that you have any segments. A - zero-segment large object is not erroneous; it is just empty. - """ - if self.validated_first_segment: - return - - try: - self.peeked_chunk = next(self.app_iter) - except StopIteration: - pass - finally: - self.validated_first_segment = True - - def __iter__(self): - if self.peeked_chunk is not None: - pc = self.peeked_chunk - self.peeked_chunk = None - return itertools.chain([pc], self.app_iter) - else: - return self.app_iter - - def close(self): - """ - Called when the client disconnect. Ensure that the connection to the - backend server is closed. - """ - close_if_possible(self.app_iter) - - -# Taken from swift/common/utils.py, commit e001c02 -def list_from_csv(comma_separated_str): - """ - Splits the str given and returns a properly stripped list of the comma - separated values. - """ - if comma_separated_str: - return [v.strip() for v in comma_separated_str.split(',') if v.strip()] - return [] - - -# Taken from swift/common/request_helpers.py, commit e001c02 -def resolve_etag_is_at_header(req, metadata): - """ - Helper function to resolve an alternative etag value that may be stored in - metadata under an alternate name. - - The value of the request's X-Backend-Etag-Is-At header (if it exists) is a - comma separated list of alternate names in the metadata at which an - alternate etag value may be found. This list is processed in order until an - alternate etag is found. - - The left most value in X-Backend-Etag-Is-At will have been set by the left - most middleware, or if no middleware, by ECObjectController, if an EC - policy is in use. The left most middleware is assumed to be the authority - on what the etag value of the object content is. - - The resolver will work from left to right in the list until it finds a - value that is a name in the given metadata. So the left most wins, IF it - exists in the metadata. - - By way of example, assume the encrypter middleware is installed. If an - object is *not* encrypted then the resolver will not find the encrypter - middleware's alternate etag sysmeta (X-Object-Sysmeta-Crypto-Etag) but will - then find the EC alternate etag (if EC policy). But if the object *is* - encrypted then X-Object-Sysmeta-Crypto-Etag is found and used, which is - correct because it should be preferred over X-Object-Sysmeta-Crypto-Etag. - - :param req: a swob Request - :param metadata: a dict containing object metadata - :return: an alternate etag value if any is found, otherwise None - """ - alternate_etag = None - metadata = HeaderKeyDict(metadata) - if "X-Backend-Etag-Is-At" in req.headers: - names = list_from_csv(req.headers["X-Backend-Etag-Is-At"]) - for name in names: - if name in metadata: - alternate_etag = metadata[name] - break - return alternate_etag - - -# Taken from swift/proxy/controllers/container.py, commit e001c02 -# -# Modified to no longer dangle off a controller -def clean_acls(req): - if 'swift.clean_acl' in req.environ: - for header in ('x-container-read', 'x-container-write'): - if header in req.headers: - try: - req.headers[header] = \ - req.environ['swift.clean_acl'](header, - req.headers[header]) - except ValueError as err: - return HTTPBadRequest(request=req, body=str(err)) - return None - - -# Taken from swift/common/swob.py, commit a5a6a27 -# -# Modified to pass through Nones -def wsgi_to_str(wsgi_str): - if six.PY2 or wsgi_str is None: - return wsgi_str - return wsgi_str.encode('latin1').decode('utf8', errors='surrogateescape') - - -def str_to_wsgi(native_str): - if six.PY2 or native_str is None: - return native_str - return native_str.encode('utf8', errors='surrogateescape').decode('latin1') - - -def bytes_to_wsgi(byte_str): - if six.PY2: - return byte_str - return byte_str.decode('latin1') - - -# this isn't in swift -def text_to_wsgi(text): - if six.PY2 and isinstance(text, six.text_type): - return text.encode('utf-8') - return text diff --git a/pfs_middleware/pfs_middleware/utils.py b/pfs_middleware/pfs_middleware/utils.py deleted file mode 100644 index cf9767ee..00000000 --- a/pfs_middleware/pfs_middleware/utils.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -import contextlib -import eventlet -import json -import socket -import re -from six.moves.urllib import parse as urllib_parse - - -ENV_IS_BIMODAL = 'pfs.is_bimodal' -ENV_OWNING_PROXYFS = 'pfs.owning_proxyfs' -ENV_BIMODAL_CHECKER = 'pfs.bimodal_checker' - - -class RpcError(Exception): - def __init__(self, errno, *a, **kw): - self.errno = errno - super(RpcError, self).__init__(*a, **kw) - - -class RpcTimeout(Exception): - pass - - -class NoSuchHostnameError(Exception): - pass - - -def parse_path(path): - """ - Takes the path component of an URL and returns a 4-element list of (API - version, account, container, object). - - Similar to, but distinct from, swift.common.utils.split_path(). - - >>> parse_path("/v1/AUTH_test/con/obj") - ("v1", "AUTH_test", "con", "obj") - >>> parse_path("/v1/AUTH_test/con") - ("v1", "AUTH_test", "con", None) - >>> parse_path("/info") - ("info", None, None, None) - """ - segs = urllib_parse.unquote(path).split('/', 4) - # throw away the first segment; paths start with /, so the first segment - # is always empty - segs = [(s if s else None) for s in segs[1:]] - if len(segs) < 4: - segs.extend([None] * (4 - len(segs))) - return segs - - -PFS_ERRNO_RE = re.compile(r'^errno: (\d+)$') - - -def extract_errno(errstr): - """ - Given an error response from a proxyfs RPC, extracts the error number - from it, or None if the error isn't in the usual format. - """ - # A proxyfs error response looks like "errno: 18" - m = re.match(PFS_ERRNO_RE, errstr) - if m: - return int(m.group(1)) - - -class JsonRpcClient(object): - def __init__(self, addrinfo): - self.addrinfo = addrinfo - - def call(self, rpc_request, timeout, raise_on_rpc_error=True): - """ - Call a remote procedure using JSON-RPC. - - :param rpc_request: Python dictionary containing the request - (method, args, etc.) in JSON-RPC format. - - :returns: the (deserialized) result of the RPC, whatever that looks - like - - :raises: socket.error if the TCP connection breaks - :raises: RpcTimeout if the RPC takes too long - :raises: ValueError if the response is not valid JSON - :raises: RpcError if the RPC returns an error and raise_on_rpc_error - is True - """ - serialized_req = json.dumps(rpc_request).encode("utf-8") - - # for the format of self.addrinfo, see the docs for - # socket.getaddrinfo() - addr_family = self.addrinfo[0] - sock_type = self.addrinfo[1] - sock_proto = self.addrinfo[2] - # self.addrinfo[3] is the canonical name, which isn't useful here - addr = self.addrinfo[4] - - # XXX TODO keep sockets around in a pool or something? - sock = socket.socket(addr_family, sock_type, sock_proto) - try: - with eventlet.Timeout(timeout): - sock.connect(addr) - with contextlib.closing(sock): - sock.send(serialized_req) - # This is JSON-RPC over TCP: we write a JSON document to - # the socket, then read a JSON document back. The only - # way we know when we're done is when what we've read is - # valid JSON. - # - # Of course, Python's builtin JSON can't consume just - # one document from a file, so for now, we'll use the - # fact that the sender is sending one JSON object per - # line and just call readline(). At some point, we - # should replace this with a real incremental JSON - # parser like ijson. - sock_filelike = sock.makefile("r") - with contextlib.closing(sock_filelike): - line = sock_filelike.readline() - try: - response = json.loads(line) - except ValueError as err: - raise ValueError( - "Error decoding JSON: %s (tried to decode %r)" - % (err, line)) - - errstr = response.get("error") - if errstr and raise_on_rpc_error: - errno = extract_errno(errstr) - raise RpcError(errno, "Error in %s: %s" % ( - rpc_request.get("method", ""), - errstr)) - return response - except eventlet.Timeout as exc: - raise RpcTimeout( - "Timeout (%.6fs) communicating with %s:%d, calling %s" - % (exc, addr[0], addr[1], - rpc_request.get("method", ""))) diff --git a/pfs_middleware/setup.py b/pfs_middleware/setup.py deleted file mode 100644 index d834a314..00000000 --- a/pfs_middleware/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -from setuptools import setup - - -try: - import multiprocessing.util # noqa -except ImportError: - pass - - -setup( - name="pfs_middleware", - version="0.0.1", - author="SwiftStack Inc.", - description="WSGI middleware component of ProxyFS", - license="Apache", - packages=["pfs_middleware"], - - test_suite="tests", - tests_require=['mock'], - - # TODO: classifiers and such - entry_points={ - 'paste.filter_factory': [ - 'pfs = pfs_middleware:filter_factory', - ], - }, -) diff --git a/pfs_middleware/test-requirements.txt b/pfs_middleware/test-requirements.txt deleted file mode 100644 index 932a8957..00000000 --- a/pfs_middleware/test-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -mock diff --git a/pfs_middleware/tests/__init__.py b/pfs_middleware/tests/__init__.py deleted file mode 100644 index b93cd683..00000000 --- a/pfs_middleware/tests/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -# Work around http://bugs.python.org/issue9501 -import logging -logging.raiseExceptions = False diff --git a/pfs_middleware/tests/helpers.py b/pfs_middleware/tests/helpers.py deleted file mode 100644 index 739a6d66..00000000 --- a/pfs_middleware/tests/helpers.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -import collections -from swift.common import swob - - -class FakeProxy(object): - """ - Vaguely Swift-proxy-server-ish WSGI application used in testing the - ProxyFS middleware. - """ - def __init__(self): - self._calls = [] - - # key is a 2-tuple (request-method, url-path) - # - # value is how many WSGI iterables were created but not destroyed; - # if all is well-behaved, the value should be 0 upon completion of - # the user's request. - self._unclosed_req_paths = collections.defaultdict(int) - - # key is a 2-tuple (request-method, url-path) - # - # value is a 3-tuple (response status, headers, body) - self._responses = {} - - @property - def calls(self): - return tuple(self._calls) - - def register(self, method, path, response_status, headers, body=''): - self._responses[(method, path)] = (response_status, headers, body) - - def __call__(self, env, start_response): - method = env['REQUEST_METHOD'] - path = env['PATH_INFO'] - - req = swob.Request(env) - self._calls.append((method, path, swob.HeaderKeyDict(req.headers))) - - if (env.get('swift.authorize') and not env.get( - 'swift.authorize_override')): - denial_response = env['swift.authorize'](req) - if denial_response: - return denial_response - - try: - status_int, headers, body = self._responses[(method, path)] - except KeyError: - print("Didn't find \"%s %s\" in registered responses" % ( - method, path)) - raise - - if method in ('PUT', 'COALESCE'): - bytes_read = 0 - # consume the whole request body, just like a PUT would - for chunk in iter(env['wsgi.input'].read, b''): - bytes_read += len(chunk) - cl = req.headers.get('Content-Length') - - if cl is not None and int(cl) != bytes_read: - error_resp = swob.HTTPClientDisconnect( - request=req, - body=("Content-Length didn't match" - " body length (says FakeProxy)")) - return error_resp(env, start_response) - - if cl is None \ - and "chunked" not in req.headers.get("Transfer-Encoding", ""): - error_resp = swob.HTTPLengthRequired( - request=req, - body="No Content-Length (says FakeProxy)") - return error_resp(env, start_response) - - resp = swob.Response( - body=body, status=status_int, headers=headers, - # We cheat a little here and use swob's handling of the Range - # header instead of doing it ourselves. - conditional_response=True) - - return resp(env, start_response) - - -class FakeJsonRpc(object): - """ - Fake out JSON-RPC calls. - - This object is used to replace the JsonRpcClient helper object. - """ - def __init__(self): - self._calls = [] - self._rpc_handlers = {} - - @property - def calls(self): - return tuple(self._calls) - - def register_handler(self, method, handler): - """ - :param method: name of JSON-RPC method, e.g. Server.RpcGetObject - - :param handler: callable to handle that method. Callable must take - one JSON-RPC object (dict) as argument and return a single - JSON-RPC object (dict). - """ - self._rpc_handlers[method] = handler - - def call(self, rpc_request, _timeout, raise_on_rpc_error=True): - # Note: rpc_request here is a JSON-RPC request object. In Python - # terms, it's a dictionary with a particular format. - - call_id = rpc_request.get('id') - - # let any exceptions out so callers can see them and fix their - # request generators - assert rpc_request['jsonrpc'] == '2.0' - method = rpc_request['method'] - - # rpc.* are reserved by JSON-RPC 2.0 - assert not method.startswith("rpc.") - - # let KeyError out so callers can see it and fix their mocks - handler = self._rpc_handlers[method] - - # params may be omitted, a list (positional), or a dict (by name) - if 'params' not in rpc_request: - rpc_response = handler() - self._calls.append((method, ())) - elif isinstance(rpc_request['params'], (list, tuple)): - rpc_response = handler(*(rpc_request['params'])) - self._calls.append((method, tuple(rpc_request['params']))) - elif isinstance(rpc_request['params'], (dict,)): - raise NotImplementedError("haven't needed this yet") - else: - raise ValueError( - "FakeJsonRpc can't handle params of type %s (%r)" % - (type(rpc_request['params']), rpc_request['params'])) - - if call_id is None: - # a JSON-RPC request without an "id" parameter is a - # "notification", i.e. a special request that receives no - # response. - return None - elif 'id' in rpc_response and rpc_response['id'] != call_id: - # We don't enforce that the handler pay any attention to 'id', - # but if it does, it has to get it right. - raise ValueError("handler for %s set 'id' attr to bogus value" % - (method,)) - else: - rpc_response['id'] = call_id - return rpc_response diff --git a/pfs_middleware/tests/test_bimodal_checker.py b/pfs_middleware/tests/test_bimodal_checker.py deleted file mode 100644 index 82e926d1..00000000 --- a/pfs_middleware/tests/test_bimodal_checker.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -import errno -import mock -import os -import socket -import unittest - -from swift.common import swob -from . import helpers - -import pfs_middleware.bimodal_checker as bimodal_checker -import pfs_middleware.utils as utils - - -class FakeJsonRpcWithErrors(helpers.FakeJsonRpc): - def __init__(self, *a, **kw): - super(FakeJsonRpcWithErrors, self).__init__(*a, **kw) - self._errors = [] - - def add_call_error(self, ex): - self._errors.append(ex) - - def call(self, *a, **kw): - # Call super() so we get call tracking. - retval = super(FakeJsonRpcWithErrors, self).call(*a, **kw) - if self._errors: - raise self._errors.pop(0) - return retval - - -class BimodalHeaderinator(object): - def __init__(self, app): - self.app = app - - def __call__(self, environ, start_response): - is_bimodal = bool(environ.get(utils.ENV_IS_BIMODAL)) - - def my_sr(status, headers, exc_info=None): - my_headers = list(headers) - my_headers.append(("Is-Bimodal", ('yes' if is_bimodal else 'no'))) - return start_response(status, my_headers, exc_info) - - return self.app(environ, my_sr) - - -class TestDecision(unittest.TestCase): - def setUp(self): - self.app = helpers.FakeProxy() - bh = BimodalHeaderinator(self.app) - self.bc = bimodal_checker.BimodalChecker(bh, { - 'bimodal_recheck_interval': '5.0', - }) - self.fake_rpc = helpers.FakeJsonRpc() - patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', - lambda *_: self.fake_rpc) - patcher.start() - self.addCleanup(patcher.stop) - - self.app.register('HEAD', '/v1/alice', 204, {}, '') - self.app.register('HEAD', '/v1/bob', 204, - {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, - '') - self.app.register('HEAD', '/v1/carol', 204, - {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, - '') - - def fake_RpcIsAccountBimodal(request): - return { - "error": None, - "result": { - "IsBimodal": True, - "ActivePeerPrivateIPAddr": - "fc00:a377:29bc:fc90:9808:ba1f:e94b:1215"}} - - self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", - fake_RpcIsAccountBimodal) - - def test_not_bimodal(self): - req = swob.Request.blank("/v1/alice", - environ={"REQUEST_METHOD": "HEAD"}) - resp = req.get_response(self.bc) - - self.assertEqual("no", resp.headers["Is-Bimodal"]) - # Swift said no, so we didn't bother with an RPC. - self.assertEqual(tuple(), self.fake_rpc.calls) - - def test_bimodal(self): - req = swob.Request.blank("/v1/bob", - environ={"REQUEST_METHOD": "HEAD"}) - resp = req.get_response(self.bc) - - self.assertEqual("yes", resp.headers["Is-Bimodal"]) - self.assertEqual(self.fake_rpc.calls, ( - ('Server.RpcIsAccountBimodal', ({'AccountName': 'bob'},)),)) - - def test_bad_path(self): - self.app.register('GET', '//v1/alice', 404, {}, '') - req = swob.Request.blank("//v1/alice") - # (in)sanity check - self.assertNotEqual(req.environ['PATH_INFO'], '//v1/alice') - # fix it - req.environ['PATH_INFO'] = '//v1/alice' - resp = req.get_response(self.bc) - - self.assertEqual("no", resp.headers["Is-Bimodal"]) - # We just passed through -- didn't bother with an RPC. - self.assertEqual(tuple(), self.fake_rpc.calls) - - def test_disagreement(self): - def fake_RpcIsAccountBimodal(request): - return { - "error": None, - "result": {"IsBimodal": False, - "ActivePeerPrivateIPAddr": ""}} - - self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", - fake_RpcIsAccountBimodal) - - req = swob.Request.blank("/v1/bob", - environ={"REQUEST_METHOD": "HEAD"}) - resp = req.get_response(self.bc) - - self.assertEqual(resp.status_int, 503) - - -class TestBimodalCaching(unittest.TestCase): - def setUp(self): - self.app = helpers.FakeProxy() - self.bc = bimodal_checker.BimodalChecker(self.app, { - 'bimodal_recheck_interval': '5.0', - }) - self.fake_rpc = helpers.FakeJsonRpc() - patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', - lambda *_: self.fake_rpc) - patcher.start() - self.addCleanup(patcher.stop) - - self.app.register('HEAD', '/v1/alice', 204, {}, '') - self.app.register('HEAD', '/v1/bob', 204, - {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, - '') - self.app.register('HEAD', '/v1/carol', 204, - {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, - '') - self.app.register('HEAD', '/v1/dave', 204, - {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, - '') - - def test_caching(self): - the_time = [12345.6] - rpc_iab_calls = [] - - def fake_time_function(): - now = the_time[0] - the_time[0] += 0.001 - return now - - fake_time_module = mock.Mock(time=fake_time_function) - - def fake_RpcIsAccountBimodal(request): - acc = request["AccountName"] - rpc_iab_calls.append(acc) - - if acc == 'alice': - # Not bimodal - return { - "error": None, - "result": { - "IsBimodal": False, - "ActivePeerPrivateIPAddr": ""}} - elif acc == 'bob': - # Normal, happy bimodal account - return { - "error": None, - "result": { - "IsBimodal": True, - "ActivePeerPrivateIPAddr": "10.221.76.210"}} - elif acc == 'carol': - # Temporarily in limbo as it's being moved from one proxyfsd - # to another - return { - "error": None, - "result": { - "IsBimodal": True, - "ActivePeerPrivateIPAddr": ""}} - elif acc == 'dave': - # Another bimodal account - return { - "error": None, - "result": { - "IsBimodal": True, - "ActivePeerPrivateIPAddr": "10.221.76.210"}} - else: - raise ValueError("test helper can't handle %r" % (acc,)) - self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", - fake_RpcIsAccountBimodal) - - status = [None] - - def start_response(s, h, ei=None): - status[0] = s - - with mock.patch('pfs_middleware.bimodal_checker.time', - fake_time_module): - a_req = swob.Request.blank("/v1/alice", - environ={"REQUEST_METHOD": "HEAD"}) - b_req = swob.Request.blank("/v1/bob", - environ={"REQUEST_METHOD": "HEAD"}) - c_req = swob.Request.blank("/v1/carol", - environ={"REQUEST_METHOD": "HEAD"}) - d_req = swob.Request.blank("/v1/dave", - environ={"REQUEST_METHOD": "HEAD"}) - - # Negative results are served without any RPCs at all - list(self.bc(a_req.environ, start_response)) - self.assertEqual(status[0], '204 No Content') # sanity check - self.assertEqual(rpc_iab_calls, []) - - # First time, we have a completely empty cache, so an RPC is made - list(self.bc(d_req.environ, start_response)) - self.assertEqual(status[0], '204 No Content') # sanity check - self.assertEqual(rpc_iab_calls, ['dave']) - - # A couple seconds later, a second request for the same account - # comes in, and is handled from cache - the_time[0] += 2 - del rpc_iab_calls[:] - list(self.bc(d_req.environ, start_response)) - self.assertEqual(status[0], '204 No Content') # sanity check - self.assertEqual(rpc_iab_calls, []) - - # If a request for another account comes in, it is cached - # separately. - del rpc_iab_calls[:] - list(self.bc(b_req.environ, start_response)) - self.assertEqual(status[0], '204 No Content') # sanity check - self.assertEqual(rpc_iab_calls, ["bob"]) - - # Each account has its own cache time - the_time[0] += 3 # "dave" is now invalid, "bob" remains valid - del rpc_iab_calls[:] - list(self.bc(d_req.environ, start_response)) - list(self.bc(b_req.environ, start_response)) - self.assertEqual(rpc_iab_calls, ["dave"]) - - # In-transit accounts don't get cached - del rpc_iab_calls[:] - list(self.bc(c_req.environ, start_response)) - list(self.bc(c_req.environ, start_response)) - self.assertEqual(rpc_iab_calls, ["carol", "carol"]) - - -class TestRetry(unittest.TestCase): - def setUp(self): - self.app = helpers.FakeProxy() - self.bc = bimodal_checker.BimodalChecker(self.app, { - 'bimodal_recheck_interval': '5.0', - 'proxyfsd_host': '10.1.1.1, 10.2.2.2', - }) - self.fake_rpc = FakeJsonRpcWithErrors() - patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', - lambda *_: self.fake_rpc) - patcher.start() - self.addCleanup(patcher.stop) - self.app.register('HEAD', '/v1/AUTH_test', 204, - {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, - '') - - def fake_RpcIsAccountBimodal(request): - return { - "error": None, - "result": { - "IsBimodal": True, - "ActivePeerPrivateIPAddr": "10.9.8.7", - }} - - self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", - fake_RpcIsAccountBimodal) - - def test_retry_socketerror(self): - self.fake_rpc.add_call_error( - socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED))) - - req = swob.Request.blank( - "/v1/AUTH_test", - environ={'REQUEST_METHOD': 'HEAD'}) - resp = req.get_response(self.bc) - self.assertEqual(resp.status_int, 204) - - self.assertEqual(self.fake_rpc.calls, ( - ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)), - ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)))) - - def test_no_retry_timeout(self): - err = utils.RpcTimeout() - self.fake_rpc.add_call_error(err) - - req = swob.Request.blank( - "/v1/AUTH_test", - environ={'REQUEST_METHOD': 'HEAD'}) - resp = req.get_response(self.bc) - self.assertEqual(resp.status_int, 503) - - self.assertEqual(self.fake_rpc.calls, ( - ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)),)) - - def test_no_catch_other_error(self): - self.fake_rpc.add_call_error(ZeroDivisionError) - - req = swob.Request.blank( - "/v1/AUTH_test", - environ={'REQUEST_METHOD': 'HEAD'}) - - self.assertRaises(ZeroDivisionError, req.get_response, self.bc) - self.assertEqual(self.fake_rpc.calls, ( - ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)),)) diff --git a/pfs_middleware/tests/test_pfs_middleware.py b/pfs_middleware/tests/test_pfs_middleware.py deleted file mode 100644 index 0fb4237c..00000000 --- a/pfs_middleware/tests/test_pfs_middleware.py +++ /dev/null @@ -1,4882 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -import base64 -import collections -import hashlib -import json -import mock -import unittest -from io import BytesIO -from swift.common import swob -from xml.etree import ElementTree - -import pfs_middleware.middleware as mware -import pfs_middleware.bimodal_checker as bimodal_checker -from . import helpers - - -class FakeLogger(object): - def critical(fmt, *args): - pass - - def exception(fmt, *args): - pass - - def error(fmt, *args): - pass - - def warn(fmt, *args): - pass - - def info(fmt, *args): - pass - - def debug(fmt, *args, **kwargs): - pass - - -class TestHelpers(unittest.TestCase): - def test_deserialize_metadata(self): - self.assertEqual(mware.deserialize_metadata(None), {}) - self.assertEqual(mware.deserialize_metadata(''), {}) - self.assertEqual(mware.deserialize_metadata('{}'), {}) - self.assertEqual(mware.deserialize_metadata('{"foo": "bar"}'), - {"foo": "bar"}) - self.assertEqual(mware.deserialize_metadata( - '{"unicode-\\u1234":"meta \\ud83c\\udf34"}' - ), { - # NB: WSGI strings - 'unicode-\xe1\x88\xb4': 'meta \xf0\x9f\x8c\xb4', - }) - - def test_serialize_metadata(self): - self.assertEqual(mware.serialize_metadata({}), '{}') - self.assertEqual( - mware.serialize_metadata({ - # NB: WSGI strings - 'unicode-\xe1\x88\xb4': 'meta \xf0\x9f\x8c\xb4', - }), - # But it comes out as good Unicode - '{"unicode-\\u1234": "meta \\ud83c\\udf34"}') - - -class BaseMiddlewareTest(unittest.TestCase): - # no test cases in here, just common setup and utility functions - def setUp(self): - super(BaseMiddlewareTest, self).setUp() - self.app = helpers.FakeProxy() - self.pfs = mware.PfsMiddleware(self.app, { - 'bypass_mode': 'read-only', - }, FakeLogger()) - self.bimodal_checker = bimodal_checker.BimodalChecker(self.pfs, { - 'bimodal_recheck_interval': 'inf', # avoid timing dependencies - }, FakeLogger()) - self.bimodal_accounts = {"AUTH_test"} - - self.app.register('HEAD', '/v1/AUTH_test', 204, - {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, - '') - - self.swift_info = { - # some stuff omitted - "slo": { - "max_manifest_segments": 1000, - "max_manifest_size": 2097152, - "min_segment_size": 1}, - "swift": { - "account_autocreate": True, - "account_listing_limit": 9876, # note: not default - "allow_account_management": True, - "container_listing_limit": 6543, # note: not default - "extra_header_count": 0, - "max_account_name_length": 128, - "max_container_name_length": 256, - "max_file_size": 5368709122, - "max_header_size": 8192, - "max_meta_count": 90, - "max_meta_name_length": 128, - "max_meta_overall_size": 4096, - "max_meta_value_length": 256, - "max_object_name_length": 1024, - "policies": [{ - "aliases": "default", - "default": True, - "name": "default", - }, { - "aliases": "not-default", - "name": "not-default", - }], - "strict_cors_mode": True, - "version": "2.9.1.dev47" - }, - "tempauth": {"account_acls": True}} - - self.app.register( - 'GET', '/info', - 200, {'Content-Type': 'application/json'}, - json.dumps(self.swift_info)) - - for method in ('GET', 'HEAD', 'PUT', 'POST', 'DELETE'): - self.app.register( - method, '/v1/AUTH_test//o', - 412, {'Content-Type': 'text/html'}, 'Bad URL') - - self.fake_rpc = helpers.FakeJsonRpc() - patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', - lambda *_: self.fake_rpc) - patcher.start() - self.addCleanup(patcher.stop) - - # For the sake of the listing format tests, assume old Swift - patcher = mock.patch( - 'pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', False) - patcher.start() - self.addCleanup(patcher.stop) - - def fake_RpcIsAccountBimodal(request): - return { - "error": None, - "result": { - "IsBimodal": - request["AccountName"] in self.bimodal_accounts, - "ActivePeerPrivateIPAddr": "127.0.0.1", - }} - - self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", - fake_RpcIsAccountBimodal) - - def call_app(self, req, app=None, expect_exception=False): - # Normally this happens in eventlet.wsgi.HttpProtocol.get_environ(). - req.environ.setdefault('CONTENT_TYPE', None) - - if app is None: - app = self.app - - status = [None] - headers = [None] - - def start_response(s, h, ei=None): - status[0] = s - headers[0] = swob.HeaderKeyDict(h) - - body_iter = app(req.environ, start_response) - body = b'' - caught_exc = None - try: - try: - for chunk in body_iter: - body += chunk - finally: - # WSGI says we have to do this. Plus, if we don't, then - # our leak detector gets false positives. - if callable(getattr(body_iter, 'close', None)): - body_iter.close() - except Exception as exc: - if expect_exception: - caught_exc = exc - else: - raise - - if expect_exception: - return status[0], headers[0], body, caught_exc - else: - return status[0], headers[0], body - - def call_pfs(self, req, **kwargs): - return self.call_app(req, app=self.bimodal_checker, **kwargs) - - -class TestAccountGet(BaseMiddlewareTest): - def setUp(self): - super(TestAccountGet, self).setUp() - - self.app.register( - 'HEAD', '/v1/AUTH_test', - 204, - {'X-Account-Meta-Flavor': 'cherry', - 'X-Account-Sysmeta-Shipping-Class': 'ultraslow', - 'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, - '') - - def mock_RpcGetAccount(_): - return { - "error": None, - "result": { - "ModificationTime": 1498766381451119000, - "AccountEntries": [{ - "Basename": "chickens", - "ModificationTime": 1510958440808682000, - }, { - "Basename": "cows", - "ModificationTime": 1510958450657045000, - }, { - "Basename": "goats", - "ModificationTime": 1510958452544251000, - }, { - "Basename": "pigs", - "ModificationTime": 1510958459200130000, - }], - }} - - self.fake_rpc.register_handler( - "Server.RpcGetAccount", mock_RpcGetAccount) - - def test_headers(self): - req = swob.Request.blank("/v1/AUTH_test") - status, headers, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers.get("Accept-Ranges"), "bytes") - self.assertEqual(headers.get("X-Timestamp"), "1498766381.45112") - - # These we just lie about for now - self.assertEqual( - headers.get("X-Account-Object-Count"), "0") - self.assertEqual( - headers.get("X-Account-Storage-Policy-Default-Object-Count"), "0") - self.assertEqual( - headers.get("X-Account-Bytes-Used"), "0") - self.assertEqual( - headers.get("X-Account-Storage-Policy-Default-Bytes-Used"), "0") - - # We pretend all our containers are in the default storage policy. - self.assertEqual( - headers.get("X-Account-Container-Count"), "4") - self.assertEqual( - headers.get("X-Account-Storage-Policy-Default-Container-Count"), - "4") - - def test_escape_hatch(self): - self.app.register( - 'GET', '/v1/AUTH_test', 200, {}, - '000000000000DACA\n000000000000DACC\n') - - # Using path - req = swob.Request.blank("/proxyfs/AUTH_test", environ={ - 'swift_owner': True}) - status, _, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'000000000000DACA\n000000000000DACC\n') - - # Non-bypass - req = swob.Request.blank("/v1/AUTH_test", environ={ - 'swift_owner': True}) - status, _, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'chickens\ncows\ngoats\npigs\n') - - req = swob.Request.blank("/proxyfs/AUTH_test", method='PUT', - environ={'swift_owner': True}) - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '405 Method Not Allowed') - - # Non-owner - req = swob.Request.blank("/proxyfs/AUTH_test") - status, _, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'chickens\ncows\ngoats\npigs\n') - - def test_text(self): - req = swob.Request.blank("/v1/AUTH_test") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, b"chickens\ncows\ngoats\npigs\n") - - def test_json(self): - self.maxDiff = None - - req = swob.Request.blank("/v1/AUTH_test?format=json") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(json.loads(body), [{ - "bytes": 0, - "count": 0, - "last_modified": "2017-11-17T22:40:40.808682", - "name": "chickens", - }, { - "bytes": 0, - "count": 0, - "last_modified": "2017-11-17T22:40:50.657045", - "name": "cows", - }, { - "bytes": 0, - "count": 0, - "last_modified": "2017-11-17T22:40:52.544251", - "name": "goats", - }, { - "bytes": 0, - "count": 0, - "last_modified": "2017-11-17T22:40:59.200130", - "name": "pigs" - }]) - - def test_xml(self): - req = swob.Request.blank("/v1/AUTH_test?format=xml") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], - "application/xml; charset=utf-8") - self.assertTrue(body.startswith( - b"""""")) - - root_node = ElementTree.fromstring(body) - self.assertEqual(root_node.tag, 'account') - self.assertEqual(root_node.attrib["name"], 'AUTH_test') - - containers = list(root_node) - self.assertEqual(containers[0].tag, 'container') - - # The XML account listing doesn't use XML attributes for data, but - # rather a sequence of tags like X Y ... - con_attr_tags = list(containers[0]) - self.assertEqual(len(con_attr_tags), 4) - - name_node = con_attr_tags[0] - self.assertEqual(name_node.tag, 'name') - self.assertEqual(name_node.text, 'chickens') - self.assertEqual(name_node.attrib, {}) # nothing extra in there - - count_node = con_attr_tags[1] - self.assertEqual(count_node.tag, 'count') - self.assertEqual(count_node.text, '0') - self.assertEqual(count_node.attrib, {}) - - bytes_node = con_attr_tags[2] - self.assertEqual(bytes_node.tag, 'bytes') - self.assertEqual(bytes_node.text, '0') - self.assertEqual(bytes_node.attrib, {}) - - lm_node = con_attr_tags[3] - self.assertEqual(lm_node.tag, 'last_modified') - self.assertEqual(lm_node.text, '2017-11-17T22:40:40.808682') - self.assertEqual(lm_node.attrib, {}) - - def test_metadata(self): - req = swob.Request.blank("/v1/AUTH_test") - status, headers, body = self.call_pfs(req) - self.assertEqual(headers.get("X-Account-Meta-Flavor"), "cherry") - self.assertEqual(headers.get("X-Account-Sysmeta-Shipping-Class"), - "ultraslow") - - def test_account_acl(self): - def do_test(acl, expected_header, swift_owner): - head_resp_hdrs = {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'} - if acl is not None: - head_resp_hdrs['X-Account-Sysmeta-Core-Access-Control'] = acl - self.app.register( - 'HEAD', '/v1/AUTH_test', 204, head_resp_hdrs, '') - req = swob.Request.blank("/v1/AUTH_test", - environ={"swift_owner": swift_owner}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, "200 OK") - actual_header = headers.get("X-Account-Access-Control") - self.assertEqual(expected_header, actual_header) - - # swift_owner - do_test(None, None, True) - do_test('', None, True) - do_test('not a dict', None, True) - do_test('{"admin": ["someone"]}', '{"admin":["someone"]}', True) - # not swift_owner - do_test(None, None, False) - do_test('', None, False) - do_test('not a dict', None, False) - do_test('{"admin": ["someone"]}', None, False) - - def test_marker(self): - req = swob.Request.blank("/v1/AUTH_test?marker=mk") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(2, len(self.fake_rpc.calls)) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - self.assertEqual(self.fake_rpc.calls[1][1][0]['Marker'], 'mk') - - def test_end_marker(self): - req = swob.Request.blank("/v1/AUTH_test?end_marker=mk") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(2, len(self.fake_rpc.calls)) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - self.assertEqual(self.fake_rpc.calls[1][1][0]['EndMarker'], 'mk') - - def test_limit(self): - req = swob.Request.blank("/v1/AUTH_test?limit=101") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(2, len(self.fake_rpc.calls)) - self.assertEqual(self.fake_rpc.calls[1][1][0]['MaxEntries'], 101) - - def test_default_limit(self): - req = swob.Request.blank("/v1/AUTH_test") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(2, len(self.fake_rpc.calls)) - # value from GET /info - self.assertEqual(self.fake_rpc.calls[1][1][0]['MaxEntries'], 9876) - - def test_not_found(self): - self.app.register('HEAD', '/v1/AUTH_missing', 404, {}, '') - self.app.register('GET', '/v1/AUTH_missing', 404, {}, '') - req = swob.Request.blank("/v1/AUTH_missing") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_rpc_error(self): - def broken_RpcGetAccount(_): - return { - "error": "errno: 123", # meaningless errno - "result": None} - - self.fake_rpc.register_handler( - "Server.RpcGetAccount", broken_RpcGetAccount) - req = swob.Request.blank("/v1/AUTH_test") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '500 Internal Error') - - def test_empty_last_page(self): - def last_page_RpcGetAccount(_): - return { - "error": None, - # None, not [], for mysterious reasons - "result": { - "ModificationTime": 1510966502886466000, - "AccountEntries": None, - }} - - self.fake_rpc.register_handler( - "Server.RpcGetAccount", last_page_RpcGetAccount) - req = swob.Request.blank("/v1/AUTH_test?marker=zzz") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - self.assertEqual(body, b'') - - def test_spaces(self): - self.bimodal_accounts.add('AUTH_test with spaces') - self.app.register('HEAD', '/v1/AUTH_test with spaces', 204, - {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, - '') - req = swob.Request.blank("/v1/AUTH_test with spaces") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], - '/v1/AUTH_test with spaces') - - def test_empty(self): - def mock_RpcGetAccount(_): - return { - "error": None, - "result": { - "ModificationTime": 1510966489413995000, - "AccountEntries": []}} - - self.fake_rpc.register_handler( - "Server.RpcGetAccount", mock_RpcGetAccount) - req = swob.Request.blank("/v1/AUTH_test") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - - -class TestAccountHead(BaseMiddlewareTest): - def setUp(self): - super(TestAccountHead, self).setUp() - - self.app.register( - 'HEAD', '/v1/AUTH_test', - 204, - {'X-Account-Meta-Beans': 'lots of', - 'X-Account-Sysmeta-Proxyfs-Bimodal': 'true'}, - '') - - def test_indicator_header(self): - req = swob.Request.blank("/v1/AUTH_test", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - self.assertEqual(headers.get("ProxyFS-Enabled"), "yes") - self.assertEqual(body, b'') - - def test_in_transit(self): - - def fake_RpcIsAccountBimodal(request): - return { - "error": None, - "result": { - "IsBimodal": True, - "ActivePeerPrivateIPAddr": ""}} - self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", - fake_RpcIsAccountBimodal) - - req = swob.Request.blank("/v1/AUTH_test", - environ={"REQUEST_METHOD": "HEAD"}) - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '503 Service Unavailable') - - -class TestObjectGet(BaseMiddlewareTest): - def setUp(self): - super(TestObjectGet, self).setUp() - - def dummy_rpc(request): - return {"error": None, "result": {}} - - self.fake_rpc.register_handler("Server.RpcRenewLease", dummy_rpc) - self.fake_rpc.register_handler("Server.RpcReleaseLease", dummy_rpc) - - def test_info_passthrough(self): - self.app.register( - 'GET', '/info', 200, {}, '{"stuff": "yes"}') - - req = swob.Request.blank('/info') - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'{"stuff": "yes"}') - - def test_non_bimodal_account(self): - self.app.register( - 'HEAD', '/v1/AUTH_unimodal', 204, {}, '') - self.app.register( - 'GET', '/v1/AUTH_unimodal/c/o', 200, {}, 'squirrel') - - req = swob.Request.blank('/v1/AUTH_unimodal/c/o') - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'squirrel') - - def test_GET_basic(self): - # ProxyFS log segments look a lot like actual file contents followed - # by a bunch of fairly opaque binary data plus some JSON-looking - # bits. The actual bytes in the file here are the 8 that spell - # "burritos"; the rest of the bytes are there so we can omit them in - # the response to the user. - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000c11fbd', - 200, {}, - ("burritos\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/notes/lunch") - self.assertEqual(get_object_req['ReadEntsIn'], []) - - return { - "error": None, - "result": { - "FileSize": 8, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "prominority-sarcocyst", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000c11fbd"), - "Offset": 0, - "Length": 8}]}} - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - - req = swob.Request.blank('/v1/AUTH_test/notes/lunch') - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers.get("Accept-Ranges"), "bytes") - self.assertEqual(headers["Last-Modified"], - "Wed, 07 Dec 2016 23:08:55 GMT") - self.assertEqual(headers["ETag"], - mware.construct_etag("AUTH_test", 1245, 2424)) - self.assertEqual(body, b'burritos') - - req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan=on', - environ={'swift_owner': True}) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers['Content-Type'], 'application/json') - self.assertEqual(headers['X-Object-Content-Type'], - 'application/octet-stream') - self.assertEqual(headers['X-Object-Content-Length'], '8') - self.assertIn('ETag', headers) - self.assertIn('Last-Modified', headers) - self.assertEqual(json.loads(body), [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainerName" - "/0000000000c11fbd"), - "Offset": 0, - "Length": 8, - }]) - - # Can explicitly say you *don't* want the read plan - req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan=no', - environ={'swift_owner': True}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'burritos') - - # Can handle Range requests, too - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/notes/lunch") - self.assertEqual(get_object_req['ReadEntsIn'], [ - {"Len": 2, "Offset": 2}]) - - return { - "error": None, - "result": { - "FileSize": 8, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "prominority-sarcocyst", - "ReadEntsOut": [ - { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000c11fbd"), - "Offset": 587, - "Length": 1 - }, - { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000c11798"), - "Offset": 25, - "Length": 1 - }, - ]}} - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - - # Present-but-blank query param is truthy - req = swob.Request.blank('/v1/AUTH_test/notes/lunch?get-read-plan', - headers={'Range': 'bytes=2-3'}, - environ={'swift_owner': True}) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers['Content-Type'], 'application/json') - self.assertEqual(headers['X-Object-Content-Type'], - 'application/octet-stream') - self.assertEqual(headers['X-Object-Content-Length'], '8') - self.assertIn('ETag', headers) - self.assertIn('Last-Modified', headers) - self.assertEqual(json.loads(body), [ - { - "ObjectPath": ("/v1/AUTH_test/InternalContainerName" - "/0000000000c11fbd"), - "Offset": 587, - "Length": 1, - }, - { - "ObjectPath": ("/v1/AUTH_test/InternalContainerName" - "/0000000000c11798"), - "Offset": 25, - "Length": 1, - }, - ]) - - def test_GET_slo_manifest(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000c11fbd', - 200, {}, - ("blah[]\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/notes/lunch") - self.assertEqual(get_object_req['ReadEntsIn'], []) - - return { - "error": None, - "result": { - "FileSize": 2, - "Metadata": base64.b64encode( - json.dumps({ - 'X-Object-Sysmeta-Slo-Etag': 'some etag', - 'X-Object-Sysmeta-Slo-Size': '0', - 'Content-Type': 'text/plain;swift_bytes=0', - }).encode('ascii')).decode('ascii'), - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "prominority-sarcocyst", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000c11fbd"), - "Offset": 4, - "Length": 2}]}} - - req = swob.Request.blank('/v1/AUTH_test/notes/lunch') - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers, { - 'Content-Type': 'text/plain', # no swift_bytes! - 'Content-Length': '2', - # ^^^ needs to be actual size, but slo may/will use vvv for HEADs - 'X-Object-Sysmeta-Slo-Size': '0', - 'Etag': '"pfsv2/AUTH_test/000004DD/00000978-32"', - # slo may/will use vvv to fix up ^^^ - 'X-Object-Sysmeta-Slo-Etag': 'some etag', - 'Accept-Ranges': 'bytes', - 'X-Timestamp': '1481152134.33186', - 'Last-Modified': 'Wed, 07 Dec 2016 23:08:55 GMT', - }) - self.assertEqual(body, b'[]') - - def test_GET_authed(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000001178995', - 200, {}, - ("burr\x62\x6f\x6f\x74{\"Stuff\": \"probably\"}\x00\x00\x00")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000004181255', - 200, {}, - ("itos\x62\x6f\x6f\x74{\"more\": \"junk\"}\x00\x00\x00")) - - def mock_RpcHead(head_req): - return { - "error": None, - "result": { - "Metadata": None, - "ModificationTime": 1488414932080993000, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 6283109, - "NumWrites": 0, - }} - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/notes/lunch") - self.assertEqual(get_object_req['ReadEntsIn'], []) - - return { - "error": None, - "result": { - "FileSize": 8, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "a65b5591b90fe6035e669f1f216502d2", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000001178995"), - "Offset": 0, - "Length": 4}, - {"ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000004181255"), - "Offset": 0, - "Length": 4}]}} - - def auth_callback(req): - if "InternalContainerName" in req.path: - return swob.HTTPForbidden(body="lol nope", request=req) - - req = swob.Request.blank('/v1/AUTH_test/notes/lunch') - req.environ["swift.authorize"] = auth_callback - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Last-Modified"], - "Wed, 07 Dec 2016 23:08:55 GMT") - self.assertEqual(headers["ETag"], - mware.construct_etag("AUTH_test", 1245, 2424)) - self.assertEqual(body, b'burritos') - - def test_GET_sparse(self): - # This log segment, except for the obvious fake metadata at the end, - # was obtained by really doing this: - # - # with open(filename, 'w') as fh: - # fh.write('sparse') - # fh.seek(10006) - # fh.write('file') - # - # Despite its appearance, this is really what the underlying log - # segment for a sparse file looks like, at least sometimes. - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000D0', - 200, {}, - ("sparsefile\x57\x18\xa0\xf3-junkety-junk")) - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/sparse-file") - self.assertEqual(get_object_req['ReadEntsIn'], []) - - return { - "error": None, - "result": { - "FileSize": 10010, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "6840595b3370f109dc8ed388b41800a4", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/00000000000000D0"), - "Offset": 0, - "Length": 6 - }, { - "ObjectPath": "", # empty path means zero-fill - "Offset": 0, - "Length": 10000, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/00000000000000D0"), - "Offset": 6, - "Length": 4}]}} - - req = swob.Request.blank('/v1/AUTH_test/c/sparse-file') - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, "200 OK") - self.assertEqual(body, b"sparse" + (b"\x00" * 10000) + b"file") - - def test_GET_multiple_segments(self): - # Typically, a GET request will include data from multiple log - # segments. Small files written all at once might fit in a single - # log segment, but that's not always true. - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', - 200, {}, - ("There once was an X from place B,\n\xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000a3', - 200, {}, - ("That satisfied predicate P.\n\xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000a4', - 200, {}, - ("He or she did thing A,\n\xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/00000000000000e0', - 200, {}, - ("In an adjective way,\n\xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000108', - 200, {}, - ("Resulting in circumstance C.\xff\xea\x00junk")) - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/limericks/generic") - self.assertEqual(get_object_req['ReadEntsIn'], []) - - return { - "error": None, - "result": { - "FileSize": 134, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "who cares", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000001"), - "Offset": 0, - "Length": 34, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/00000000000000a3"), - "Offset": 0, - "Length": 28, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/00000000000000a4"), - "Offset": 0, - "Length": 23, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/00000000000000e0"), - "Offset": 0, - "Length": 21, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000108"), - "Offset": 0, - "Length": 28, - }]}} - - req = swob.Request.blank('/v1/AUTH_test/limericks/generic') - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(body, ( - b"There once was an X from place B,\n" - b"That satisfied predicate P.\n" - b"He or she did thing A,\n" - b"In an adjective way,\n" - b"Resulting in circumstance C.")) - - def test_GET_conditional_if_match(self): - obj_etag = "42d0f073592cdef8f602cda59fbb270e" - obj_body = b"annet-pentahydric-nuculoid-defiber" - - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/000000000097830c', - 200, {}, - "annet-pentahydric-nuculoid-defiber") - - def mock_RpcGetObject(get_object_req): - return { - "error": None, - "result": { - "FileSize": 8, - "Metadata": base64.b64encode( - json.dumps({ - mware.ORIGINAL_MD5_HEADER: "123:%s" % obj_etag, - }).encode('ascii')).decode('ascii'), - "InodeNumber": 1245, - "NumWrites": 123, - "ModificationTime": 1511222561631497000, - "IsDir": False, - "LeaseId": "dontcare", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/000000000097830c"), - "Offset": 0, - "Length": len(obj_body)}]}} - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - - # matches - req = swob.Request.blank('/v1/AUTH_test/hurdy/gurdy', - headers={"If-Match": obj_etag}) - status, _, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, obj_body) - - # doesn't match - req = swob.Request.blank('/v1/AUTH_test/hurdy/gurdy', - headers={"If-Match": obj_etag + "abc"}) - status, _, body = self.call_pfs(req) - self.assertEqual(status, '412 Precondition Failed') - - def test_GET_range(self): - # Typically, a GET response will include data from multiple log - # segments. Small files written all at once might fit in a single - # log segment, but most files won't. - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', - 200, {}, - ("Caerphilly, Cheddar, Cheshire, \xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', - 200, {}, - ("Duddleswell, Dunlop, Coquetdale, \xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000003', - 200, {}, - ("Derby, Gloucester, Wensleydale\xff\xea\x00junk")) - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/cheeses/UK") - self.assertEqual(get_object_req['ReadEntsIn'], - [{"Offset": 21, "Len": 49}]) - - return { - "error": None, - "result": { - "FileSize": 94, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "982938", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000001"), - "Offset": 21, - "Length": 10, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000002"), - "Offset": 0, - "Length": 33, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000003"), - "Offset": 0, - "Length": 5}]}} - - req = swob.Request.blank('/v1/AUTH_test/cheeses/UK', - headers={"Range": "bytes=21-69"}) - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '206 Partial Content') - self.assertEqual(headers.get('Content-Range'), "bytes 21-69/94") - self.assertEqual( - body, b'Cheshire, Duddleswell, Dunlop, Coquetdale, Derby') - - def test_GET_range_suffix(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', - 200, {}, - ("hydrogen, helium, \xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', - 200, {}, - ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/elements") - self.assertEqual(get_object_req['ReadEntsIn'], - [{"Offset": None, "Len": 10}]) - - return { - "error": None, - "result": { - "FileSize": 94, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "fc00:752b:5cca:a544:2d41:3177:2c71:85ae", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000002"), - "Offset": 35, - "Length": 10}]}} - - req = swob.Request.blank('/v1/AUTH_test/c/elements', - headers={"Range": "bytes=-10"}) - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '206 Partial Content') - self.assertEqual(headers.get('Content-Range'), "bytes 84-93/94") - self.assertEqual(body, b"nitrogen, ") - - def test_GET_range_prefix(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', - 200, {}, - ("hydrogen, helium, \xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', - 200, {}, - ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/elements") - self.assertEqual(get_object_req['ReadEntsIn'], - [{"Offset": 40, "Len": None}]) - - return { - "error": None, - "result": { - "FileSize": 62, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000002"), - "Offset": 22, - "Length": 23}]}} - - req = swob.Request.blank('/v1/AUTH_test/c/elements', - headers={"Range": "bytes=40-"}) - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '206 Partial Content') - self.assertEqual(headers.get('Content-Range'), "bytes 40-61/62") - self.assertEqual(body, b"ron, carbon, nitrogen, ") - - def test_GET_range_unsatisfiable(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', - 200, {}, - ("hydrogen, helium, \xff\xea\x00junk")) - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000002', - 200, {}, - ("lithium, beryllium, boron, carbon, nitrogen, \xff\xea\x00junk")) - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/elements") - self.assertEqual(get_object_req['ReadEntsIn'], - [{"Offset": 4000, "Len": None}]) - - return { - "error": None, - "result": { - "FileSize": 62, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2426, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "borkbork", - "ReadEntsOut": None}} - - req = swob.Request.blank('/v1/AUTH_test/c/elements', - headers={"Range": "bytes=4000-"}) - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '416 Requested Range Not Satisfiable') - self.assertEqual(headers.get('Content-Range'), 'bytes */62') - self.assertEqual(headers.get('ETag'), - mware.construct_etag("AUTH_test", 1245, 2426)) - self.assertEqual(headers.get('Last-Modified'), - 'Wed, 07 Dec 2016 23:08:55 GMT') - self.assertEqual(headers.get('X-Timestamp'), - '1481152134.33186') - - def test_GET_multiple_ranges(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', - 200, {}, "abcd1234efgh5678") - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/crud") - self.assertEqual(get_object_req['ReadEntsIn'], [ - {'Len': 3, 'Offset': 2}, - {'Len': 3, 'Offset': 6}, - {'Len': 3, 'Offset': 10}, - ]) - - return { - "error": None, - "result": { - "FileSize": 16, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "e1885b511fa445d18b1d447a5606a06d", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000001"), - "Offset": 2, - "Length": 3, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000001"), - "Offset": 6, - "Length": 3, - }, { - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000001"), - "Offset": 10, - "Length": 3, - }]}} - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - - req = swob.Request.blank('/v1/AUTH_test/c/crud', - headers={"Range": "bytes=2-4,6-8,10-12"}) - - # Lock down the MIME boundary so it doesn't change on every test run - with mock.patch('random.randint', - lambda u, l: 0xf0a9157cb1757bfb124aef22fee31051): - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '206 Partial Content') - self.assertEqual( - headers.get('Content-Type'), - 'multipart/byteranges;boundary=f0a9157cb1757bfb124aef22fee31051') - self.assertEqual( - body, - (b'--f0a9157cb1757bfb124aef22fee31051\r\n' - b'Content-Type: application/octet-stream\r\n' - b'Content-Range: bytes 2-4/16\r\n' - b'\r\n' - b'cd1\r\n' - b'--f0a9157cb1757bfb124aef22fee31051\r\n' - b'Content-Type: application/octet-stream\r\n' - b'Content-Range: bytes 6-8/16\r\n' - b'\r\n' - b'34e\r\n' - b'--f0a9157cb1757bfb124aef22fee31051\r\n' - b'Content-Type: application/octet-stream\r\n' - b'Content-Range: bytes 10-12/16\r\n' - b'\r\n' - b'gh5\r\n' - b'--f0a9157cb1757bfb124aef22fee31051--')) - - def test_GET_metadata(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000001', - 200, {}, "abcd1234efgh5678") - - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/crud") - - return { - "error": None, - "result": { - "FileSize": 16, - "Metadata": base64.b64encode(json.dumps({ - "X-Object-Meta-Cow": "moo", - }).encode('ascii')).decode('ascii'), - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "fe57a6ed-758a-23fb-f7d4-9683aee07c0e", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000001"), - "Offset": 0, - "Length": 16}]}} - - req = swob.Request.blank('/v1/AUTH_test/c/crud') - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers.get("X-Object-Meta-Cow"), "moo") - self.assertEqual(body, b'abcd1234efgh5678') - - def test_GET_bad_path(self): - bad_paths = [ - '/v1/AUTH_test/c/..', - '/v1/AUTH_test/c/../o', - '/v1/AUTH_test/c/o/..', - '/v1/AUTH_test/c/.', - '/v1/AUTH_test/c/./o', - '/v1/AUTH_test/c/o/.', - '/v1/AUTH_test/c//o', - '/v1/AUTH_test/c/o//', - '/v1/AUTH_test/c/o/', - '/v1/AUTH_test/c/' + ('x' * 256), - ] - for path in bad_paths: - req = swob.Request.blank(path) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - req.environ['REQUEST_METHOD'] = 'HEAD' - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - path = '/v1/AUTH_test//o' - req = swob.Request.blank(path, headers={'Content-Length': '0'}) - for method in ('GET', 'HEAD', 'PUT', 'POST', 'DELETE'): - req.environ['REQUEST_METHOD'] = method - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '412 Precondition Failed', - 'Got %s for %s %s' % (status, method, path)) - - def test_GET_not_found(self): - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/missing") - - return { - "error": "errno: 2", - "result": None} - - req = swob.Request.blank('/v1/AUTH_test/c/missing') - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_GET_file_as_dir(self): - # Subdirectories of files don't exist, but asking for one returns a - # different error code than asking for a file that could exist but - # doesn't. - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/thing.txt/kitten.png") - - return { - "error": "errno: 20", - "result": None} - - req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png') - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_GET_weird_error(self): - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/superborked") - - return { - "error": "errno: 23", # ENFILE (too many open files in system) - "result": None} - - req = swob.Request.blank('/v1/AUTH_test/c/superborked') - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '500 Internal Error') - - def test_GET_zero_byte(self): - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/empty") - return { - "error": None, - "result": {"FileSize": 0, "ReadEntsOut": None, "Metadata": "", - "IsDir": False, - "LeaseId": "3d73d2bcf39224df00d5ccd912d92c82", - "InodeNumber": 1245, "NumWrites": 2424, - "ModificationTime": 1481152134331862558}} - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - - req = swob.Request.blank('/v1/AUTH_test/c/empty') - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Length"], "0") - - def test_GET_dir(self): - def mock_RpcGetObject(get_object_req): - self.assertEqual(get_object_req['VirtPath'], - "/v1/AUTH_test/c/a-dir") - return { - "error": None, - "result": { - "Metadata": "", - "ModificationTime": 1479173168018879490, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 1254, - "NumWrites": 896, - "ReadEntsOut": None, - "LeaseId": "borkbork", - }} - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - - req = swob.Request.blank('/v1/AUTH_test/c/a-dir') - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Length"], "0") - self.assertEqual(headers["Content-Type"], "application/directory") - self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") - - def test_GET_special_chars(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000123', - 200, {}, "abcd1234efgh5678") - - def mock_RpcGetObject(get_object_req): - return { - "error": None, - "result": { - "FileSize": 8, - "Metadata": "", - "InodeNumber": 1245, - "NumWrites": 2424, - "ModificationTime": 1481152134331862558, - "IsDir": False, - "LeaseId": "a7ec296d3f3c39ef95407789c436f5f8", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000123"), - "Offset": 0, - "Length": 16}]}} - - req = swob.Request.blank('/v1/AUTH_test/c o n/o b j') - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'abcd1234efgh5678') - self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], - '/v1/AUTH_test/c o n/o b j') - - def test_md5_etag(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000456', - 200, {}, "stuff stuff stuff") - - def mock_RpcGetObject(_): - return { - "error": None, - "result": { - "Metadata": base64.b64encode(json.dumps({ - mware.ORIGINAL_MD5_HEADER: - "3:25152b9f7ca24b61eec895be4e89a950", - }).encode('ascii')).decode('ascii'), - "ModificationTime": 1506039770222591000, - "IsDir": False, - "FileSize": 17, - "IsDir": False, - "InodeNumber": 1433230, - "NumWrites": 3, - "LeaseId": "leaseid", - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000456"), - "Offset": 0, - "Length": 13}]}} - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png") - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Etag"], "25152b9f7ca24b61eec895be4e89a950") - - def test_lease_maintenance(self): - self.app.register( - 'GET', '/v1/AUTH_test/InternalContainerName/0000000000000456', - 200, {}, "some contents") - - expected_lease_id = "f1abb6c7fc4b4463b04f8313d71986b7" - - def mock_RpcGetObject(get_object_req): - return { - "error": None, - "result": { - "FileSize": 13, - "Metadata": "", - "InodeNumber": 7677424, - "NumWrites": 2325461, - "ModificationTime": 1488841810471415000, - "IsDir": False, - "LeaseId": expected_lease_id, - "ReadEntsOut": [{ - "ObjectPath": ("/v1/AUTH_test/InternalContainer" - "Name/0000000000000456"), - "Offset": 0, - "Length": 13}]}} - - req = swob.Request.blank('/v1/AUTH_test/some/thing') - - self.fake_rpc.register_handler( - "Server.RpcGetObject", mock_RpcGetObject) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'some contents') - - called_rpcs = [c[0] for c in self.fake_rpc.calls] - - # There's at least one RpcRenewLease, but perhaps there's more: it's - # time-based, and we aren't mocking out time in this test. - self.assertEqual(called_rpcs[0], 'Server.RpcIsAccountBimodal') - self.assertEqual(called_rpcs[1], 'Server.RpcGetObject') - self.assertEqual(called_rpcs[-2], 'Server.RpcRenewLease') - self.assertEqual(called_rpcs[-1], 'Server.RpcReleaseLease') - - # make sure we got the right lease ID - self.assertEqual(self.fake_rpc.calls[-2][1][0]['LeaseId'], - expected_lease_id) - self.assertEqual(self.fake_rpc.calls[-1][1][0]['LeaseId'], - expected_lease_id) - - -class TestContainerHead(BaseMiddlewareTest): - def setUp(self): - super(TestContainerHead, self).setUp() - - self.serialized_container_metadata = "" - - # All these tests run against the same container. - def mock_RpcHead(head_container_req): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - self.serialized_container_metadata.encode('ascii')), - "ModificationTime": 1479240397189581131, - "IsDir": False, - "FileSize": 0, - "InodeNumber": 2718, - "NumWrites": 0, - }} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - def test_special_chars(self): - req = swob.Request.blank("/v1/AUTH_test/a container", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - self.assertEqual(headers.get("Accept-Ranges"), "bytes") - self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], - '/v1/AUTH_test/a container') - - def test_content_type(self): - req = swob.Request.blank("/v1/AUTH_test/a-container?format=xml", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, _ = self.call_pfs(req) - self.assertEqual(status, '204 No Content') # sanity check - self.assertEqual(headers["Content-Type"], - "application/xml; charset=utf-8") - - req = swob.Request.blank("/v1/AUTH_test/a-container?format=json", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, _ = self.call_pfs(req) - self.assertEqual(status, '204 No Content') # sanity check - self.assertEqual(headers["Content-Type"], - "application/json; charset=utf-8") - - req = swob.Request.blank("/v1/AUTH_test/a-container?format=plain", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, _ = self.call_pfs(req) - self.assertEqual(status, '204 No Content') # sanity check - self.assertEqual(headers["Content-Type"], - "text/plain; charset=utf-8") - - with mock.patch('pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', - True): - status, headers, _ = self.call_pfs(req) - self.assertEqual(status, '204 No Content') # sanity check - self.assertEqual(headers["Content-Type"], - "application/json; charset=utf-8") - - def test_no_meta(self): - self.serialized_container_metadata = "" - - req = swob.Request.blank("/v1/AUTH_test/a-container", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - self.assertEqual(headers.get("Accept-Ranges"), "bytes") - self.assertEqual(headers["X-Container-Object-Count"], "0") - self.assertEqual(headers["X-Container-Bytes-Used"], "0") - self.assertEqual(headers["X-Storage-Policy"], "default") - self.assertEqual(headers["Last-Modified"], - "Tue, 15 Nov 2016 20:06:38 GMT") - self.assertEqual(headers["X-Timestamp"], "1479240397.18958") - self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], - '/v1/AUTH_test/a-container') - - def test_bogus_meta(self): - self.serialized_container_metadata = "{[{[{[{[{[[(((!" - - req = swob.Request.blank("/v1/AUTH_test/a-container", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - - def _check_meta(self, expected_headers, swift_owner=False): - container_metadata = { - "X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to", - "X-Container-Meta-Temp-Url-Key": "tuk", - "X-Container-Meta-Temp-Url-Key-2": "tuk2", - "X-Container-Sysmeta-Fish": "cod", - "X-Container-Meta-Fish": "trout"} - self.serialized_container_metadata = json.dumps(container_metadata) - - req = swob.Request.blank("/v1/AUTH_test/a-container", - environ={"REQUEST_METHOD": "HEAD", - "swift_owner": swift_owner}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - for k, v in expected_headers.items(): - self.assertIn(k, headers) - self.assertEqual(v, headers[k], "Expected %r but got %r for %r" % - (v, headers[k], v)) - for k in container_metadata: - self.assertFalse(k in headers and k not in expected_headers, - "Found unexpected header %r" % k) - - def test_meta_swift_owner(self): - self._check_meta({ - "X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to", - "X-Container-Meta-Temp-Url-Key": "tuk", - "X-Container-Meta-Temp-Url-Key-2": "tuk2", - "X-Container-Sysmeta-Fish": "cod", - "X-Container-Meta-Fish": "trout"}, - swift_owner=True) - - def test_meta_not_swift_owner(self): - self._check_meta({ - "X-Container-Sysmeta-Fish": "cod", - "X-Container-Meta-Fish": "trout"}) - - def test_not_found(self): - def mock_RpcHead(head_container_req): - self.assertEqual(head_container_req['VirtPath'], - '/v1/AUTH_test/a-container') - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank("/v1/AUTH_test/a-container", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_other_error(self): - def mock_RpcHead(head_container_req): - self.assertEqual(head_container_req['VirtPath'], - '/v1/AUTH_test/a-container') - return {"error": "errno: 7581", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank("/v1/AUTH_test/a-container", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '500 Internal Error') - - -class TestContainerGet(BaseMiddlewareTest): - def setUp(self): - super(TestContainerGet, self).setUp() - self.serialized_container_metadata = "" - - # All these tests run against the same container. - def mock_RpcGetContainer(get_container_req): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - self.serialized_container_metadata.encode('ascii')), - "ModificationTime": 1510790796076041000, - "ContainerEntries": [{ - "Basename": "images", - "FileSize": 0, - "ModificationTime": 1471915816359209849, - "IsDir": True, - "InodeNumber": 2489682, - "NumWrites": 0, - "Metadata": "", - }, { - "Basename": u"images/\xE1vocado.png", - "FileSize": 70, - "ModificationTime": 1471915816859209471, - "IsDir": False, - "InodeNumber": 9213768, - "NumWrites": 2, - "Metadata": base64.b64encode(json.dumps({ - "Content-Type": u"snack/m\xEDllenial" + - u";swift_bytes=3503770", - }).encode('ascii')), - }, { - "Basename": "images/banana.png", - "FileSize": 2189865, - # has fractional seconds = 0 to cover edge cases - "ModificationTime": 1471915873000000000, - "IsDir": False, - "InodeNumber": 8878410, - "NumWrites": 2, - "Metadata": "", - }, { - "Basename": "images/cherimoya.png", - "FileSize": 1636662, - "ModificationTime": 1471915917767421311, - "IsDir": False, - "InodeNumber": 8879064, - "NumWrites": 2, - # Note: this has NumWrites=2, but the original MD5 - # starts with "1:", so it is stale and must not be - # used. - "Metadata": base64.b64encode(json.dumps({ - mware.ORIGINAL_MD5_HEADER: - "1:552528fbf2366f8a4711ac0a3875188b", - }).encode('ascii')), - }, { - "Basename": "images/durian.png", - "FileSize": 8414281, - "ModificationTime": 1471985233074909930, - "IsDir": False, - "InodeNumber": 5807979, - "NumWrites": 3, - "Metadata": base64.b64encode(json.dumps({ - mware.ORIGINAL_MD5_HEADER: - "3:34f99f7784c573541e11e5ad66f065c8", - }).encode('ascii')), - }, { - "Basename": "images/elderberry.png", - "FileSize": 3178293, - "ModificationTime": 1471985240833932653, - "IsDir": False, - "InodeNumber": 4974021, - "NumWrites": 1, - "Metadata": "", - }]}} - - self.fake_rpc.register_handler( - "Server.RpcGetContainer", mock_RpcGetContainer) - - def test_text(self): - req = swob.Request.blank('/v1/AUTH_test/a-container') - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], "text/plain; charset=utf-8") - self.assertEqual(body, (b"images\n" - b"images/\xC3\xA1vocado.png\n" - b"images/banana.png\n" - b"images/cherimoya.png\n" - b"images/durian.png\n" - b"images/elderberry.png\n")) - self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], - '/v1/AUTH_test/a-container') - - with mock.patch('pfs_middleware.swift_code.LISTING_FORMATS_SWIFT', - True): - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') # sanity check - self.assertEqual(headers["Content-Type"], - "application/json; charset=utf-8") - json.loads(body) - - def test_dlo(self): - req = swob.Request.blank('/v1/AUTH_test/a-container', - environ={'swift.source': 'DLO'}) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], - "application/json; charset=utf-8") - json.loads(body) # doesn't crash - self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], - '/v1/AUTH_test/a-container') - - def _check_metadata(self, expected_headers, swift_owner=False): - container_metadata = { - "X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to", - "X-Container-Meta-Temp-Url-Key": "tuk", - "X-Container-Meta-Temp-Url-Key-2": "tuk2", - "X-Container-Sysmeta-Fish": "tilefish", - "X-Container-Meta-Fish": "haddock"} - self.serialized_container_metadata = json.dumps(container_metadata) - req = swob.Request.blank('/v1/AUTH_test/a-container', - environ={"REQUEST_METHOD": "GET", - "swift_owner": swift_owner}) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers.get("Accept-Ranges"), "bytes") - self.assertEqual(headers["X-Container-Object-Count"], "0") - self.assertEqual(headers["X-Container-Bytes-Used"], "0") - self.assertEqual(headers["X-Storage-Policy"], "default") - self.assertEqual(headers["X-Timestamp"], "1510790796.07604") - self.assertEqual(headers["Last-Modified"], - "Thu, 16 Nov 2017 00:06:37 GMT") - for k, v in expected_headers.items(): - self.assertIn(k, headers) - self.assertEqual(v, headers[k], "Expected %r but got %r for %r" % - (v, headers[k], v)) - for k in container_metadata: - self.assertFalse(k in headers and k not in expected_headers, - "Found unexpected header %r" % k) - - def test_metadata_swift_owner(self): - self._check_metadata({ - "X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to", - "X-Container-Meta-Temp-Url-Key": "tuk", - "X-Container-Meta-Temp-Url-Key-2": "tuk2", - "X-Container-Sysmeta-Fish": "tilefish", - "X-Container-Meta-Fish": "haddock"}, - swift_owner=True) - - def test_metadata_not_swift_owner(self): - self._check_metadata({ - "X-Container-Sysmeta-Fish": "tilefish", - "X-Container-Meta-Fish": "haddock"}) - - def test_bogus_metadata(self): - self.serialized_container_metadata = "{""")) - - root_node = ElementTree.fromstring(body) - self.assertEqual(root_node.tag, 'container') - self.assertEqual(root_node.attrib["name"], 'a-container') - - objects = list(root_node) - self.assertEqual(6, len(objects)) - self.assertEqual(objects[0].tag, 'object') - - # The XML container listing doesn't use XML attributes for data, but - # rather a sequence of tags like X Y ... - # - # We do an exhaustive check of one object's attributes, then - # spot-check the rest of the listing for brevity's sake. - obj_attr_tags = list(objects[1]) - self.assertEqual(len(obj_attr_tags), 5) - - name_node = obj_attr_tags[0] - self.assertEqual(name_node.tag, 'name') - self.assertEqual(name_node.text, u'images/\xE1vocado.png') - self.assertEqual(name_node.attrib, {}) # nothing extra in there - - hash_node = obj_attr_tags[1] - self.assertEqual(hash_node.tag, 'hash') - self.assertEqual(hash_node.text, mware.construct_etag( - "AUTH_test", 9213768, 2)) - self.assertEqual(hash_node.attrib, {}) - - bytes_node = obj_attr_tags[2] - self.assertEqual(bytes_node.tag, 'bytes') - self.assertEqual(bytes_node.text, '3503770') - self.assertEqual(bytes_node.attrib, {}) - - content_type_node = obj_attr_tags[3] - self.assertEqual(content_type_node.tag, 'content_type') - self.assertEqual(content_type_node.text, u'snack/m\xEDllenial') - self.assertEqual(content_type_node.attrib, {}) - - last_modified_node = obj_attr_tags[4] - self.assertEqual(last_modified_node.tag, 'last_modified') - self.assertEqual(last_modified_node.text, '2016-08-23T01:30:16.859210') - self.assertEqual(last_modified_node.attrib, {}) - - # Make sure the directory has the right type - obj_attr_tags = list(objects[0]) - self.assertEqual(len(obj_attr_tags), 5) - - name_node = obj_attr_tags[0] - self.assertEqual(name_node.tag, 'name') - self.assertEqual(name_node.text, 'images') - - hash_node = obj_attr_tags[1] - self.assertEqual(hash_node.tag, 'hash') - self.assertEqual(hash_node.text, "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(hash_node.attrib, {}) - - content_type_node = obj_attr_tags[3] - self.assertEqual(content_type_node.tag, 'content_type') - self.assertEqual(content_type_node.text, 'application/directory') - - # Check the names are correct - all_names = [list(tag)[0].text for tag in objects] - self.assertEqual( - ["images", u"images/\xE1vocado.png", "images/banana.png", - "images/cherimoya.png", "images/durian.png", - "images/elderberry.png"], - all_names) - - def test_xml_alternate_mime_type(self): - req = swob.Request.blank('/v1/AUTH_test/a-container', - headers={"Accept": "application/xml"}) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], - "application/xml; charset=utf-8") - self.assertTrue(body.startswith( - b"""""")) - - def test_xml_query_param(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?format=xml') - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], - "application/xml; charset=utf-8") - self.assertTrue(body.startswith( - b"""""")) - - def test_xml_special_chars(self): - req = swob.Request.blank('/v1/AUTH_test/c o n', - headers={"Accept": "text/xml"}) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], - "text/xml; charset=utf-8") - self.assertTrue(body.startswith( - b"""""")) - - root_node = ElementTree.fromstring(body) - self.assertEqual(root_node.tag, 'container') - self.assertEqual(root_node.attrib["name"], 'c o n') - self.assertEqual(self.fake_rpc.calls[1][1][0]['VirtPath'], - '/v1/AUTH_test/c o n') - - def test_marker(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?marker=sharpie') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - # sanity check - self.assertEqual(rpc_method, "Server.RpcGetContainer") - self.assertEqual(rpc_args[0]["Marker"], "sharpie") - - def test_end_marker(self): - req = swob.Request.blank( - '/v1/AUTH_test/a-container?end_marker=whiteboard') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - # sanity check - self.assertEqual(rpc_method, "Server.RpcGetContainer") - self.assertEqual(rpc_args[0]["EndMarker"], "whiteboard") - - def test_prefix(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?prefix=cow') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - # sanity check - self.assertEqual(rpc_method, "Server.RpcGetContainer") - self.assertEqual(rpc_args[0]["Prefix"], "cow") - - def test_delimiter(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?delimiter=/') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - # sanity check - self.assertEqual(rpc_method, "Server.RpcGetContainer") - self.assertEqual(rpc_args[0]["Delimiter"], "/") - - def test_default_limit(self): - req = swob.Request.blank('/v1/AUTH_test/a-container') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - self.assertEqual(rpc_args[0]["MaxEntries"], 6543) - - def test_valid_user_supplied_limit(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?limit=150') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - self.assertEqual(rpc_args[0]["MaxEntries"], 150) - - def test_zero_supplied_limit(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?limit=0') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - self.assertEqual(rpc_args[0]["MaxEntries"], 0) - - def test_negative_supplied_limit(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?limit=-1') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value - - def test_overlarge_supplied_limits(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?limit=6544') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '412 Precondition Failed') - - rpc_calls = self.fake_rpc.calls - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - self.assertEqual(len(rpc_calls), 1) - - def test_bogus_user_supplied_limit(self): - req = swob.Request.blank('/v1/AUTH_test/a-container?limit=chihuahua') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value - - def test_default_limit_matches_proxy_server(self): - req = swob.Request.blank('/v1/AUTH_test/a-container') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 2) - # rpc_calls[0] is a call to RpcIsAccountBimodal, which is not - # relevant to what we're testing here - rpc_method, rpc_args = rpc_calls[1] - self.assertEqual(rpc_method, "Server.RpcGetContainer") - self.assertEqual(rpc_args[0]["MaxEntries"], 6543) # default value - - def test_GET_bad_path(self): - bad_paths = [ - '/v1/AUTH_test/..', - '/v1/AUTH_test/.', - '/v1/AUTH_test/' + ('x' * 256), - ] - for path in bad_paths: - req = swob.Request.blank(path) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - req.environ['REQUEST_METHOD'] = 'HEAD' - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - def test_not_found(self): - def mock_RpcGetContainer_error(get_container_req): - self.assertEqual(get_container_req['VirtPath'], - '/v1/AUTH_test/a-container') - - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcGetContainer", mock_RpcGetContainer_error) - - req = swob.Request.blank('/v1/AUTH_test/a-container') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_other_error(self): - def mock_RpcGetContainer_error(get_container_req): - self.assertEqual(get_container_req['VirtPath'], - '/v1/AUTH_test/a-container') - - return {"error": "errno: 1661", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcGetContainer", mock_RpcGetContainer_error) - - req = swob.Request.blank('/v1/AUTH_test/a-container') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '500 Internal Error') - - -class TestContainerGetDelimiter(BaseMiddlewareTest): - def setUp(self): - super(TestContainerGetDelimiter, self).setUp() - self.serialized_container_metadata = "" - - def _mock_RpcGetContainerDelimiterNoTrailingSlash(self, get_container_req): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - self.serialized_container_metadata.encode('ascii')), - "ModificationTime": 1510790796076041000, - "ContainerEntries": [{ - "Basename": "images", - "FileSize": 0, - "ModificationTime": 1471915816359209849, - "IsDir": True, - "InodeNumber": 2489682, - "NumWrites": 0, - "Metadata": "", - }]}} - - def _mock_RpcGetContainerDelimiterWithTrailingSlash(self, - get_container_req): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - self.serialized_container_metadata.encode('ascii')), - "ModificationTime": 1510790796076041000, - "ContainerEntries": [{ - "Basename": "images/avocado.png", - "FileSize": 70, - "ModificationTime": 1471915816859209471, - "IsDir": False, - "InodeNumber": 9213768, - "NumWrites": 2, - "Metadata": base64.b64encode(json.dumps({ - "Content-Type": "snack/millenial" + - ";swift_bytes=3503770", - }).encode('ascii')), - }, { - "Basename": "images/banana.png", - "FileSize": 2189865, - # has fractional seconds = 0 to cover edge cases - "ModificationTime": 1471915873000000000, - "IsDir": False, - "InodeNumber": 8878410, - "NumWrites": 2, - "Metadata": "", - }, { - "Basename": "images/cherimoya.png", - "FileSize": 1636662, - "ModificationTime": 1471915917767421311, - "IsDir": False, - "InodeNumber": 8879064, - "NumWrites": 2, - # Note: this has NumWrites=2, but the original MD5 - # starts with "1:", so it is stale and must not be - # used. - "Metadata": base64.b64encode(json.dumps({ - mware.ORIGINAL_MD5_HEADER: - "1:552528fbf2366f8a4711ac0a3875188b", - }).encode('ascii')), - }, { - "Basename": "images/durian.png", - "FileSize": 8414281, - "ModificationTime": 1471985233074909930, - "IsDir": False, - "InodeNumber": 5807979, - "NumWrites": 3, - "Metadata": base64.b64encode(json.dumps({ - mware.ORIGINAL_MD5_HEADER: - "3:34f99f7784c573541e11e5ad66f065c8", - }).encode('ascii')), - }, { - "Basename": "images/elderberry.png", - "FileSize": 3178293, - "ModificationTime": 1471985240833932653, - "IsDir": False, - "InodeNumber": 4974021, - "NumWrites": 1, - "Metadata": "", - }]}} - - def test_json_no_trailing_slash(self): - self.fake_rpc.register_handler( - "Server.RpcGetContainer", - self._mock_RpcGetContainerDelimiterNoTrailingSlash) - - req = swob.Request.blank( - '/v1/AUTH_test/a-container?prefix=images&delimiter=/', - headers={"Accept": "application/json"}) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], - "application/json; charset=utf-8") - resp_data = json.loads(body) - self.assertIsInstance(resp_data, list) - self.assertEqual(len(resp_data), 2) - self.assertEqual(resp_data[0], { - "name": "images", - "bytes": 0, - "content_type": "application/directory", - "hash": "d41d8cd98f00b204e9800998ecf8427e", - "last_modified": "2016-08-23T01:30:16.359210"}) - self.assertEqual(resp_data[1], { - "subdir": "images/"}) - - def test_json_with_trailing_slash(self): - self.fake_rpc.register_handler( - "Server.RpcGetContainer", - self._mock_RpcGetContainerDelimiterWithTrailingSlash) - - req = swob.Request.blank( - '/v1/AUTH_test/a-container?prefix=images/&delimiter=/', - headers={"Accept": "application/json"}) - status, headers, body = self.call_pfs(req) - - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], - "application/json; charset=utf-8") - resp_data = json.loads(body) - self.assertIsInstance(resp_data, list) - self.assertEqual(len(resp_data), 5) - self.assertEqual(resp_data[0], { - "name": "images/avocado.png", - "bytes": 3503770, - "content_type": "snack/millenial", - "hash": mware.construct_etag( - "AUTH_test", 9213768, 2), - "last_modified": "2016-08-23T01:30:16.859210"}) - self.assertEqual(resp_data[1], { - "name": "images/banana.png", - "bytes": 2189865, - "content_type": "image/png", - "hash": mware.construct_etag( - "AUTH_test", 8878410, 2), - "last_modified": "2016-08-23T01:31:13.000000"}) - self.assertEqual(resp_data[2], { - "name": "images/cherimoya.png", - "bytes": 1636662, - "content_type": "image/png", - "hash": mware.construct_etag( - "AUTH_test", 8879064, 2), - "last_modified": "2016-08-23T01:31:57.767421"}) - self.assertEqual(resp_data[3], { - "name": "images/durian.png", - "bytes": 8414281, - "content_type": "image/png", - "hash": "34f99f7784c573541e11e5ad66f065c8", - "last_modified": "2016-08-23T20:47:13.074910"}) - self.assertEqual(resp_data[4], { - "name": "images/elderberry.png", - "bytes": 3178293, - "content_type": "image/png", - "hash": mware.construct_etag( - "AUTH_test", 4974021, 1), - "last_modified": "2016-08-23T20:47:20.833933"}) - - -class TestContainerPost(BaseMiddlewareTest): - def test_missing_container(self): - def mock_RpcHead(_): - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank( - "/v1/AUTH_test/new-con", - environ={"REQUEST_METHOD": "POST"}, - headers={"X-Container-Meta-One-Fish": "two fish"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("404 Not Found", status) - - rpc_calls = self.fake_rpc.calls - self.assertEqual(2, len(rpc_calls)) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") - - def test_existing_container(self): - old_meta = json.dumps({ - "X-Container-Read": "xcr", - "X-Container-Meta-One-Fish": "no fish"}) - - def mock_RpcHead(_): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - old_meta.encode('ascii')).decode('ascii'), - "ModificationTime": 1482280565956671142, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 6515443, - "NumWrites": 0}} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - self.fake_rpc.register_handler( - "Server.RpcPost", lambda *a: {"error": None, "result": {}}) - - req = swob.Request.blank( - "/v1/AUTH_test/new-con", - environ={"REQUEST_METHOD": "POST"}, - headers={"X-Container-Meta-One-Fish": "two fish"}) - with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: - status, _, _ = self.call_pfs(req) - self.assertEqual("204 No Content", status) - cic.assert_called() - - rpc_calls = self.fake_rpc.calls - self.assertEqual(3, len(rpc_calls)) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPost") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") - self.assertEqual( - base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) - new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) - self.assertEqual(new_meta["X-Container-Meta-One-Fish"], "two fish") - self.assertEqual(new_meta["X-Container-Read"], "xcr") - - -class TestContainerPut(BaseMiddlewareTest): - def setUp(self): - super(TestContainerPut, self).setUp() - - # This only returns success/failure, not any interesting data - def mock_RpcPutContainer(_): - return {"error": None, "result": {}} - - self.fake_rpc.register_handler( - "Server.RpcPutContainer", mock_RpcPutContainer) - - def test_new_container(self): - def mock_RpcHead(_): - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank( - "/v1/AUTH_test/new-con", - environ={"REQUEST_METHOD": "PUT"}, - headers={"X-Container-Meta-Red-Fish": "blue fish"}) - with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: - status, _, _ = self.call_pfs(req) - self.assertEqual("201 Created", status) - cic.assert_called() - - rpc_calls = self.fake_rpc.calls - self.assertEqual(3, len(rpc_calls)) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPutContainer") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") - self.assertEqual(args[0]["OldMetadata"], "") - self.assertEqual( - base64.b64decode(args[0]["NewMetadata"]).decode('ascii'), - json.dumps({"X-Container-Meta-Red-Fish": "blue fish"})) - - def test_PUT_bad_path(self): - bad_container_paths = [ - '/v1/AUTH_test/..', - '/v1/AUTH_test/.', - '/v1/AUTH_test/' + ('x' * 256), - ] - for path in bad_container_paths: - req = swob.Request.blank(path, - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": BytesIO(b""), - "CONTENT_LENGTH": "0"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request', - 'Got %s for %s' % (status, path)) - - req.environ['REQUEST_METHOD'] = 'POST' - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - req.environ['REQUEST_METHOD'] = 'DELETE' - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - def test_name_too_long(self): - def mock_RpcHead(_): - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - self.swift_info['swift']['max_container_name_length'] = 50 - acceptable_name = 'A' * ( - self.swift_info['swift']['max_container_name_length']) - too_long_name = 'A' * ( - self.swift_info['swift']['max_container_name_length'] + 1) - self.app.register( - 'GET', '/info', - 200, {'Content-Type': 'application/json'}, - json.dumps(self.swift_info)) - - req = swob.Request.blank( - "/v1/AUTH_test/%s" % acceptable_name, - environ={"REQUEST_METHOD": "PUT"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("201 Created", status) - - req = swob.Request.blank( - "/v1/AUTH_test/%s" % too_long_name, - environ={"REQUEST_METHOD": "PUT"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("400 Bad Request", status) - - def test_name_too_long_posix(self): - def mock_RpcHead(_): - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - posix_limit = mware.NAME_MAX - self.swift_info['swift']['max_container_name_length'] = posix_limit * 2 - self.app.register( - 'GET', '/info', - 200, {'Content-Type': 'application/json'}, - json.dumps(self.swift_info)) - - too_long_name = 'A' * (posix_limit + 1) - req = swob.Request.blank( - "/v1/AUTH_test/%s" % too_long_name, - environ={"REQUEST_METHOD": "PUT"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("400 Bad Request", status) - - def _check_existing_container(self, req_headers, expected_meta, - swift_owner=False): - old_meta = json.dumps({ - "X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to", - "X-Container-Meta-Temp-Url-Key": "tuk", - "X-Container-Meta-Temp-Url-Key-2": "tuk2", - "X-Container-Meta-Red-Fish": "dead fish"}) - - def mock_RpcHead(_): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - old_meta.encode('ascii')).decode('ascii'), - "ModificationTime": 1482270529646747881, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 8054914, - "NumWrites": 0}} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank( - "/v1/AUTH_test/new-con", - environ={"REQUEST_METHOD": "PUT", 'swift_owner': swift_owner}, - headers=req_headers) - with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: - status, _, _ = self.call_pfs(req) - self.assertEqual("202 Accepted", status) - cic.assert_called() - - rpc_calls = self.fake_rpc.calls - self.assertEqual(3, len(rpc_calls)) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPutContainer") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") - self.assertEqual( - base64.b64decode(args[0]["OldMetadata"]).decode('ascii'), old_meta) - new_meta = json.loads(base64.b64decode(args[0]["NewMetadata"])) - self.assertEqual(expected_meta, new_meta) - - def test_existing_container_swift_owner(self): - self._check_existing_container( - {"X-Container-Read": "xcr-new", - "X-Container-Write": "xcw-new", - "X-Container-Sync-Key": "sync-key-new", - "X-Container-Sync-To": "sync-to-new", - "X-Container-Meta-Temp-Url-Key": "tuk-new", - "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", - "X-Container-Meta-Red-Fish": "blue fish"}, - {"X-Container-Read": "xcr-new", - "X-Container-Write": "xcw-new", - "X-Container-Sync-Key": "sync-key-new", - "X-Container-Sync-To": "sync-to-new", - "X-Container-Meta-Temp-Url-Key": "tuk-new", - "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", - "X-Container-Meta-Red-Fish": "blue fish"}, - swift_owner=True - ) - - def test_existing_container_not_swift_owner(self): - self._check_existing_container( - {"X-Container-Read": "xcr-new", - "X-Container-Write": "xcw-new", - "X-Container-Sync-Key": "sync-key-new", - "X-Container-Sync-To": "sync-to-new", - "X-Container-Meta-Temp-Url-Key": "tuk-new", - "X-Container-Meta-Temp-Url-Key-2": "tuk2-new", - "X-Container-Meta-Red-Fish": "blue fish"}, - {"X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to", - "X-Container-Meta-Temp-Url-Key": "tuk", - "X-Container-Meta-Temp-Url-Key-2": "tuk2", - "X-Container-Meta-Red-Fish": "blue fish"} - ) - - def _check_metadata_removal(self, headers, expected_meta, - swift_owner=False): - old_meta = json.dumps({ - "X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to", - "X-Versions-Location": "loc", - "X-Container-Meta-Box": "cardboard" - }) - - def mock_RpcHead(_): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - old_meta.encode('ascii')).decode('ascii'), - "ModificationTime": 1511224700123739000, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 8017342, - "NumWrites": 0}} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank( - "/v1/AUTH_test/new-con", - environ={"REQUEST_METHOD": "PUT", "swift_owner": swift_owner}, - headers=headers) - status, _, _ = self.call_pfs(req) - self.assertEqual("202 Accepted", status) - - rpc_calls = self.fake_rpc.calls - method, args = rpc_calls[-1] - self.assertEqual(method, "Server.RpcPutContainer") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/new-con") - self.assertEqual( - base64.b64decode(args[0]["OldMetadata"]).decode('ascii'), old_meta) - new_meta = json.loads(base64.b64decode(args[0]["NewMetadata"])) - self.assertEqual(expected_meta, new_meta) - - def test_metadata_removal_swift_owner(self): - self._check_metadata_removal( - {"X-Container-Read": "", - "X-Container-Write": "", - "X-Container-Sync-Key": "", - "X-Container-Sync-To": "", - "X-Versions-Location": "", - "X-Container-Meta-Box": ""}, - {}, swift_owner=True) - - def test_metadata_removal_with_remove_swift_owner(self): - self._check_metadata_removal( - {"X-Remove-Container-Read": "", - "X-Remove-Container-Write": "", - "X-Remove-Container-Sync-Key": "", - "X-Remove-Container-Sync-To": "", - "X-Remove-Versions-Location": "", - "X-Remove-Container-Meta-Box": ""}, - {}, swift_owner=True) - - def test_metadata_removal_not_swift_owner(self): - self._check_metadata_removal( - {"X-Container-Read": "", - "X-Container-Write": "", - "X-Container-Sync-Key": "", - "X-Container-Sync-To": "", - "X-Versions-Location": "", - "X-Container-Meta-Box": ""}, - {"X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to"}) - - def test_metadata_removal_with_remove_not_swift_owner(self): - self._check_metadata_removal( - {"X-Remove-Container-Read": "", - "X-Remove-Container-Write": "", - "X-Remove-Container-Sync-Key": "", - "X-Remove-Container-Sync-To": "", - "X-Remove-Versions-Location": "", - "X-Remove-Container-Meta-Box": ""}, - {"X-Container-Read": "xcr", - "X-Container-Write": "xcw", - "X-Container-Sync-Key": "sync-key", - "X-Container-Sync-To": "sync-to"}) - - -class TestContainerDelete(BaseMiddlewareTest): - def test_success(self): - def mock_RpcDelete(_): - return {"error": None, "result": {}} - - self.fake_rpc.register_handler( - "Server.RpcDelete", mock_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/empty-con", - environ={"REQUEST_METHOD": "DELETE"}) - with mock.patch("pfs_middleware.middleware.clear_info_cache") as cic: - status, _, _ = self.call_pfs(req) - self.assertEqual("204 No Content", status) - self.assertNotIn("Accept-Ranges", req.headers) - self.assertEqual(2, len(self.fake_rpc.calls)) - self.assertEqual("/v1/AUTH_test/empty-con", - self.fake_rpc.calls[1][1][0]["VirtPath"]) - cic.assert_called() - - def test_special_chars(self): - def mock_RpcDelete(_): - return {"error": None, "result": {}} - - self.fake_rpc.register_handler( - "Server.RpcDelete", mock_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/e m p t y", - environ={"REQUEST_METHOD": "DELETE"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("204 No Content", status) - self.assertEqual("/v1/AUTH_test/e m p t y", - self.fake_rpc.calls[1][1][0]["VirtPath"]) - - def test_not_found(self): - def mock_RpcDelete(_): - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcDelete", mock_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/empty-con", - environ={"REQUEST_METHOD": "DELETE"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("404 Not Found", status) - - def test_not_empty(self): - def mock_RpcDelete(_): - return {"error": "errno: 39", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcDelete", mock_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/empty-con", - environ={"REQUEST_METHOD": "DELETE"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("409 Conflict", status) - - def test_other_error(self): - def mock_RpcDelete(_): - return {"error": "errno: 987654321", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcDelete", mock_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/empty-con", - environ={"REQUEST_METHOD": "DELETE"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("500 Internal Error", status) - - -class TestObjectPut(BaseMiddlewareTest): - def setUp(self): - super(TestObjectPut, self).setUp() - - # These mocks act as though everything was successful. Failure tests - # can override the relevant mocks in the individual test cases. - def mock_RpcHead(head_container_req): - # Empty container, but it exists. That's enough for testing - # object PUT. - return { - "error": None, - "result": { - "Metadata": "", - "ModificationTime": 14792389930244718933, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 1828, - "NumWrites": 893, - }} - - put_loc_count = collections.defaultdict(int) - - def mock_RpcPutLocation(put_location_req): - # Give a different sequence of physical paths for each object - # name - virt_path = put_location_req["VirtPath"] - obj_name = hashlib.sha1( - virt_path.encode('utf8')).hexdigest().upper() - phys_path = "/v1/AUTH_test/PhysContainer_1/" + obj_name - if put_loc_count[virt_path] > 0: - phys_path += "-%02x" % put_loc_count[virt_path] - put_loc_count[virt_path] += 1 - - # Someone's probably about to PUT an object there, so let's set - # up the mock to allow it. Doing it here ensures that the - # location comes out of this RPC and nowhere else. - - self.app.register('PUT', phys_path, 201, {}, "") - - return { - "error": None, - "result": {"PhysPath": phys_path}} - - def mock_RpcPutComplete(put_complete_req): - return {"error": None, "result": { - "ModificationTime": 12345, - "InodeNumber": 678, - "NumWrites": 9}} - - def mock_RpcMiddlewareMkdir(middleware_mkdir_req): - return {"error": None, "result": { - "ModificationTime": 1504652321749543000, - "InodeNumber": 9268022, - "NumWrites": 0}} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - self.fake_rpc.register_handler( - "Server.RpcPutLocation", mock_RpcPutLocation) - self.fake_rpc.register_handler( - "Server.RpcPutComplete", mock_RpcPutComplete) - self.fake_rpc.register_handler( - "Server.RpcMiddlewareMkdir", mock_RpcMiddlewareMkdir) - - def test_basic(self): - wsgi_input = BytesIO(b"sparkleberry-displeasurably") - cl = str(len(wsgi_input.getvalue())) - - req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input, - "CONTENT_LENGTH": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - self.assertEqual(headers["ETag"], - hashlib.md5(wsgi_input.getvalue()).hexdigest()) - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 4) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/a-container") - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPutLocation") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/a-container/an-object") - - method, args = rpc_calls[3] - expected_phys_path = ("/v1/AUTH_test/PhysContainer_1/" - "80D184B041B9BF0C2EE8D55D8DC9797BF7129E13") - self.assertEqual(method, "Server.RpcPutComplete") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/a-container/an-object") - self.assertEqual(args[0]["PhysPaths"], [expected_phys_path]) - self.assertEqual(args[0]["PhysLengths"], [len(wsgi_input.getvalue())]) - - def test_directory(self): - req = swob.Request.blank( - "/v1/AUTH_test/a-container/a-dir", - environ={"REQUEST_METHOD": "PUT"}, - headers={"Content-Length": 0, - "Content-Type": "application/directory", - "X-Object-Sysmeta-Abc": "DEF"}, - body="") - - # directories always return the hard coded value for EMPTY_OBJECT_ETAG - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 3) - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcMiddlewareMkdir") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/a-container/a-dir") - - serialized_metadata = args[0]["Metadata"] - metadata = json.loads(base64.b64decode(serialized_metadata)) - self.assertEqual(metadata.get("X-Object-Sysmeta-Abc"), "DEF") - - def test_modification_time(self): - def mock_RpcPutComplete(put_complete_req): - return {"error": None, "result": { - "ModificationTime": 1481311245635845000, - "InodeNumber": 4116394, - "NumWrites": 1}} - - self.fake_rpc.register_handler( - "Server.RpcPutComplete", mock_RpcPutComplete) - - wsgi_input = BytesIO(b"Rhodothece-cholesterinuria") - cl = str(len(wsgi_input.getvalue())) - - req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input, - "CONTENT_LENGTH": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, "201 Created") - self.assertEqual(headers["Last-Modified"], - "Fri, 09 Dec 2016 19:20:46 GMT") - - def test_special_chars(self): - wsgi_input = BytesIO(b"pancreas-mystagogically") - cl = str(len(wsgi_input.getvalue())) - - req = swob.Request.blank("/v1/AUTH_test/c o n/o b j", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input, - "CONTENT_LENGTH": cl}) - - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 4) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/c o n") - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPutLocation") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/c o n/o b j") - - method, args = rpc_calls[3] - self.assertEqual(method, "Server.RpcPutComplete") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/c o n/o b j") - - def test_PUT_bad_path(self): - bad_container_paths = [ - '/v1/AUTH_test/../o', - '/v1/AUTH_test/./o', - '/v1/AUTH_test/' + ('x' * 256) + '/o', - ] - for path in bad_container_paths: - req = swob.Request.blank(path, - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": BytesIO(b""), - "CONTENT_LENGTH": "0"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - req.environ['REQUEST_METHOD'] = 'POST' - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - req.environ['REQUEST_METHOD'] = 'DELETE' - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - bad_paths = [ - '/v1/AUTH_test/c/..', - '/v1/AUTH_test/c/../o', - '/v1/AUTH_test/c/o/..', - '/v1/AUTH_test/c/.', - '/v1/AUTH_test/c/./o', - '/v1/AUTH_test/c/o/.', - '/v1/AUTH_test/c//o', - '/v1/AUTH_test/c/o//', - '/v1/AUTH_test/c/o/', - '/v1/AUTH_test/c/' + ('x' * 256), - ] - for path in bad_paths: - req = swob.Request.blank(path, - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": BytesIO(b""), - "CONTENT_LENGTH": "0"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request', - 'Got %s for %s' % (status, path)) - - req.environ['REQUEST_METHOD'] = 'POST' - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - req.environ['REQUEST_METHOD'] = 'DELETE' - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found', - 'Got %s for %s' % (status, path)) - - def test_big(self): - wsgi_input = BytesIO(b'A' * 100 + b'B' * 100 + b'C' * 75) - self.pfs.max_log_segment_size = 100 - - req = swob.Request.blank("/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input}, - headers={"X-Trans-Id": "big-txid", - "Content-Length": "275"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - self.assertEqual(headers["Content-Type"], "application/octet-stream") - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 6) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con") - - # 3 calls to RpcPutLocation since this was spread across 3 log - # segments - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPutLocation") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/con/obj") - - method, args = rpc_calls[3] - self.assertEqual(method, "Server.RpcPutLocation") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/con/obj") - - method, args = rpc_calls[4] - self.assertEqual(method, "Server.RpcPutLocation") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/con/obj") - - method, args = rpc_calls[5] - self.assertEqual(method, "Server.RpcPutComplete") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/con/obj") - pre = "/v1/AUTH_test/PhysContainer_1/" - self.assertEqual( - args[0]["PhysPaths"], - [pre + "1550057D8B0039185EB6184C599C940E51953403", - pre + "1550057D8B0039185EB6184C599C940E51953403-01", - pre + "1550057D8B0039185EB6184C599C940E51953403-02"]) - self.assertEqual(args[0]["PhysLengths"], [100, 100, 75]) - - # check the txids as well - put_calls = [c for c in self.app.calls if c[0] == 'PUT'] - self.assertEqual( - "big-txid-000", put_calls[0][2]["X-Trans-Id"]) # 1st PUT - self.assertEqual( - "big-txid-001", put_calls[1][2]["X-Trans-Id"]) # 2nd PUT - self.assertEqual( - "big-txid-002", put_calls[2][2]["X-Trans-Id"]) # 3rd PUT - - # If we sent the original Content-Length, the first PUT would fail. - # At some point, we should send the correct Content-Length value - # when we can compute it, but for now, we just send nothing. - self.assertNotIn("Content-Length", put_calls[0][2]) # 1st PUT - self.assertNotIn("Content-Length", put_calls[1][2]) # 2nd PUT - self.assertNotIn("Content-Length", put_calls[2][2]) # 3rd PUT - - def test_big_exact_multiple(self): - wsgi_input = BytesIO(b'A' * 100 + b'B' * 100) - cl = str(len(wsgi_input.getvalue())) - self.pfs.max_log_segment_size = 100 - - req = swob.Request.blank("/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input, - "CONTENT_LENGTH": cl}, - headers={"X-Trans-Id": "big-txid"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - self.assertEqual(headers["Content-Type"], "application/octet-stream") - - rpc_calls = self.fake_rpc.calls - self.assertEqual(len(rpc_calls), 5) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con") - - # 2 calls to RpcPutLocation since this was spread across 2 log - # segments. We didn't make a 0-length log segment and try to splice - # that in. - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPutLocation") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/con/obj") - - method, args = rpc_calls[3] - self.assertEqual(method, "Server.RpcPutLocation") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/con/obj") - - method, args = rpc_calls[4] - self.assertEqual(method, "Server.RpcPutComplete") - self.assertEqual(args[0]["VirtPath"], - "/v1/AUTH_test/con/obj") - pre = "/v1/AUTH_test/PhysContainer_1/" - self.assertEqual( - args[0]["PhysPaths"], - [pre + "1550057D8B0039185EB6184C599C940E51953403", - pre + "1550057D8B0039185EB6184C599C940E51953403-01"]) - self.assertEqual(args[0]["PhysLengths"], [100, 100]) - - def test_missing_container(self): - def mock_RpcHead_not_found(head_container_req): - # This is what you get for no-such-container. - return { - "error": "errno: 2", - "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead_not_found) - - wsgi_input = BytesIO(b"toxicum-brickcroft") - - req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, "404 Not Found") - - def test_metadata(self): - wsgi_input = BytesIO(b"extranean-paleophysiology") - cl = str(len(wsgi_input.getvalue())) - - headers_in = { - "X-Object-Meta-Color": "puce", - "X-Object-Sysmeta-Flavor": "bbq", - - "Content-Disposition": "recycle when no longer needed", - "Content-Encoding": "quadruple rot13", - "Content-Type": "application/eggplant", - # NB: we'll never actually see these two together, but it's fine - # for this test since we're only looking at which headers get - # saved and which don't. - "X-Object-Manifest": "pre/fix", - "X-Static-Large-Object": "yes", - - # These are not included - "X-Timestamp": "1473968446.11364", - "X-Object-Qmeta-Shape": "trapezoidal", - } - - req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", - headers=headers_in, - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input, - "CONTENT_LENGTH": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - - serialized_metadata = self.fake_rpc.calls[3][1][0]["Metadata"] - metadata = json.loads(base64.b64decode(serialized_metadata)) - self.assertEqual(metadata.get("X-Object-Meta-Color"), "puce") - self.assertEqual(metadata.get("X-Object-Sysmeta-Flavor"), "bbq") - self.assertEqual(metadata.get("X-Object-Manifest"), "pre/fix") - self.assertEqual(metadata.get("X-Static-Large-Object"), "yes") - self.assertEqual(metadata.get("Content-Disposition"), - "recycle when no longer needed") - self.assertEqual(metadata.get("Content-Encoding"), "quadruple rot13") - self.assertEqual(metadata.get("Content-Type"), "application/eggplant") - - def test_directory_in_the_way(self): - # If "thing.txt" is a nonempty directory, we get an error that the - # middleware turns into a 409 Conflict response. - wsgi_input = BytesIO(b"Celestine-malleal") - cl = str(len(wsgi_input.getvalue())) - - def mock_RpcPutComplete_isdir(head_container_req): - # This is what you get when there's a nonempty directory in - # place of your file. - return { - "error": "errno: 39", - "result": None} - - self.fake_rpc.register_handler( - "Server.RpcPutComplete", mock_RpcPutComplete_isdir) - - req = swob.Request.blank("/v1/AUTH_test/a-container/d1/d2/thing.txt", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input, - "CONTENT_LENGTH": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '409 Conflict') - - def test_file_in_the_way(self): - # If "thing.txt" is a nonempty directory, we get an error that the - # middleware turns into a 409 Conflict response. - wsgi_input = BytesIO(b"Celestine-malleal") - cl = str(len(wsgi_input.getvalue())) - - def mock_RpcPutComplete_notdir(head_container_req): - # This is what you get when there's a file where your path - # contains a subdirectory. - return { - "error": "errno: 20", - "result": None} - - self.fake_rpc.register_handler( - "Server.RpcPutComplete", mock_RpcPutComplete_notdir) - - req = swob.Request.blank("/v1/AUTH_test/a-container/a-file/thing.txt", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input, - "CONTENT_LENGTH": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '409 Conflict') - - def test_stripping_bad_headers(self): - # Someday, we'll have to figure out how to expire objects in - # proxyfs. For now, though, we remove X-Delete-At and X-Delete-After - # because having a log segment expire will do bad things to our - # file's integrity. - # - # We strip ETag because Swift will treat that as the expected MD5 of - # the log segment, and if we split across multiple log segments, - # then any user-supplied ETag will be wrong. Also, this makes - # POST-as-COPY work despite ProxyFS's ETag values not being MD5 - # checksums. - wsgi_input = BytesIO(b"extranean-paleophysiology") - cl = str(len(wsgi_input.getvalue())) - - headers_in = {"X-Delete-After": 86400, - "ETag": hashlib.md5(wsgi_input.getvalue()).hexdigest()} - - req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", - headers=headers_in, - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input, - "CONTENT_LENGTH": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - - serialized_metadata = self.fake_rpc.calls[3][1][0]["Metadata"] - metadata = json.loads(base64.b64decode(serialized_metadata)) - # it didn't get saved in metadata (not that it matters too much) - self.assertNotIn("X-Delete-At", metadata) - self.assertNotIn("X-Delete-After", metadata) - self.assertNotIn("ETag", metadata) - - # it didn't make it to Swift (this is important) - put_method, put_path, put_headers = self.app.calls[-1] - self.assertEqual(put_method, 'PUT') # sanity check - self.assertNotIn("X-Delete-At", put_headers) - self.assertNotIn("X-Delete-After", put_headers) - self.assertNotIn("ETag", put_headers) - - def test_etag_checking(self): - wsgi_input = BytesIO(b"unsplashed-comprest") - right_etag = hashlib.md5(wsgi_input.getvalue()).hexdigest() - wrong_etag = hashlib.md5(wsgi_input.getvalue() + b"abc").hexdigest() - non_checksum_etag = "pfsv2/AUTH_test/2226116/4341333-32" - cl = str(len(wsgi_input.getvalue())) - - wsgi_input.seek(0) - req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input}, - headers={"ETag": right_etag, - "Content-Length": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - - wsgi_input.seek(0) - req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input}, - headers={"ETag": wrong_etag, - "Content-Length": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '422 Unprocessable Entity') - - wsgi_input.seek(0) - req = swob.Request.blank("/v1/AUTH_test/a-container/an-object", - environ={"REQUEST_METHOD": "PUT", - "wsgi.input": wsgi_input}, - headers={"ETag": non_checksum_etag, - "Content-Length": cl}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - - def test_etag_checking_dir(self): - req = swob.Request.blank( - "/v1/AUTH_test/a-container/a-dir-object", - environ={"REQUEST_METHOD": "PUT"}, - headers={"ETag": hashlib.md5(b"x").hexdigest(), - "Content-Length": 0, - "Content-Type": "application/directory"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '422 Unprocessable Entity') - - req = swob.Request.blank( - "/v1/AUTH_test/a-container/a-dir-object", - environ={"REQUEST_METHOD": "PUT"}, - headers={"ETag": hashlib.md5(b"").hexdigest(), - "Content-Length": 0, - "Content-Type": "application/directory"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - - req = swob.Request.blank( - "/v1/AUTH_test/a-container/a-dir-object", - environ={"REQUEST_METHOD": "PUT"}, - headers={"Content-Length": 0, - "Content-Type": "application/directory"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - - -class TestObjectPost(BaseMiddlewareTest): - def test_missing_object(self): - def mock_RpcHead(_): - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "POST"}, - headers={"X-Object-Meta-One-Fish": "two fish"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("404 Not Found", status) - - rpc_calls = self.fake_rpc.calls - self.assertEqual(2, len(rpc_calls)) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") - - def test_file_as_dir(self): - # Subdirectories of files don't exist, but asking for one returns a - # different error code than asking for a file that could exist but - # doesn't. - def mock_RpcHead(head_object_req): - self.assertEqual(head_object_req['VirtPath'], - "/v1/AUTH_test/c/thing.txt/kitten.png") - - return { - "error": "errno: 20", - "result": None} - - req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png', - method='POST') - self.fake_rpc.register_handler("Server.RpcHead", mock_RpcHead) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_existing_object(self): - old_meta = json.dumps({ - "Content-Type": "application/fishy", - mware.ORIGINAL_MD5_HEADER: "1:a860580f9df567516a3f0b55c6b93b67", - "X-Object-Meta-One-Fish": "two fish"}) - - def mock_RpcHead(_): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - old_meta.encode('ascii')).decode('ascii'), - "ModificationTime": 1482345542483719281, - "FileSize": 551155, - "IsDir": False, - "InodeNumber": 6519913, - "NumWrites": 1}} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - self.fake_rpc.register_handler( - "Server.RpcPost", lambda *a: {"error": None, "result": {}}) - - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "POST"}, - headers={"X-Object-Meta-Red-Fish": "blue fish"}) - status, headers, _ = self.call_pfs(req) - - # For reference, the result of a real object POST request. The - # request contained new metadata items, but notice that those are - # not reflected in the response. - # - # HTTP/1.1 202 Accepted - # Content-Length: 76 - # Content-Type: text/html; charset=UTF-8 - # X-Trans-Id: tx65d5632fe4a548999ee9e-005a39781b - # X-Openstack-Request-Id: tx65d5632fe4a548999ee9e-005a39781b - # Date: Tue, 19 Dec 2017 20:35:39 GMT - self.assertEqual("202 Accepted", status) - self.assertNotIn('Last-Modified', headers) - self.assertNotIn('Etag', headers) - self.assertEqual("0", headers["Content-Length"]) - self.assertEqual("text/html; charset=UTF-8", headers["Content-Type"]) - # Date and X-Trans-Id are added in by other parts of the WSGI stack - - rpc_calls = self.fake_rpc.calls - self.assertEqual(3, len(rpc_calls)) - - method, args = rpc_calls[0] - self.assertEqual(method, "Server.RpcIsAccountBimodal") - self.assertEqual(args[0]["AccountName"], "AUTH_test") - - method, args = rpc_calls[1] - self.assertEqual(method, "Server.RpcHead") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPost") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") - self.assertEqual( - base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) - new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) - self.assertEqual(new_meta["X-Object-Meta-Red-Fish"], "blue fish") - self.assertEqual(new_meta["Content-Type"], "application/fishy") - self.assertNotIn("X-Object-Meta-One-Fish", new_meta) - - def test_preservation(self): - old_meta = json.dumps({ - "Content-Type": "application/fishy", - mware.ORIGINAL_MD5_HEADER: "1:a860580f9df567516a3f0b55c6b93b67", - "X-Static-Large-Object": "true", - "X-Object-Manifest": "solo/duet", - "X-Object-Sysmeta-Dog": "collie", - "X-Object-Meta-Fish": "perch"}) - - def mock_RpcHead(_): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - old_meta.encode('ascii')).decode('ascii'), - "ModificationTime": 1510873171878460000, - "FileSize": 7748115, - "IsDir": False, - "InodeNumber": 3741569, - "NumWrites": 1}} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - self.fake_rpc.register_handler( - "Server.RpcPost", lambda *a: {"error": None, "result": {}}) - - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "POST"}, - headers={"X-Object-Meta-Fish": "trout"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("202 Accepted", status) # sanity check - - method, args = self.fake_rpc.calls[2] - self.assertEqual(method, "Server.RpcPost") - new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) - self.assertEqual(new_meta["Content-Type"], "application/fishy") - self.assertEqual(new_meta["X-Object-Sysmeta-Dog"], "collie") - self.assertEqual(new_meta["X-Static-Large-Object"], "true") - self.assertEqual(new_meta["X-Object-Meta-Fish"], "trout") - self.assertNotIn("X-Object-Manifest", new_meta) - - def test_change_content_type(self): - old_meta = json.dumps({"Content-Type": "old/type"}) - - def mock_RpcHead(_): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - old_meta.encode('ascii')).decode('ascii'), - "ModificationTime": 1482345542483719281, - "FileSize": 551155, - "IsDir": False, - "InodeNumber": 6519913, - "NumWrites": 381}} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - self.fake_rpc.register_handler( - "Server.RpcPost", lambda *a: {"error": None, "result": {}}) - - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "POST"}, - headers={"Content-Type": "new/type"}) - status, _, _ = self.call_pfs(req) - self.assertEqual("202 Accepted", status) - - rpc_calls = self.fake_rpc.calls - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPost") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") - self.assertEqual( - base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) - new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) - self.assertEqual(new_meta["Content-Type"], "new/type") - - def test_metadata_blanks(self): - old_meta = json.dumps({"X-Object-Meta-Color": "red"}) - - def mock_RpcHead(_): - return { - "error": None, - "result": { - "Metadata": base64.b64encode( - old_meta.encode('ascii')).decode('ascii'), - "ModificationTime": 1482345542483719281, - "FileSize": 551155, - "IsDir": False, - "InodeNumber": 6519913, - "NumWrites": 381}} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - self.fake_rpc.register_handler( - "Server.RpcPost", lambda *a: {"error": None, "result": {}}) - - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "POST"}, - headers={"X-Object-Meta-Color": ""}) - status, _, _ = self.call_pfs(req) - self.assertEqual("202 Accepted", status) - - rpc_calls = self.fake_rpc.calls - - method, args = rpc_calls[2] - self.assertEqual(method, "Server.RpcPost") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") - self.assertEqual( - base64.b64decode(args[0]["OldMetaData"]).decode('ascii'), old_meta) - new_meta = json.loads(base64.b64decode(args[0]["NewMetaData"])) - self.assertNotIn("X-Object-Meta-Color", new_meta) - - -class TestObjectDelete(BaseMiddlewareTest): - def test_success(self): - def fake_RpcDelete(delete_request): - return {"error": None, "result": {}} - - self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "DELETE"}) - - status, _, _ = self.call_pfs(req) - self.assertEqual(status, "204 No Content") - - # the first call is a call to RpcIsAccountBimodal - self.assertEqual(len(self.fake_rpc.calls), 2) - self.assertEqual(self.fake_rpc.calls[1][1][0]["VirtPath"], - "/v1/AUTH_test/con/obj") - - def test_not_found(self): - def fake_RpcDelete(delete_request): - return {"error": "errno: 2", # NotFoundError / ENOENT - "result": None} - - self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "DELETE"}) - - status, _, _ = self.call_pfs(req) - self.assertEqual(status, "404 Not Found") - - def test_no_such_dir(self): - def fake_RpcDelete(delete_request): - return {"error": "errno: 20", # NotDirError - "result": None} - - self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "DELETE"}) - - status, _, _ = self.call_pfs(req) - self.assertEqual(status, "404 Not Found") - - def test_not_empty(self): - def fake_RpcDelete(delete_request): - return {"error": "errno: 39", # NotEmptyError / ENOTEMPTY - "result": None} - - self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "DELETE"}) - - status, _, _ = self.call_pfs(req) - self.assertEqual(status, "409 Conflict") - - def test_other_failure(self): - def fake_RpcDelete(delete_request): - return {"error": "errno: 9", # BadFileError / EBADF - "result": None} - - self.fake_rpc.register_handler("Server.RpcDelete", fake_RpcDelete) - - req = swob.Request.blank("/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "DELETE"}) - - status, _, _ = self.call_pfs(req) - self.assertEqual(status, "500 Internal Error") - - -class TestObjectHead(BaseMiddlewareTest): - def setUp(self): - super(TestObjectHead, self).setUp() - - self.serialized_object_metadata = "" - - # All these tests run against the same object. - def mock_RpcHead(head_object_req): - self.assertEqual(head_object_req['VirtPath'], - '/v1/AUTH_test/c/an-object.png') - - if self.serialized_object_metadata: - md = base64.b64encode( - self.serialized_object_metadata.encode('ascii') - ).decode('ascii') - else: - md = "" - - return { - "error": None, - "result": { - "Metadata": md, - "ModificationTime": 1479173168018879490, - "FileSize": 2641863, - "IsDir": False, - "InodeNumber": 4591, - "NumWrites": 874, - }} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - # For reference: the result of a real HEAD request - # - # swift@saio:~/swift$ curl -I -H "X-Auth-Token: $TOKEN" \ - # http://localhost:8080/v1/AUTH_test/s/tox.ini - # HTTP/1.1 200 OK - # Content-Length: 3954 - # Content-Type: application/octet-stream - # Accept-Ranges: bytes - # Last-Modified: Mon, 14 Nov 2016 23:29:00 GMT - # Etag: ceffb3138058597bd7f8a09bdd3865d0 - # X-Timestamp: 1479166139.38308 - # X-Object-Meta-Mtime: 1476487400.000000 - # X-Trans-Id: txeaa367b1809a4583af3c8-00582a4a6c - # Date: Mon, 14 Nov 2016 23:36:12 GMT - # - - def test_file_as_dir(self): - # Subdirectories of files don't exist, but asking for one returns a - # different error code than asking for a file that could exist but - # doesn't. - def mock_RpcHead(head_object_req): - self.assertEqual(head_object_req['VirtPath'], - "/v1/AUTH_test/c/thing.txt/kitten.png") - - return { - "error": "errno: 20", - "result": None} - - req = swob.Request.blank('/v1/AUTH_test/c/thing.txt/kitten.png', - method='HEAD') - self.fake_rpc.register_handler("Server.RpcHead", mock_RpcHead) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_no_meta(self): - self.serialized_object_metadata = "" - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - self.assertEqual(headers["Content-Length"], "2641863") - self.assertEqual(headers["Content-Type"], "image/png") - self.assertEqual(headers["Accept-Ranges"], "bytes") - self.assertEqual(headers["ETag"], - '"pfsv2/AUTH_test/000011EF/0000036A-32"') - self.assertEqual(headers["Last-Modified"], - "Tue, 15 Nov 2016 01:26:09 GMT") - self.assertEqual(headers["X-Timestamp"], "1479173168.01888") - - def test_explicit_content_type(self): - self.serialized_object_metadata = json.dumps({ - "Content-Type": "Pegasus/inartistic"}) - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Type"], "Pegasus/inartistic") - - def test_bogus_meta(self): - self.serialized_object_metadata = "{[{[{[{[{[[(((!" - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - def test_meta(self): - self.serialized_object_metadata = json.dumps({ - "X-Object-Sysmeta-Fish": "cod", - "X-Object-Meta-Fish": "trout"}) - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers["X-Object-Sysmeta-Fish"], "cod") - self.assertEqual(headers["X-Object-Meta-Fish"], "trout") - - def test_none_meta(self): - # Sometimes we get a null in the response instead of an empty - # string. No idea why. - self.serialized_object_metadata = None - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - def test_special_chars(self): - def mock_RpcHead(head_object_req): - self.assertEqual(head_object_req['VirtPath'], - '/v1/AUTH_test/c/a cat.png') - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank("/v1/AUTH_test/c/a cat.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_not_found(self): - def mock_RpcHead(head_object_req): - self.assertEqual(head_object_req['VirtPath'], - '/v1/AUTH_test/c/an-object.png') - return {"error": "errno: 2", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_other_error(self): - def mock_RpcHead(head_object_req): - self.assertEqual(head_object_req['VirtPath'], - '/v1/AUTH_test/c/an-object.png') - return {"error": "errno: 7581", "result": None} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '500 Internal Error') - - def test_md5_etag(self): - self.serialized_object_metadata = json.dumps({ - "Content-Type": "Pegasus/inartistic", - mware.ORIGINAL_MD5_HEADER: "1:b61d068208b52f4acbd618860d30faae", - }) - - def mock_RpcHead(head_object_req): - md = base64.b64encode( - self.serialized_object_metadata.encode('ascii') - ).decode('ascii') - return { - "error": None, - "result": { - "Metadata": md, - "ModificationTime": 1506039770222591000, - "FileSize": 3397331, - "IsDir": False, - "InodeNumber": 1433230, - "NumWrites": 1, - }} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Etag"], "b61d068208b52f4acbd618860d30faae") - - def test_conditional_if_match(self): - obj_etag = "c2185d19ada5f5c3aa47d6b55dc912df" - - self.serialized_object_metadata = json.dumps({ - mware.ORIGINAL_MD5_HEADER: "1:%s" % obj_etag, - }) - - def mock_RpcHead(head_object_req): - md = base64.b64encode( - self.serialized_object_metadata.encode('ascii') - ).decode('ascii') - return { - "error": None, - "result": { - "Metadata": md, - "ModificationTime": 1511223407565078000, - "FileSize": 152874, - "IsDir": False, - "InodeNumber": 9566555, - "NumWrites": 1, - }} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}, - headers={"If-Match": obj_etag}) - # matches - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '200 OK') - - # doesn't match - req = swob.Request.blank("/v1/AUTH_test/c/an-object.png", - environ={"REQUEST_METHOD": "HEAD"}, - headers={"If-Match": obj_etag + "XYZZY"}) - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '412 Precondition Failed') - - -class TestObjectHeadDir(BaseMiddlewareTest): - def test_dir(self): - def mock_RpcHead(head_object_req): - self.assertEqual(head_object_req['VirtPath'], - '/v1/AUTH_test/c/a-dir') - - return { - "error": None, - "result": { - "Metadata": "", - "ModificationTime": 1479173168018879490, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 1254, - "NumWrites": 896, - }} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - req = swob.Request.blank("/v1/AUTH_test/c/a-dir", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(headers["Content-Length"], "0") - self.assertEqual(headers["Content-Type"], "application/directory") - self.assertEqual(headers["ETag"], "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(headers["Last-Modified"], - "Tue, 15 Nov 2016 01:26:09 GMT") - - -class TestObjectCoalesce(BaseMiddlewareTest): - def setUp(self): - super(TestObjectCoalesce, self).setUp() - - def mock_RpcHead(head_container_req): - return { - "error": None, - "result": { - "Metadata": "", - "ModificationTime": 1485814697697650000, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 1828, - "NumWrites": 893, - }} - - # this handler gets overwritten by the first test that - # registers a "Server.RpcHead" - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - def test_success(self): - - # a map from an object's "virtual path" to its metadata - self.obj_metadata = {} - - def mock_RpcHead(head_req): - '''Return the object informattion for the new coalesced object. This - assumes that COALESCE operation has already created it. - ''' - - resp = { - "error": None, - "result": { - "Metadata": "", - "ModificationTime": 1488323796002909000, - "FileSize": 80 * 1024 * 1024, - "IsDir": False, - "InodeNumber": 283253, - "NumWrites": 5, - } - } - virt_path = head_req['VirtPath'] - if self.obj_metadata[virt_path] is not None: - resp['result']['Metadata'] = self.obj_metadata[virt_path] - return resp - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - def mock_RpcCoalesce(coalesce_req): - - # if there's metadata for the new object, save it to return later - if coalesce_req['NewMetaData'] != "": - virt_path = coalesce_req['VirtPath'] - self.obj_metadata[virt_path] = coalesce_req['NewMetaData'] - - numWrites = len(coalesce_req['ElementAccountRelativePaths']) - return { - "error": None, - "result": { - "ModificationTime": 1488323796002909000, - "InodeNumber": 283253, - "NumWrites": numWrites, - }} - - self.fake_rpc.register_handler( - "Server.RpcCoalesce", mock_RpcCoalesce) - - # have the coalesce request suppply the headers that would - # come from s3api for a "complete multi-part upload" request - request_headers = { - 'X-Object-Sysmeta-S3Api-Acl': - '{"Owner":"fc",' + - '"Grant":[{"Grantee":"fc","Permission":"FULL_CONTROL"}]}', - 'X-Object-Sysmeta-S3Api-Etag': - 'cb45770d6cf51effdfb2ea35322459c3-205', - 'X-Object-Sysmeta-Slo-Etag': '363d958f0f4c8501a50408a728ba5599', - 'X-Object-Sysmeta-Slo-Size': '1073741824', - 'X-Object-Sysmeta-Container-Update-Override-Etag': - '10340ab593ac8c32290a278e36d1f8df; ' + - 's3_etag=cb45770d6cf51effdfb2ea35322459c3-205; ' + - 'slo_etag=363d958f0f4c8501a50408a728ba5599', - 'X-Static-Large-Object': 'True', - 'Content-Type': - 'application/x-www-form-urlencoded; ' + - 'charset=utf-8;swift_bytes=1073741824', - 'Etag': '10340ab593ac8c32290a278e36d1f8df', - 'Date': 'Mon, 01 Apr 2019 22:53:31 GMT', - 'Host': 'sm-p1.swiftstack.org', - 'Accept': 'application/json', - 'Content-Length': '18356', - 'X-Auth-Token': None, - } - request_body = json.dumps({ - "elements": [ - "c1/seg1a", - "c1/seg1b", - "c2/seg space 2a", - "c2/seg space 2b", - "c3/seg3", - ]}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - headers=request_headers, - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '201 Created') - self.assertEqual(headers["Etag"], '10340ab593ac8c32290a278e36d1f8df') - - # The first call is a call to RpcIsAccountBimodal, the last is the one - # we care about: RpcCoalesce. There *could* be some intervening calls - # to RpcHead to authorize read/write access to the segments, but since - # there's no authorize callback installed, we skip it. - self.assertEqual([method for method, args in self.fake_rpc.calls], [ - "Server.RpcIsAccountBimodal", - "Server.RpcCoalesce", - ]) - method, args = self.fake_rpc.calls[-1] - self.assertEqual(method, "Server.RpcCoalesce") - self.assertEqual(args[0]["VirtPath"], "/v1/AUTH_test/con/obj") - self.assertEqual(args[0]["ElementAccountRelativePaths"], [ - "c1/seg1a", - "c1/seg1b", - "c2/seg space 2a", - "c2/seg space 2b", - "c3/seg3", - ]) - - # verify that the metadata was munged correctly - # (SLO headers stripped out, etc.) - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "HEAD"}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK') - self.assertEqual(body, b'') - self.assertEqual(headers["Etag"], - '10340ab593ac8c32290a278e36d1f8df') - self.assertIn('X-Object-Sysmeta-S3Api-Acl', headers) - self.assertIn('X-Object-Sysmeta-S3Api-Etag', headers) - self.assertIn('X-Object-Sysmeta-Container-Update-Override-Etag', - headers) - self.assertNotIn('X-Static-Large-Object', headers) - self.assertNotIn('X-Object-Sysmeta-Slo-Etag', headers) - self.assertNotIn('X-Object-Sysmeta-Slo-Size', headers) - - def test_not_authed(self): - def mock_RpcHead(get_container_req): - path = get_container_req['VirtPath'] - return { - "error": None, - "result": { - "Metadata": base64.b64encode(json.dumps({ - "X-Container-Read": path + '\x00read-acl', - "X-Container-Write": path + '\x00write-acl', - }).encode('ascii')).decode('ascii'), - "ModificationTime": 1479240451156825194, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 1255, - "NumWrites": 897, - }} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - acls = [] - - def auth_cb(req): - acls.append(req.acl) - # write access for the target, plus 3 distince element containers - # each needing both read and write checks -- fail the last - if len(acls) >= 7: - return swob.HTTPForbidden(request=req) - - request_body = json.dumps({ - "elements": [ - "c1/seg1a", - "c1/seg1b", - "c2/seg space 2a", - "c2/seg space 2b", - "c3/seg3", - ]}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body), - "swift.authorize": auth_cb}) - status, headers, body = self.call_pfs(req) - # The first call is a call to RpcIsAccountBimodal, then a bunch of - # RpcHead calls as we authorize the four containers involved. Since - # we fail the authorization, we never make it to RpcCoalesce. - self.assertEqual([method for method, args in self.fake_rpc.calls], [ - "Server.RpcIsAccountBimodal", - "Server.RpcHead", - "Server.RpcHead", - "Server.RpcHead", - "Server.RpcHead", - ]) - container_paths = [ - "/v1/AUTH_test/con", - "/v1/AUTH_test/c1", - "/v1/AUTH_test/c2", - "/v1/AUTH_test/c3", - ] - self.assertEqual(container_paths, [ - args[0]['VirtPath'] for method, args in self.fake_rpc.calls[1:]]) - self.assertEqual(status, '403 Forbidden') - self.assertEqual(acls, [ - "/v1/AUTH_test/con\x00write-acl", - "/v1/AUTH_test/c1\x00read-acl", - "/v1/AUTH_test/c1\x00write-acl", - "/v1/AUTH_test/c2\x00read-acl", - "/v1/AUTH_test/c2\x00write-acl", - "/v1/AUTH_test/c3\x00read-acl", - "/v1/AUTH_test/c3\x00write-acl", - ]) - - def test_malformed_json(self): - request_body = b"{{{[[[(((" - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request') - - def test_incorrect_json(self): - request_body = b"{}" - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request') - - def test_incorrect_json_wrong_type(self): - request_body = b"[]" - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request') - - def test_incorrect_json_wrong_elements_type(self): - request_body = json.dumps({"elements": {1: 2}}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request') - - def test_incorrect_json_wrong_element_type(self): - request_body = json.dumps({"elements": [1, "two", {}]}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request') - - def test_incorrect_json_subtle(self): - request_body = b'["elements"]' - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request') - - def test_too_big(self): - request_body = b'{' * (self.pfs.max_coalesce_request_size + 1) - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '413 Request Entity Too Large') - - def test_too_many_elements(self): - request_body = json.dumps({ - "elements": ["/c/o"] * (self.pfs.max_coalesce + 1), - }).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '413 Request Entity Too Large') - - def test_not_found(self): - # Note: this test covers all sorts of thing-not-found errors, as the - # Server.RpcCoalesce remote procedure does not distinguish between - # them. In particular, this covers the case when an element is not - # found as well as the case when the destination container is not - # found. - def mock_RpcCoalesce(coalese_req): - return { - "error": "errno: 2", - "result": None, - } - - self.fake_rpc.register_handler( - "Server.RpcCoalesce", mock_RpcCoalesce) - - request_body = json.dumps({ - "elements": [ - "some/stuff", - ]}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '404 Not Found') - - def test_coalesce_directory(self): - # This happens when one of the named elements is a directory. - def mock_RpcCoalesce(coalese_req): - return { - "error": "errno: 21", - "result": None, - } - - self.fake_rpc.register_handler( - "Server.RpcCoalesce", mock_RpcCoalesce) - - request_body = json.dumps({ - "elements": [ - "some/stuff", - ]}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '409 Conflict') - - def test_coalesce_file_instead_of_dir(self): - # This happens when one of the named elements is not a regular file. - # Yes, you get IsDirError for not-a-file, even if it's a symlink. - def mock_RpcCoalesce(coalese_req): - return { - "error": "errno: 20", - "result": None, - } - - self.fake_rpc.register_handler( - "Server.RpcCoalesce", mock_RpcCoalesce) - - request_body = json.dumps({ - "elements": [ - "some/stuff", - ]}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '409 Conflict') - - def test_element_has_multiple_links(self): - # If a file has multiple links to it (hard links, not symlinks), - # then we can't coalesce it. - def mock_RpcCoalesce(coalese_req): - return { - "error": "errno: 31", - "result": None, - } - - self.fake_rpc.register_handler( - "Server.RpcCoalesce", mock_RpcCoalesce) - - request_body = json.dumps({ - "elements": [ - "some/stuff", - ]}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '409 Conflict') - - def test_other_error(self): - # If a file has multiple links to it (hard links, not symlinks), - # then we can't coalesce it. - def mock_RpcCoalesce(coalese_req): - return { - "error": "errno: 1159268", - "result": None, - } - - self.fake_rpc.register_handler( - "Server.RpcCoalesce", mock_RpcCoalesce) - - request_body = json.dumps({ - "elements": [ - "thing/one", - "thing/two", - ]}).encode('ascii') - req = swob.Request.blank( - "/v1/AUTH_test/con/obj", - environ={"REQUEST_METHOD": "COALESCE", - "wsgi.input": BytesIO(request_body)}) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '500 Internal Error') - - -class TestAuth(BaseMiddlewareTest): - def setUp(self): - super(TestAuth, self).setUp() - - def mock_RpcHead(get_container_req): - return { - "error": None, - "result": { - "Metadata": base64.b64encode(json.dumps({ - "X-Container-Read": "the-x-con-read", - "X-Container-Write": "the-x-con-write", - }).encode('ascii')).decode('ascii'), - "ModificationTime": 1479240451156825194, - "FileSize": 0, - "IsDir": True, - "InodeNumber": 1255, - "NumWrites": 897, - }} - - self.fake_rpc.register_handler( - "Server.RpcHead", mock_RpcHead) - - def test_auth_callback_args(self): - want_x_container_read = (('GET', '/v1/AUTH_test/con/obj'), - ('HEAD', '/v1/AUTH_test/con/obj'), - ('GET', '/v1/AUTH_test/con'), - ('HEAD', '/v1/AUTH_test/con')) - - want_x_container_write = (('PUT', '/v1/AUTH_test/con/obj'), - ('POST', '/v1/AUTH_test/con/obj'), - ('DELETE', '/v1/AUTH_test/con/obj')) - got_acls = [] - - def capture_acl_and_deny(request): - got_acls.append(request.acl) - return swob.HTTPForbidden(request=request) - - for method, path in want_x_container_read: - req = swob.Request.blank( - path, environ={'REQUEST_METHOD': method, - 'swift.authorize': capture_acl_and_deny}) - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '403 Forbidden') - self.assertEqual( - got_acls, ["the-x-con-read"] * len(want_x_container_read)) - - del got_acls[:] - for method, path in want_x_container_write: - req = swob.Request.blank( - path, environ={'REQUEST_METHOD': method, - 'swift.authorize': capture_acl_and_deny}) - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '403 Forbidden') - self.assertEqual( - got_acls, ["the-x-con-write"] * len(want_x_container_write)) - - def test_auth_override(self): - def auth_nope(request): - return swob.HTTPForbidden(request=request) - - req = swob.Request.blank( - "/v1/AUTH_test/con", - environ={'REQUEST_METHOD': 'HEAD', - 'swift.authorize': auth_nope, - 'swift.authorize_override': True}) - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '403 Forbidden') - - def test_auth_bypass(self): - def auth_nope(request): - return swob.HTTPForbidden(request=request) - - # pfs is re-entrant when getting ACLs from what may be other accounts. - req = swob.Request.blank( - "/v1/AUTH_test/con", - environ={'REQUEST_METHOD': 'HEAD', - 'swift.authorize': auth_nope, - 'swift.authorize_override': True, - 'swift.source': 'PFS'}) - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - - def test_auth_allowed(self): - def auth_its_fine(request): - return None - - req = swob.Request.blank( - "/v1/AUTH_test/con", - environ={'REQUEST_METHOD': 'HEAD', - 'swift.authorize': auth_its_fine}) - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '204 No Content') - - -class TestEtagHandling(unittest.TestCase): - '''Test that mung_etags()/unmung_etags() are inverse functions (more - or less), that the unmung_etags() correctly unmungs the current - disk layout for the header, and that best_possible_etag() returns - the correct etag value. - ''' - - # Both the header names and the way the values are calculated and - # the way they are formatted are part of the "disk layout". If we - # change them, we have to consider handling old data. - # - # Names and values duplicated here so this test will break if they - # are changed. - HEADER = "X-Object-Sysmeta-ProxyFS-Initial-MD5" - - ETAG_HEADERS = { - "ORIGINAL_MD5": { - "name": "X-Object-Sysmeta-ProxyFS-Initial-MD5", - "value": "5484c2634aa61c69fc02ef5400a61c94", - "num_writes": 7, - "munged_value": "7:5484c2634aa61c69fc02ef5400a61c94" - }, - "S3API_ETAG": { - "name": "X-Object-Sysmeta-S3Api-Etag", - "value": "cb0dc66d591395cdf93555dafd4145ad", - "num_writes": 3, - "munged_value": "3:cb0dc66d591395cdf93555dafd4145ad" - }, - "LISTING_ETAG_OVERRIDE": { - "name": "X-Object-Sysmeta-Container-Update-Override-Etag", - "value": "dbca19e0c46aa9b10e8de2f5856abc86", - "num_writes": 9, - "munged_value": "9:dbca19e0c46aa9b10e8de2f5856abc86" - }, - } - - EMPTY_OBJECT_ETAG = "d41d8cd98f00b204e9800998ecf8427e" - - def test_mung_etags(self): - '''Verify that mung_etags() mungs each of the etag header values to - the correct on-disk version with num_writes and unmung_etags() - recovers the original value. - ''' - for header in self.ETAG_HEADERS.keys(): - - name = self.ETAG_HEADERS[header]["name"] - value = self.ETAG_HEADERS[header]["value"] - num_writes = self.ETAG_HEADERS[header]["num_writes"] - munged_value = self.ETAG_HEADERS[header]["munged_value"] - - obj_metadata = {name: value} - mware.mung_etags(obj_metadata, None, num_writes) - - # The handling of ORIGINAL_MD5_HEADER is a bit strange. - # If an etag value is passed to mung_etags() as the 2nd - # argument then the header is created and assigned the - # munged version of that etag value. But if its None than - # an ORIGINAL_MD5_HEADER, if present, is passed through - # unmolested (but will be dropped by the unmung code if - # the format is wrong or num_writes does not match, which - # is likely). - if header == "ORIGINAL_MD5": - self.assertEqual(obj_metadata[name], value) - else: - self.assertEqual(obj_metadata[name], munged_value) - - # same test but with an etag value passed to mung_etags() - # instead of None - md5_etag_value = self.ETAG_HEADERS["ORIGINAL_MD5"]["value"] - - obj_metadata = {name: value} - mware.mung_etags(obj_metadata, md5_etag_value, num_writes) - self.assertEqual(obj_metadata[name], munged_value) - - # and verify that it gets unmunged to the original value - mware.unmung_etags(obj_metadata, num_writes) - self.assertEqual(obj_metadata[name], value) - - def test_unmung_etags(self): - '''Verify that unmung_etags() recovers the correct value for - each ETag header and discards header values where num_writes - is incorrect or the value is corrupt. - ''' - # build metadata with all of the etag headers - orig_obj_metadata = {} - for header in self.ETAG_HEADERS.keys(): - - name = self.ETAG_HEADERS[header]["name"] - munged_value = self.ETAG_HEADERS[header]["munged_value"] - orig_obj_metadata[name] = munged_value - - # unmung_etags() should strip out headers with stale - # num_writes so only the one for header remains - for header in self.ETAG_HEADERS.keys(): - - name = self.ETAG_HEADERS[header]["name"] - value = self.ETAG_HEADERS[header]["value"] - num_writes = self.ETAG_HEADERS[header]["num_writes"] - munged_value = self.ETAG_HEADERS[header]["munged_value"] - - obj_metadata = orig_obj_metadata.copy() - mware.unmung_etags(obj_metadata, num_writes) - - # header should be the only one left - for hdr2 in self.ETAG_HEADERS.keys(): - - if hdr2 == header: - self.assertEqual(obj_metadata[name], value) - else: - self.assertTrue( - self.ETAG_HEADERS[hdr2]["name"] not in obj_metadata) - - # verify mung_etags() takes it back to the munged value - if header != "ORIGINAL_MD5": - mware.mung_etags(obj_metadata, None, num_writes) - self.assertEqual(obj_metadata[name], munged_value) - - # Try to unmung on-disk headers with corrupt values. Instead - # of a panic it should simply elide the headers (and possibly - # log the corrupt value, which it doesn't currently do). - for bad_value in ("counterfact-preformative", - "magpie:interfollicular"): - - obj_metadata = {} - for header in self.ETAG_HEADERS.keys(): - - name = self.ETAG_HEADERS[header]["name"] - obj_metadata[name] = bad_value - - mware.unmung_etags(obj_metadata, 0) - self.assertEqual(obj_metadata, {}) - - def test_best_possible_etag(self): - '''Test that best_possible_etag() returns the best ETag. - ''' - - # Start with metadata that consists of all possible ETag - # headers and verify that best_possible_headers() chooses the - # right ETag based on the circumstance. - obj_metadata = {} - for header in self.ETAG_HEADERS.keys(): - - name = self.ETAG_HEADERS[header]["name"] - value = self.ETAG_HEADERS[header]["value"] - obj_metadata[name] = value - - # directories always return the same etag - etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7, - is_dir=True) - self.assertEqual(etag, self.EMPTY_OBJECT_ETAG) - - # a container listing should return the Container Listing Override ETag - etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7, - container_listing=True) - self.assertEqual(etag, - self.ETAG_HEADERS["LISTING_ETAG_OVERRIDE"]["value"]) - - # if its not a directory and its not a container list, then - # the function should return the original MD5 ETag (the - # S3API_ETAG is returned as a header, but not as an ETag). - etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7) - self.assertEqual(etag, - self.ETAG_HEADERS["ORIGINAL_MD5"]["value"]) - - # if none of the headers provide the correct ETag then - # best_possible_etag() will construct a unique one (which also - # should not be changed without handling "old data", i.e. - # its part of the "disk layout"). - del obj_metadata[self.ETAG_HEADERS["ORIGINAL_MD5"]["name"]] - - etag = mware.best_possible_etag(obj_metadata, "AUTH_test", 42, 7) - self.assertEqual(etag, - '"pfsv2/AUTH_test/0000002A/00000007-32"') - - -class TestProxyfsMethod(BaseMiddlewareTest): - def test_non_swift_owner(self): - req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS') - status, _, _ = self.call_pfs(req) - self.assertEqual(status, '405 Method Not Allowed') - - def test_bad_content_type(self): - req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS', - environ={'swift_owner': True}, body=b'') - status, _, body = self.call_pfs(req) - self.assertEqual(status, '415 Unsupported Media Type') - self.assertEqual( - body, b'RPC body must have Content-Type application/json') - - req = swob.Request.blank("/v1/AUTH_test", method='PROXYFS', - headers={'Content-Type': 'text/plain'}, - environ={'swift_owner': True}, body=b'') - status, _, body = self.call_pfs(req) - self.assertEqual(status, '415 Unsupported Media Type') - self.assertEqual(body, (b'RPC body must have Content-Type ' - b'application/json, not text/plain')) - - def test_exactly_one_payload(self): - def expect_one_payload(body): - req = swob.Request.blank( - "/v1/AUTH_test", method='PROXYFS', - headers={'Content-Type': 'application/json'}, - environ={'swift_owner': True}, body=body) - status, _, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request') - self.assertEqual(body, b'Expected exactly one JSON payload') - - expect_one_payload(b'') - expect_one_payload(b'\n') - one_payload = json.dumps({ - 'jsonrpc': '2.0', - 'method': 'Server.RpcPing', - 'params': [{}], - }) - expect_one_payload(one_payload + '\n' + one_payload) - - def test_bad_body(self): - def expect_parse_error(req_body): - req = swob.Request.blank( - "/v1/AUTH_test", method='PROXYFS', - headers={'Content-Type': 'application/json'}, - environ={'swift_owner': True}, body=req_body) - status, _, body = self.call_pfs(req) - self.assertEqual(status, '400 Bad Request') - exp_start = (b'Could not parse/validate JSON payload #0 ' - b'%s:' % req_body) - self.assertEqual(body[:len(exp_start)], exp_start) - - expect_parse_error(b'[') - expect_parse_error(b'{') - expect_parse_error(b'[]') - expect_parse_error(b'{}') - - expect_parse_error(json.dumps({ - 'jsonrpc': '1.0', - 'method': 'Server.PingReq', - 'params': [{}], - }).encode('ascii')) - - expect_parse_error(json.dumps({ - 'jsonrpc': '2.0', - 'method': 'asdf', - 'params': [{}], - }).encode('ascii')) - - expect_parse_error(json.dumps({ - 'jsonrpc': '2.0', - 'method': 'Server.PingReq', - 'params': [{}, {}], - }).encode('ascii')) - expect_parse_error(json.dumps({ - 'jsonrpc': '2.0', - 'method': 'Server.PingReq', - 'params': [], - }).encode('ascii')) - expect_parse_error(json.dumps({ - 'jsonrpc': '2.0', - 'method': 'Server.PingReq', - 'params': [[]], - }).encode('ascii')) - expect_parse_error(json.dumps({ - 'jsonrpc': '2.0', - 'method': 'Server.PingReq', - 'params': {}, - }).encode('ascii')) - - def test_success(self): - def mock_RpcGetAccount(params): - self.assertEqual(params, { - "AccountName": "AUTH_test", - }) - return { - "error": None, - "result": { - "ModificationTime": 1498766381451119000, - "AccountEntries": [{ - "Basename": "chickens", - "ModificationTime": 1510958440808682000, - }, { - "Basename": "cows", - "ModificationTime": 1510958450657045000, - }, { - "Basename": "goats", - "ModificationTime": 1510958452544251000, - }, { - "Basename": "pigs", - "ModificationTime": 1510958459200130000, - }], - }} - - self.fake_rpc.register_handler( - "Server.RpcGetAccount", mock_RpcGetAccount) - - req = swob.Request.blank( - "/v1/AUTH_test", method='PROXYFS', - headers={'Content-Type': 'application/json'}, - environ={'swift_owner': True}, body=json.dumps({ - 'jsonrpc': '2.0', - 'method': 'Server.RpcGetAccount', - 'params': [{}], - })) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK', body) - self.assertEqual(headers.get('content-type'), 'application/json') - resp = json.loads(body) - self.assertIsNotNone(resp.pop('id', None)) - self.assertEqual(resp, { - "error": None, - "result": { - "ModificationTime": 1498766381451119000, - "AccountEntries": [{ - "Basename": "chickens", - "ModificationTime": 1510958440808682000, - }, { - "Basename": "cows", - "ModificationTime": 1510958450657045000, - }, { - "Basename": "goats", - "ModificationTime": 1510958452544251000, - }, { - "Basename": "pigs", - "ModificationTime": 1510958459200130000, - }] - } - }) - - def test_error(self): - def mock_RpcGetAccount(params): - self.assertEqual(params, { - "AccountName": "AUTH_test", - "some": "args", - }) - return {"error": "errno 2"} - - self.fake_rpc.register_handler( - "Server.RpcGetAccount", mock_RpcGetAccount) - - req = swob.Request.blank( - "/v1/AUTH_test", method='PROXYFS', - headers={'Content-Type': 'application/json'}, - environ={'swift_owner': True}, body=json.dumps({ - 'jsonrpc': '2.0', - 'method': 'Server.RpcGetAccount', - 'params': [{'some': 'args'}], - })) - status, headers, body = self.call_pfs(req) - self.assertEqual(status, '200 OK', body) - self.assertEqual(headers.get('content-type'), 'application/json') - resp = json.loads(body) - self.assertIsNotNone(resp.pop('id', None)) - self.assertEqual(resp, {"error": "errno 2"}) diff --git a/pfs_middleware/tests/test_rpc.py b/pfs_middleware/tests/test_rpc.py deleted file mode 100644 index a0c6e2f5..00000000 --- a/pfs_middleware/tests/test_rpc.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -import unittest - -import pfs_middleware.rpc as rpc - - -class TestAddressParsing(unittest.TestCase): - def test_ipv4(self): - resp = {"IsBimodal": True, - "ActivePeerPrivateIPAddr": "10.2.3.4"} - - _, parsed_ip = rpc.parse_is_account_bimodal_response(resp) - self.assertEqual(parsed_ip, "10.2.3.4") - - def test_ipv6_no_brackets(self): - addr = "fc00:df02:c928:4ef2:f085:8af2:cf1b:6b4" - resp = {"IsBimodal": True, - "ActivePeerPrivateIPAddr": addr} - - _, parsed_ip = rpc.parse_is_account_bimodal_response(resp) - self.assertEqual(parsed_ip, addr) - - def test_ipv6_brackets(self): - addr = "fc00:bbb9:a634:aa7d:8cb1:1c1e:d1cb:519c" - resp = {"IsBimodal": True, - "ActivePeerPrivateIPAddr": "[{0}]".format(addr)} - - _, parsed_ip = rpc.parse_is_account_bimodal_response(resp) - self.assertEqual(parsed_ip, addr) - - -class TestResponseParsing(unittest.TestCase): - # These maybe aren't great, but they're more than what we had - def test_get_object(self): - resp = { - "ReadEntsOut": 'a', - "Metadata": 'Yg==', # 'b' - "FileSize": 'c', - "ModificationTime": 'd', - "AttrChangeTime": 'e', - "IsDir": 'f', - "InodeNumber": 'g', - "NumWrites": 'h', - "LeaseId": 'i', - } - self.assertEqual(rpc.parse_get_object_response(resp), ( - 'a', 'b', 'c', 'e', 'f', 'g', 'h', 'i')) - - # older proxyfsd didn't send IsDir, so it will not be present in - # older responses, but older GET would fail on a directory object - # so False is correct if IsDir is not present. - del resp["IsDir"] - self.assertEqual(rpc.parse_get_object_response(resp), ( - 'a', 'b', 'c', 'e', False, 'g', 'h', 'i')) - - # Old proxyfsd didn't send AttrChangeTime, but we've always had - # ModificationTime available, which is the next-best option - del resp["AttrChangeTime"] - self.assertEqual(rpc.parse_get_object_response(resp), ( - 'a', 'b', 'c', 'd', False, 'g', 'h', 'i')) - - def test_coalesce_object(self): - resp = { - "ModificationTime": 'a', - "AttrChangeTime": 'b', - "InodeNumber": 'c', - "NumWrites": 'd', - } - self.assertEqual(rpc.parse_coalesce_object_response(resp), ( - 'b', 'c', 'd')) - # Old proxyfsd didn't send AttrChangeTime, but we've always had - # ModificationTime available, which is the next-best option - del resp["AttrChangeTime"] - self.assertEqual(rpc.parse_coalesce_object_response(resp), ( - 'a', 'c', 'd')) - - def test_put_complete(self): - resp = { - "ModificationTime": 'a', - "AttrChangeTime": 'b', - "InodeNumber": 'c', - "NumWrites": 'd', - } - self.assertEqual(rpc.parse_put_complete_response(resp), ( - 'b', 'c', 'd')) - # Old proxyfsd didn't send AttrChangeTime, but we've always had - # ModificationTime available, which is the next-best option - del resp["AttrChangeTime"] - self.assertEqual(rpc.parse_put_complete_response(resp), ( - 'a', 'c', 'd')) - - def test_mkdir(self): - resp = { - "ModificationTime": 'a', - "AttrChangeTime": 'b', - "InodeNumber": 'c', - "NumWrites": 'd', - } - self.assertEqual(rpc.parse_middleware_mkdir_response(resp), ( - 'b', 'c', 'd')) - # Old proxyfsd didn't send AttrChangeTime, but we've always had - # ModificationTime available, which is the next-best option - del resp["AttrChangeTime"] - self.assertEqual(rpc.parse_middleware_mkdir_response(resp), ( - 'a', 'c', 'd')) - - def test_get_account(self): - resp = { - "ModificationTime": 'a', - "AttrChangeTime": 'b', - "AccountEntries": ['c'], - } - self.assertEqual(rpc.parse_get_account_response(resp), ('b', ['c'])) - del resp["AttrChangeTime"] - self.assertEqual(rpc.parse_get_account_response(resp), ('a', ['c'])) - - resp = { - "ModificationTime": 'a', - "AttrChangeTime": 'b', - "AccountEntries": None, - } - self.assertEqual(rpc.parse_get_account_response(resp), ('b', [])) - del resp["AttrChangeTime"] - self.assertEqual(rpc.parse_get_account_response(resp), ('a', [])) - - def test_head(self): - resp = { - "Metadata": 'YQ==', # 'a' - "ModificationTime": 'b', - "AttrChangeTime": 'c', - "FileSize": 'd', - "IsDir": 'e', - "InodeNumber": 'f', - "NumWrites": 'g', - } - self.assertEqual(rpc.parse_head_response(resp), ( - 'a', 'c', 'd', 'e', 'f', 'g')) - - # Old proxyfsd didn't send AttrChangeTime, but we've always had - # ModificationTime available, which is the next-best option - del resp["AttrChangeTime"] - self.assertEqual(rpc.parse_head_response(resp), ( - 'a', 'b', 'd', 'e', 'f', 'g')) diff --git a/pfs_middleware/tests/test_s3_compat.py b/pfs_middleware/tests/test_s3_compat.py deleted file mode 100644 index 5ac64ba2..00000000 --- a/pfs_middleware/tests/test_s3_compat.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -import json -import unittest - -from swift.common import swob - -from . import helpers -import pfs_middleware.s3_compat as s3_compat -import pfs_middleware.utils as utils - - -class TestS3Compat(unittest.TestCase): - def setUp(self): - super(TestS3Compat, self).setUp() - self.app = helpers.FakeProxy() - self.s3_compat = s3_compat.S3Compat(self.app, {}) - - self.app.register( - 'PUT', '/v1/AUTH_test/con/obj', - 201, {}, '') - - def test_not_bimodal(self): - req = swob.Request.blank( - "/v1/AUTH_test/con/obj?multipart-manifest=put", - environ={'REQUEST_METHOD': 'PUT', - 'wsgi.input': swob.WsgiBytesIO(b"")}, - headers={'Content-Length': '0'}) - - resp = req.get_response(self.s3_compat) - self.assertEqual(resp.status_int, 201) - - def test_bimodal_but_not_swift3(self): - req = swob.Request.blank( - "/v1/AUTH_test/con/obj?multipart-manifest=put", - environ={'REQUEST_METHOD': 'PUT', - 'wsgi.input': swob.WsgiBytesIO(b""), - utils.ENV_IS_BIMODAL: True}, - headers={'Content-Length': '0'}) - - resp = req.get_response(self.s3_compat) - self.assertEqual(resp.status_int, 201) - - def test_conversion(self): - self.app.register( - 'COALESCE', '/v1/AUTH_test/con/obj', - 201, {}, '') - - # Note that this is in SLO's internal format since this request has - # already passed through SLO. - slo_manifest = [{ - "name": "/con-segments/obj/1506721327.316611/1", - "hash": "dontcare1", - "bytes": 12345678901, - }, { - "name": "/con-segments/obj/1506721327.316611/2", - "hash": "dontcare2", - "bytes": 12345678902, - }, { - "name": "/con-segments/obj/1506721327.316611/3", - "hash": "dontcare3", - "bytes": 12345678903, - }] - serialized_slo_manifest = json.dumps(slo_manifest).encode('utf-8') - - req = swob.Request.blank( - "/v1/AUTH_test/con/obj?multipart-manifest=put", - environ={'REQUEST_METHOD': 'PUT', - 'wsgi.input': swob.WsgiBytesIO(serialized_slo_manifest), - utils.ENV_IS_BIMODAL: True, - 'swift.source': 'S3'}, - headers={'Content-Length': str(len(slo_manifest))}) - - resp = req.get_response(self.s3_compat) - self.assertEqual(resp.status_int, 201) - self.assertEqual(self.app.calls[0][0], 'COALESCE') - - self.assertEqual(s3_compat.convert_slo_to_coalesce(slo_manifest), { - "elements": [ - "con-segments/obj/1506721327.316611/1", - "con-segments/obj/1506721327.316611/2", - "con-segments/obj/1506721327.316611/3"]}) diff --git a/pfs_middleware/tests/test_utils.py b/pfs_middleware/tests/test_utils.py deleted file mode 100644 index 56bd2abb..00000000 --- a/pfs_middleware/tests/test_utils.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - - -import unittest -import pfs_middleware.utils as utils - - -class TestHelperFunctions(unittest.TestCase): - def test_extract_errno(self): - self.assertEqual(2, utils.extract_errno("errno: 2")) - self.assertEqual(17, utils.extract_errno("errno: 17")) - self.assertEqual(None, utils.extract_errno("it broke")) - - def test_parse_path(self): - self.assertEqual( - utils.parse_path("/v1/a/c/o"), - ["v1", "a", "c", "o"]) - self.assertEqual( - utils.parse_path("/v1/a/c/obj/with/slashes"), - ["v1", "a", "c", "obj/with/slashes"]) - self.assertEqual( - utils.parse_path("/v1/a/c/obj/trailing/slashes///"), - ["v1", "a", "c", "obj/trailing/slashes///"]) - self.assertEqual( - utils.parse_path("/v1/a/c/"), - ["v1", "a", "c", None]) - self.assertEqual( - utils.parse_path("/v1/a/c"), - ["v1", "a", "c", None]) - self.assertEqual( - utils.parse_path("/v1/a/"), - ["v1", "a", None, None]) - self.assertEqual( - utils.parse_path("/v1/a"), - ["v1", "a", None, None]) - self.assertEqual( - utils.parse_path("/info"), - ["info", None, None, None]) - self.assertEqual( - utils.parse_path("/"), - [None, None, None, None]) diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini deleted file mode 100644 index 3f3ace19..00000000 --- a/pfs_middleware/tox.ini +++ /dev/null @@ -1,31 +0,0 @@ -[tox] -envlist = {py27,py36,py37}-{release,master},py27-minver,lint - -[testenv] -usedevelop = True -# minver tests with the earliest version of Swift we support. -deps = - lint: flake8 - !lint: -r{toxinidir}/test-requirements.txt - release: git+https://github.com/NVIDIA/swift.git@{env:LATEST_SWIFT_TAG} - minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz - master: http://tarballs.openstack.org/swift/swift-master.tar.gz -commands = python -m unittest discover - -[testenv:lint] -usedevelop = False -commands = flake8 {posargs:pfs_middleware tests setup.py} - -[flake8] -# flake8 has opinions with which we agree, for the most part. However, -# flake8 has a plugin mechanism by which other people can add their -# opinions; we do not necessarily like those opinions. In particular, -# "hacking" adds many different checks, a significant number of which -# are completely bogus. Fortunately, they have a convention: hacking -# checks start with "H", so that's what we ignore. -ignore = H, - # Both stupid binary opeator things - W503, - W504 -exclude = .venv,.tox,dist,*egg -show-source = true diff --git a/pfsagentConfig/Makefile b/pfsagentConfig/Makefile deleted file mode 100644 index 1ae0a6c6..00000000 --- a/pfsagentConfig/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfsagentConfig - -include ../GoMakefile diff --git a/pfsagentConfig/dataStructures.go b/pfsagentConfig/dataStructures.go deleted file mode 100644 index dbe5a2e6..00000000 --- a/pfsagentConfig/dataStructures.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package pfsagentConfig - -import "fmt" - -type stringStack struct { - items []string -} - -type stringArrayStack struct { - items [][]string -} - -type stringMapStack struct { - items []map[string]string -} - -func (s stringStack) push(val string) (err error) { - if len(val) == 0 { - err = fmt.Errorf("value cannot be zero-length") - } else { - s.items = append(s.items, val) - } - return -} - -func (s stringStack) pop() (val string) { - if len(s.items) == 0 { - return - } - val = s.peek() - s.items = s.items[:len(s.items)-2] - - return -} - -func (s stringStack) peek() (val string) { - val = s.items[len(s.items)-1] - return -} - -func (s stringArrayStack) push(val []string) (err error) { - if len(val) == 0 { - err = fmt.Errorf("value cannot be zero-length") - } else { - s.items = append(s.items, val) - } - return -} - -func (s stringArrayStack) pop() (val []string) { - if len(s.items) == 0 { - return - } - val = s.peek() - s.items = s.items[:len(s.items)-2] - - return -} - -func (s stringArrayStack) peek() (val []string) { - val = s.items[len(s.items)-1] - return -} - -func (s stringMapStack) push(val map[string]string) (err error) { - if len(val) == 0 { - err = fmt.Errorf("value cannot be zero-length") - } else { - s.items = append(s.items, val) - } - return -} - -func (s stringMapStack) pop() (val map[string]string) { - if len(s.items) == 0 { - return - } - val = s.peek() - s.items = s.items[:len(s.items)-2] - - return -} - -func (s stringMapStack) peek() (val map[string]string) { - val = s.items[len(s.items)-1] - return -} diff --git a/pfsagentConfig/menu.go b/pfsagentConfig/menu.go deleted file mode 100644 index 6681c85e..00000000 --- a/pfsagentConfig/menu.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package pfsagentConfig - -import ( - "fmt" -) - -func nextMenu(menuText string, menuOptions []string, menuOptionsTexts map[string]string) (userResponse string, err error) { - fmt.Println(menuText) - for { - displayOptions(menuOptions, menuOptionsTexts) - userResponse, err = getUserInput() - // fmt.Printf("raw response: %v\n", response) - if nil != err { - fmt.Println("Error retrieving user input", err) - continue - } - if len(userResponse) != 1 { - fmt.Printf("Input should be a single character. You entered %v. Please try again\n", userResponse) - continue - } - for _, validInput := range menuOptions { - if validInput == userResponse { - return menuOptionsTexts[userResponse], nil - } - - } - fmt.Printf("Your input is not valid: '%v'. Options are: %v. Please try again\n\n", userResponse, menuOptions) - } - // return "", fmt.Errorf("Something went wrong trying to read user input") -} - -func displayOptions(optionsList []string, optionsTextsMap map[string]string) { - for _, option := range optionsList { - fmt.Printf("%v:\t%v\n", option, optionsTextsMap[option]) - } - fmt.Printf("\nYour Selection: ") -} diff --git a/pfsagentConfig/pfsagentConfig/Makefile b/pfsagentConfig/pfsagentConfig/Makefile deleted file mode 100644 index d43c7af2..00000000 --- a/pfsagentConfig/pfsagentConfig/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfsagentConfig/pfsagentConfig - -include ../../GoMakefile diff --git a/pfsagentConfig/pfsagentConfig/dummy_test.go b/pfsagentConfig/pfsagentConfig/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfsagentConfig/pfsagentConfig/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfsagentConfig/pfsagentConfig/help.go b/pfsagentConfig/pfsagentConfig/help.go deleted file mode 100644 index 02d39ea8..00000000 --- a/pfsagentConfig/pfsagentConfig/help.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "os" -) - -var ( - usageText string = fmt.Sprintf(` -%v is a config utility for the SwiftStack ProxyFS Agent (pfsagentd). - -It can be used interactively by just calling it with no parameters, as such: -%[1]v - -[FUTURE] Or it can be used non-interactively by stating the parameter(s) you wish to modify, as such: -%[1]v swiftAuthURL swiftAuthUser .... - -For both interactive and non-interactive, the utility expects to find the the PFSagent config file at %[2]v. If you wish to use a different path, type -%[1]v configFile /path/to/config/file - -For a longer explanation, you can type -%[1]v help - -Which will also list all paraemters available for change. -For explanation about a specific parameter, type -%[1]v help - -To print this short usage message, type -%[1]v usage - -`, os.Args[0], defaultConfigPath) -) - -var version = fmt.Sprintf(` - pfsagentConfig - Client version: %v - API version: %v -`, "0.0.1", "0.0.1") - -// `, globalConf.Version, globalConf.APIVersion) diff --git a/pfsagentConfig/pfsagentConfig/main.go b/pfsagentConfig/pfsagentConfig/main.go deleted file mode 100644 index a2037381..00000000 --- a/pfsagentConfig/pfsagentConfig/main.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "os" - - pfsagentConfig "github.com/NVIDIA/proxyfs/pfsagentConfig" -) - -const ( - defaultConfigPath string = "/etc/pfsagent/" -) - -func main() { - if len(os.Args) == 1 { - loadError := pfsagentConfig.LoadConfig("") - if nil != loadError { - fmt.Println("Failed loading config. Error:", loadError) - os.Exit(1) - } - pfsagentConfig.RunStateMachine() - os.Exit(0) - } - - switch os.Args[1] { - case "firstRun": - err := firstTimeRun() - if nil != err { - fmt.Println(err) - os.Exit(2) - } - os.Exit(0) - case "configPath": - if len(os.Args) > 1 { - pfsagentConfig.ConfigPath = os.Args[2] - loadError := pfsagentConfig.LoadConfig("") - if nil != loadError { - fmt.Println("Failed loading config. Error:", loadError) - os.Exit(1) - } - pfsagentConfig.RunStateMachine() - os.Exit(0) - } - fmt.Print(usageText) - os.Exit(2) - case "version": - fmt.Print(version) - os.Exit(0) - case "usage": - fmt.Print(usageText) - os.Exit(0) - default: - fmt.Print(usageText) - os.Exit(2) - } -} - -func firstTimeRun() error { - return pfsagentConfig.FirstTimeRun() -} diff --git a/pfsagentConfig/stateMachine.go b/pfsagentConfig/stateMachine.go deleted file mode 100644 index afb6dc54..00000000 --- a/pfsagentConfig/stateMachine.go +++ /dev/null @@ -1,419 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package pfsagentConfig - -import ( - "fmt" - "os" -) - -// RunStateMachine run the state machine controlling the wizard flow -func RunStateMachine() (err error) { - // fmt.Println("runWizard starting") - prevMenuText := new(stringStack) - prevMenuOptions := new(stringArrayStack) - prevMenuOptionsTexts := new(stringMapStack) - prevMenuText.push(mainMenuText) - prevMenuOptions.push(mainMenuOptions) - prevMenuOptionsTexts.push(mainMenuOptionsTexts) - - nextMenuText := mainMenuText - nextMenuOptions := mainMenuOptions - nextMenuOptionsTexts := mainMenuOptionsTexts - for { - menuResponse, displayErr := nextMenu(nextMenuText, nextMenuOptions, nextMenuOptionsTexts) - // fmt.Printf("menuResponse: %v\n", menuResponse) - if nil != displayErr { - // fmt.Println("ERROR while displaying menu item", displayErr) - err = fmt.Errorf("error trying to display menu item") - return - } - switch menuResponse { - case quitMenuOptionText: - fmt.Println("Thank you for using the pfsagent config util") - return - case backMenuOptionText: - nextMenuText = prevMenuText.pop() - nextMenuOptions = prevMenuOptions.pop() - nextMenuOptionsTexts = prevMenuOptionsTexts.pop() - case changeCredsOptionText: - // fmt.Printf("got %v\n", changeCredsOptionText) - prevMenuText.push(nextMenuText) - prevMenuOptions.push(nextMenuOptions) - prevMenuOptionsTexts.push(nextMenuOptionsTexts) - nextMenuText = credentialsMenuTexts - nextMenuOptions = credentialsMenuOptions - nextMenuOptionsTexts = credentialsMenuOptionsTexts - case changeOtherOptionText: - fmt.Printf("got %v\n", changeOtherOptionText) - - case changeAuthURLOptionText: - userResponse, userInputErr := getValueFromUser("Swift Auth URL", "", confMap["Agent"]["SwiftAuthURL"][0]) - if nil != userInputErr { - fmt.Printf(userInputErrorMessage, userInputErr) - return userInputErr - } - prevAuthURL := confMap["Agent"]["SwiftAuthURL"][0] - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthURL", userResponse)) - whatFailed, accessErr := ValidateAccess() - if nil != accessErr { - switch whatFailed { - case typeAuthURL: - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthURL", prevAuthURL)) - fmt.Printf(failureMessageHeader) - fmt.Printf(authURLFailedMessage, accessErr) - fmt.Printf(failureMessageFooter) - case typeCredentails: - fmt.Printf(needMoreInfoMessageHeader) - fmt.Printf(credentialsFailedMessage, confMap["Agent"]["SwiftAuthUser"][0], confMap["Agent"]["SwiftAuthKey"][0], accessErr) - SaveCurrentConfig() - fmt.Printf(authURLSetMessage, confMap["Agent"]["SwiftAuthURL"][0]) - fmt.Printf(needMoreInfoMessageFooter) - case typeAccount: - fmt.Printf(needMoreInfoMessageHeader) - fmt.Printf(accountFailedMessage, confMap["Agent"]["SwiftAccountName"][0], confMap["Agent"]["SwiftAuthUser"][0], accessErr) - SaveCurrentConfig() - fmt.Printf(authURLSetMessage, confMap["Agent"]["SwiftAuthURL"][0]) - fmt.Println(changesSavedMessage) - fmt.Printf(needMoreInfoMessageFooter) - } - } else { - fmt.Printf(successMessageHeader) - fmt.Printf(accessCheckSucceededMessage) - fmt.Printf(authURLSetMessage, confMap["Agent"]["SwiftAuthURL"][0]) - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(successMessageFooter) - nextMenuText = mainMenuText - nextMenuOptions = mainMenuOptions - nextMenuOptionsTexts = mainMenuOptionsTexts - } - - case changeUsernameOptionText: - userResponse, userInputErr := getValueFromUser("Swift Username", "", confMap["Agent"]["SwiftAuthUser"][0]) - if nil != userInputErr { - fmt.Printf(userInputErrorMessage, userInputErr) - return userInputErr - } - prevAuthUser := confMap["Agent"]["SwiftAuthUser"][0] - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthUser", userResponse)) - whatFailed, accessErr := ValidateAccess() - if nil != accessErr { - switch whatFailed { - case typeAuthURL: - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthUser", prevAuthUser)) - fmt.Printf(failureMessageHeader) - fmt.Printf(authURLFailedMessage, accessErr) - fmt.Printf(failureMessageFooter) - case typeCredentails: - fmt.Printf(userSetMessage, confMap["Agent"]["SwiftAuthUser"][0]) - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(failureMessageHeader) - fmt.Printf(credentialsFailedMessage, userResponse, confMap["Agent"]["SwiftAuthKey"][0], accessErr) - fmt.Printf(failureMessageFooter) - case typeAccount: - fmt.Printf(needMoreInfoMessageHeader) - fmt.Printf(accountFailedMessage, confMap["Agent"]["SwiftAccountName"][0], confMap["Agent"]["SwiftAuthUser"][0], accessErr) - // MyConfig.SwiftAuthUser = userResponse - fmt.Printf(userSetMessage, confMap["Agent"]["SwiftAuthUser"][0]) - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(needMoreInfoMessageFooter) - } - } else { - fmt.Printf(successMessageHeader) - fmt.Printf(accessCheckSucceededMessage) - // MyConfig.SwiftAuthUser = userResponse - fmt.Printf(userSetMessage, confMap["Agent"]["SwiftAuthUser"][0]) - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(successMessageFooter) - nextMenuText = mainMenuText - nextMenuOptions = mainMenuOptions - nextMenuOptionsTexts = mainMenuOptionsTexts - } - - case changeKeyOptionText: - userResponse, userInputErr := getValueFromUser("Swift User Key", "", confMap["Agent"]["SwiftAuthKey"][0]) - if nil != userInputErr { - fmt.Printf(userInputErrorMessage, userInputErr) - return userInputErr - } - prevAuthKey := confMap["Agent"]["SwiftAuthKey"][0] - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthKey", userResponse)) - whatFailed, accessErr := ValidateAccess() - if nil != accessErr { - switch whatFailed { - case typeAuthURL: - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthKey", prevAuthKey)) - fmt.Printf(failureMessageHeader) - fmt.Printf(authURLFailedMessage, accessErr) - fmt.Printf(failureMessageFooter) - case typeCredentails: - // MyConfig.SwiftAuthKey = prevAuthKey - fmt.Printf(keySetMessage, confMap["Agent"]["SwiftAuthKey"][0]) - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(failureMessageHeader) - fmt.Printf(credentialsFailedMessage, confMap["Agent"]["SwiftAuthUser"][0], userResponse, accessErr) - fmt.Printf(failureMessageFooter) - case typeAccount: - fmt.Printf(needMoreInfoMessageHeader) - fmt.Printf(accountFailedMessage, confMap["Agent"]["SwiftAccountName"][0], confMap["Agent"]["SwiftAuthUser"][0], accessErr) - // MyConfig.SwiftAuthKey = userResponse - fmt.Printf(keySetMessage, confMap["Agent"]["SwiftAuthKey"][0]) - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(needMoreInfoMessageFooter) - } - } else { - fmt.Printf(successMessageHeader) - fmt.Printf(accessCheckSucceededMessage) - // MyConfig.SwiftAuthKey = userResponse - fmt.Printf(keySetMessage, confMap["Agent"]["SwiftAuthKey"][0]) - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(successMessageFooter) - nextMenuText = mainMenuText - nextMenuOptions = mainMenuOptions - nextMenuOptionsTexts = mainMenuOptionsTexts - } - - case changeAccountOptionText: - userResponse, userInputErr := getValueFromUser("Swift Account", "", confMap["Agent"]["SwiftAccountName"][0]) - if nil != userInputErr { - fmt.Printf(userInputErrorMessage, userInputErr) - return userInputErr - } - prevAccountName := confMap["Agent"]["SwiftAccountName"][0] - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAccountName", userResponse)) - whatFailed, accessErr := ValidateAccess() - if nil != accessErr { - fmt.Printf(failureMessageHeader) - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAccountName", prevAccountName)) - switch whatFailed { - case typeAuthURL: - fmt.Printf(authURLFailedMessage, accessErr) - case typeCredentails: - fmt.Printf(credentialsFailedMessage, confMap["Agent"]["SwiftAuthUser"][0], confMap["Agent"]["SwiftAuthKey"][0], accessErr) - case typeAccount: - fmt.Printf(accountFailedMessage, confMap["Agent"]["SwiftAccountName"][0], confMap["Agent"]["SwiftAuthUser"][0], accessErr) - } - fmt.Printf(failureMessageFooter) - } else { - fmt.Printf(successMessageHeader) - fmt.Printf(accessCheckSucceededMessage) - // MyConfig.SwiftAccountName = userResponse - fmt.Printf(accountSetMessage, confMap["Agent"]["SwiftAccountName"][0]) - - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(successMessageFooter) - nextMenuText = mainMenuText - nextMenuOptions = mainMenuOptions - nextMenuOptionsTexts = mainMenuOptionsTexts - } - - default: - fmt.Printf("got unknown response: %v\n", menuResponse) - } - } -} - -// FirstTimeRun is to be called for the first (initial) run of pfsagentConfig -// to create an initial real config -func FirstTimeRun() error { - loadError := LoadConfig("") - if nil != loadError { - fmt.Println("Failed loading config. Error:", loadError) - os.Exit(1) - } - - var oldAuthURL string - var oldAuthUser string - var oldAuthKey string - var oldAccount string - var oldMount string - var oldVolName string - var oldLogPath string - - if len(confMap["Agent"]["SwiftAuthURL"]) > 0 { - oldAuthURL = confMap["Agent"]["SwiftAuthURL"][0] - } - if len(confMap["Agent"]["SwiftAuthUser"]) > 0 { - oldAuthUser = confMap["Agent"]["SwiftAuthUser"][0] - } - if len(confMap["Agent"]["SwiftAuthKey"]) > 0 { - oldAuthKey = confMap["Agent"]["SwiftAuthKey"][0] - } - if len(confMap["Agent"]["SwiftAccountName"]) > 0 { - oldAccount = confMap["Agent"]["SwiftAccountName"][0] - } - if len(confMap["Agent"]["FUSEMountPointPath"]) > 0 { - oldMount = confMap["Agent"]["FUSEMountPointPath"][0] - } - if len(confMap["Agent"]["FUSEVolumeName"]) > 0 { - oldVolName = confMap["Agent"]["FUSEVolumeName"][0] - } - if len(confMap["Agent"]["LogFilePath"]) > 0 { - oldLogPath = confMap["Agent"]["LogFilePath"][0] - } - - fmt.Println(firstTimeCredentialsMenu) - mySwiftParams := new(SwiftParams) - - me := confMap["Agent"] - fmt.Println(me) - // validAuthURL := false - for { - userURLResponse, userURLInputErr := getValueFromUser("Swift Auth URL", authURLHint, "") - fmt.Println() - if nil != userURLInputErr { - return fmt.Errorf(userInputErrorMessage, userURLInputErr) - } - mySwiftParams.AuthURL = userURLResponse - userURLValidateErr := validateURL(mySwiftParams) - if nil != userURLValidateErr { - fmt.Printf(failureMessageHeader) - fmt.Printf("%v\n\n", userURLValidateErr) - fmt.Printf(failureMessageFooter) - } else { - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthURL", userURLResponse)) - break - } - } - - for { - userUserResponse, userUserInputErr := getValueFromUser("Swift Auth User", usernameHint, "") - fmt.Println() - if nil != userUserInputErr { - return fmt.Errorf(userInputErrorMessage, userUserInputErr) - } - - userKeyResponse, userKeyInputErr := getValueFromUser("Swift Auth Key", keyHint, "") - fmt.Println() - if nil != userKeyInputErr { - return fmt.Errorf(userInputErrorMessage, userKeyInputErr) - } - mySwiftParams.User = userUserResponse - mySwiftParams.Key = userKeyResponse - - token, credValidationErr := validateCredentails(mySwiftParams) - if nil != credValidationErr { - fmt.Printf(failureMessageHeader) - fmt.Printf("%v\n\n", credValidationErr) - fmt.Printf(failureMessageFooter) - } else { - confMap.UpdateFromStrings([]string{ - fmt.Sprintf("%v : %v", "Agent.SwiftAuthUser", userUserResponse), - fmt.Sprintf("%v : %v", "Agent.SwiftAuthKey", userKeyResponse), - }) - mySwiftParams.AuthToken = token - break - } - } - fmt.Printf("mySwiftParams: %v\n", mySwiftParams) - - validAccount := false - for !validAccount { - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAccountName", fmt.Sprintf("AUTH_%v", confMap["Agent"]["SwiftAuthUser"][0]))) - - userAccountResponse, userAccountInputErr := getValueFromUser("Swift Account", accountHint, confMap["Agent"]["SwiftAccountName"][0]) - fmt.Println() - if nil != userAccountInputErr { - return fmt.Errorf(userInputErrorMessage, userAccountInputErr) - } - if len(userAccountResponse) == 0 { - userAccountResponse = confMap["Agent"]["SwiftAccountName"][0] - } - - mySwiftParams.Account = userAccountResponse - accountValidationErr := validateAccount(mySwiftParams) - if nil != accountValidationErr { - fmt.Printf(failureMessageHeader) - fmt.Printf("%v\n\n", accountValidationErr) - fmt.Printf(failureMessageFooter) - mySwiftParams.StorageURL = "" - } else { - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAccountName", userAccountResponse)) - validAccount = true - } - } - - suggestedMount := fmt.Sprintf("%v/vol_%v", defaultMountPath, confMap["Agent"]["SwiftAccountName"][0]) - suggestedLogs := fmt.Sprintf("%v/log.%v", defaultLogPath, confMap["Agent"]["SwiftAccountName"][0]) - - confMap.UpdateFromStrings([]string{ - fmt.Sprintf("%v : %v", "Agent.LogFilePath", suggestedLogs), - fmt.Sprintf("%v : %v", "Agent.FUSEMountPointPath", suggestedMount), - fmt.Sprintf("%v : %v", "Agent.FUSEVolumeName", confMap["Agent"]["SwiftAccountName"][0]), - }) - - volNameResponse, volNameInputErr := getValueFromUser("Volume Name", volNameHint, confMap["Agent"]["SwiftAccountName"][0]) - fmt.Println() - if nil != volNameInputErr { - return fmt.Errorf(userInputErrorMessage, volNameInputErr) - } - if len(volNameResponse) > 0 { - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.FUSEVolumeName", volNameResponse)) - } - - mountPathResponse, mountPathInputErr := getValueFromUser("Mount Point", mountPointHint, confMap["Agent"]["FUSEMountPointPath"][0]) - fmt.Println() - if nil != mountPathInputErr { - return fmt.Errorf(userInputErrorMessage, mountPathInputErr) - } - if len(mountPathResponse) > 0 { - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.FUSEMountPointPath", mountPathResponse)) - } - - whatFailed, accessErr := ValidateAccess() - if nil != accessErr { - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthURL", oldAuthURL)) - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthUser", oldAuthUser)) - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAuthKey", oldAuthKey)) - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.SwiftAccountName", oldAccount)) - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.FUSEMountPointPath", oldMount)) - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.FUSEVolumeName", oldVolName)) - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.LogFilePath", oldLogPath)) - fmt.Printf(failureMessageHeader) - switch whatFailed { - case typeAuthURL: - fmt.Printf(authURLFailedMessage, accessErr) - case typeCredentails: - fmt.Printf(credentialsFailedMessage, confMap["Agent"]["SwiftAuthUser"][0], confMap["Agent"]["SwiftAuthKey"][0], accessErr) - fmt.Printf(authURLSetMessage, confMap["Agent"]["SwiftAuthURL"][0]) - case typeAccount: - fmt.Printf(accountFailedMessage, confMap["Agent"]["SwiftAccountName"][0], confMap["Agent"]["SwiftAuthUser"][0], accessErr) - fmt.Printf(authURLSetMessage, confMap["Agent"]["SwiftAuthURL"][0]) - fmt.Println(changesSavedMessage) - } - fmt.Printf(failureMessageFooter) - } else { - fmt.Printf(successMessageHeader) - fmt.Println(accessCheckSucceededMessage) - - if _, err := os.Stat(confMap["Agent"]["LogFilePath"][0]); os.IsNotExist(err) { - err = os.MkdirAll(confMap["Agent"]["LogFilePath"][0], 0755) - if err != nil { - fmt.Printf(failureMessageHeader) - panic(err) - } - } - - if _, err := os.Stat(confMap["Agent"]["FUSEMountPointPath"][0]); os.IsNotExist(err) { - err = os.MkdirAll(confMap["Agent"]["FUSEMountPointPath"][0], 0755) - if err != nil { - fmt.Printf(failureMessageHeader) - panic(err) - } - } - SaveCurrentConfig() - fmt.Println(changesSavedMessage) - fmt.Printf(successMessageFooter) - } - - return nil -} diff --git a/pfsagentConfig/strings.go b/pfsagentConfig/strings.go deleted file mode 100644 index c794502d..00000000 --- a/pfsagentConfig/strings.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package pfsagentConfig - -const ( - typeAuthURL int = iota - typeCredentails - typeAccount -) - -const ( - mainMenuText = ` -This is a utility to set the parameters for your PFSagentd. It can help you set/modify the access credentials or change any runtime parameters. -Please choose the right option from the menu below: -` - - firstTimeCredentialsMenu = `It seems like this is the first time, so let's run through everything:` - - credentialsMenuTexts = ` -This menu lets you set the access parameters. -` - - mainMenuOptionText = "Return To Main Menu" - quitMenuOptionText = "Quit" - backMenuOptionText = "Back" - - changeCredsOptionText = "Change Access Credentials" - changeOtherOptionText = "Change Other Parameters" - - changeAuthURLOptionText = "Change Auth URL" - changeUsernameOptionText = "Change Username" - changeKeyOptionText = "Change User Key" - changeAccountOptionText = "Change Account" - changeMountOptionText = "Change Mount Point" - successMessageHeader = "\n\n++++++++++++++++++++++++++ SUCCESS ++++++++++++++++++++++++++\n\n" - successMessageFooter = "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n\n" - failureMessageHeader = "\n\n************************** ERROR **************************\n\n" - failureMessageFooter = "\n*******************************************************************\n\n" - needMoreInfoMessageHeader = "\n\n////////////////////////// ATTENTION //////////////////////////\n\n" - needMoreInfoMessageFooter = "\n///////////////////////////////////////////////////////////////////\n\n" - authURLHint = "The Auth URL Is Used To Authenticate And Get An Auth Token" - usernameHint = "A Valid User For The SwiftStack Cluster" - keyHint = "A Valid Key (Password) For The SwiftStack Cluster User" - accountHint = "The Account To Log Into. Usually In This Form: AUTH_xxxx" - volNameHint = "The Name For This Volume .This Will Be Used To Identify This Configuration, As Well As The Mount Point And Log File" - mountPointHint = "The Mount Point For This Volume" -) - -var ( - mainMenuOptions = []string{"1", "2", "q"} - mainMenuOptionsTexts = map[string]string{ - mainMenuOptions[0]: changeCredsOptionText, - mainMenuOptions[1]: changeOtherOptionText, - mainMenuOptions[2]: quitMenuOptionText, - } - credentialsMenuOptions = []string{"1", "2", "3", "4", "b", "q"} - credentialsMenuOptionsTexts = map[string]string{ - credentialsMenuOptions[0]: changeAuthURLOptionText, - credentialsMenuOptions[1]: changeUsernameOptionText, - credentialsMenuOptions[2]: changeKeyOptionText, - credentialsMenuOptions[3]: changeAccountOptionText, - credentialsMenuOptions[4]: mainMenuOptionText, - credentialsMenuOptions[5]: quitMenuOptionText, - } -) - -const ( - userInputErrorMessage = "Error Reading Input From User\n%v" - authURLFailedMessage = "Auth URL Failed, So I Could Not Check User And Key. Please Verify Auth URL\n%v\n\n" - credentialsFailedMessage = "Auth URL Works, But I Got An Error Trying To Login With Credentails\nUser: %v\nKey: %v\n%v\n\n" - accountFailedMessage = "Auth URL And Credentials Works, But I Could Not Gain Access To Account %v. Please Verify The Account Exists And User %v Has The Correct Access Permissions\n%v\n\n" - changesSavedMessage = "Changes Saved To File" - accessCheckSucceededMessage = "All Access Checks Succeeded" - accountSetMessage = "Swift Account Set To %v\n" - userSetMessage = "Swift User Set To %v\n" - keySetMessage = "Swift User Key Set to %v\n" - authURLSetMessage = "Swift Auth URL Set To %v\n" -) diff --git a/pfsagentConfig/swiftStuff.go b/pfsagentConfig/swiftStuff.go deleted file mode 100644 index 12fb8121..00000000 --- a/pfsagentConfig/swiftStuff.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package pfsagentConfig - -import ( - "fmt" - "net/http" - "strings" -) - -// SwiftParams is a struct to hold all parameters needed to connect to a swift account -type SwiftParams struct { - // Account is the account name - Account string - // User is the username - User string - // Key is the username's key (password) - Key string - // AuthURL is the URL to get an auth token with - AuthURL string - // StorageURL is the URL for data access - StorageURL string - // AuthToken is the token used for data access - AuthToken string -} - -// ConstructStorageURL calculates the storage URL based on the auth URL and the account name -func ConstructStorageURL(mySwiftParams *SwiftParams) (storageURL string, err error) { - upTo := strings.Index(mySwiftParams.AuthURL, "/auth") - if upTo < 0 { - err = fmt.Errorf("Could Not Construct Storage URL From Auth URL %v. Missing '/auth/ in URL'", mySwiftParams.AuthURL) - return - } - storageURL = fmt.Sprintf("%v/v1/%v", mySwiftParams.AuthURL[:upTo], mySwiftParams.Account) - return -} - -// GetAuthToken retrieves the auth token for the user/key combination -func GetAuthToken(mySwiftParams *SwiftParams) (authToken string, err error) { - authClient := &http.Client{} - authTokenRequest, err := http.NewRequest("GET", mySwiftParams.AuthURL, nil) - if nil != err { - fmt.Printf("tokenErr:\n%v\n", err) - return - } - authTokenRequest.Header.Add("x-auth-user", mySwiftParams.User) - authTokenRequest.Header.Add("x-auth-key", mySwiftParams.Key) - // authTokenRequest.Header.Add("Content-Type", "application/json") - // fmt.Printf("auth request:\n%v\n", authTokenRequest) - authTokenResponse, authErr := authClient.Do(authTokenRequest) - // fmt.Printf("authTokenResponse: %v", authTokenResponse) - if nil != authErr { - err = fmt.Errorf("Error executing auth token request:\n\tError: %v", authErr) - return - } - defer authTokenResponse.Body.Close() - if authTokenResponse.StatusCode > 400 { - err = fmt.Errorf("Error authenticating for auth token. Check your user and key:\n\tStatus code: %v", authTokenResponse.Status) - return - } - if authTokenResponse.StatusCode > 299 { - err = fmt.Errorf("Error response status code from %v\n\tStatus code: %v", confMap["Agent"]["SwiftAuthURL"][0], authTokenResponse.StatusCode) - return - } - authToken = authTokenResponse.Header.Get("X-Auth-Token") - // authTokenBody, authTokenBodyReadErr := ioutil.ReadAll(authTokenResponse.Body) - if len(authToken) == 0 { - err = fmt.Errorf("Error finding auth token in response from %v\n\tResponse: %v", confMap["Agent"]["SwiftAuthURL"][0], authTokenResponse) - return - } - // fmt.Printf("auth token:\n%v\n", authToken) - return -} - -// GetAccountHeaders retrieves the headers for an account -func GetAccountHeaders(mySwiftParams *SwiftParams) (headers http.Header, err error) { - if len(mySwiftParams.StorageURL) == 0 { - tempStorageURL, urlErr := ConstructStorageURL(mySwiftParams) - if nil != urlErr { - err = urlErr - return - } - mySwiftParams.StorageURL = tempStorageURL - } - // fmt.Printf("mySwiftParams.StorageURL: %v\n", mySwiftParams.StorageURL) - - if len(mySwiftParams.AuthToken) == 0 { - tempAuthToken, authTokenErr := GetAuthToken(mySwiftParams) - if nil != authTokenErr { - err = authTokenErr - return - } - mySwiftParams.AuthToken = tempAuthToken - } - headClient := &http.Client{} - accountHeadRequest, accountHeadErr := http.NewRequest("HEAD", mySwiftParams.StorageURL, nil) - if nil != accountHeadErr { - err = fmt.Errorf("Error executing HEAD request:\n\tError: %v", accountHeadErr) - return - } - accountHeadRequest.Header.Add("x-auth-token", mySwiftParams.AuthToken) - accountHeadResponse, accountHeadErr := headClient.Do(accountHeadRequest) - - // fmt.Printf("accountHeadResponse: %v", accountHeadResponse) - if nil != accountHeadErr { - err = fmt.Errorf("Error executing HEAD request:\n\tError: %v", accountHeadErr) - return - } - if accountHeadResponse.StatusCode >= 400 { - err = fmt.Errorf("Bad Status Code Getting Account Headers:\n\tStatus Code: %v", accountHeadResponse.StatusCode) - return - } - // fmt.Printf("accountHeadResponse: %v", accountHeadResponse) - headers = accountHeadResponse.Header - return -} diff --git a/pfsagentConfig/util_test.go b/pfsagentConfig/util_test.go deleted file mode 100644 index 08561473..00000000 --- a/pfsagentConfig/util_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package pfsagentConfig - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfsagentConfig/utils.go b/pfsagentConfig/utils.go deleted file mode 100644 index 3fe8ffa8..00000000 --- a/pfsagentConfig/utils.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package pfsagentConfig - -import ( - "bufio" - "errors" - "fmt" - "log" - "os" - "os/user" - - "github.com/NVIDIA/proxyfs/conf" -) - -const ( - defaultConfigPath string = "/etc/pfsagent" - defaultLogPath string = "/var/log/pfsagent" - configTmplFile string = "pfsagent.tmpl" -) - -var ( - confMap conf.ConfMap - defaultMountPath = func() string { - usr, err := user.Current() - if err != nil { - log.Fatal(err) - } - return usr.HomeDir - }() + "/pfsagentMount" - ConfigPath = defaultConfigPath -) - -func cloneFromTemplate() (configName string, err error) { - tmplPath := fmt.Sprintf("%v/%v", ConfigPath, configTmplFile) - // fmt.Printf("tmplPath: %v\n", tmplPath) - if _, err = os.Stat(tmplPath); err != nil { - fmt.Println("Template file not found at", tmplPath, err) - return - } - confMap, err = conf.MakeConfMapFromFile(tmplPath) - if err != nil { - log.Println("Failed loading template file", tmplPath, err) - return - } - return -} - -func renameConfig(newName string) (err error) { - if len(newName) == 0 { - err = fmt.Errorf("no new name provided") - return - } - oldName := confMap["Agent"]["FUSEVolumeName"][0] - if newName == oldName { - return - } - oldFilePath := fmt.Sprintf("%v/%v", ConfigPath, oldName) - if _, err = os.Stat(oldFilePath); err != nil { - log.Printf("Config file not found at %v\n%v\n", oldFilePath, err) - return - } - newFilePath := fmt.Sprintf("%v/%v", ConfigPath, newName) - if _, err = os.Stat(newFilePath); err == nil { - log.Printf("%v already has a file: %v\n%v\n", newName, newFilePath, err) - return - } - confMap.UpdateFromString(fmt.Sprintf("%v : %v", "Agent.FUSEVolumeName", newName)) - err = SaveCurrentConfig() - if err == nil { - os.Remove(oldFilePath) - } - return -} - -func LoadConfig(configName string) (err error) { - if len(configName) == 0 { - log.Printf("Cloning config from %v\n", configTmplFile) - configName, err = cloneFromTemplate() - return - } - - log.Printf("Initializing config from %v/%v\n", ConfigPath, configName) - configFilePath := fmt.Sprintf("%v/%v", ConfigPath, configName) - if _, err = os.Stat(configFilePath); err != nil { - log.Println("Config file not found at", configFilePath, err) - return - } - confMap, err = conf.MakeConfMapFromFile(configFilePath) - // iniContent, loadErr := ini.Load(ConfigPath) - if err != nil { - log.Println("Failed loading config file", ConfigPath, err) - return - } - return -} - -func SaveCurrentConfig() (err error) { - if confMap == nil { - log.Println("Config is not initialized in the utility. did loadConfig() run?") - err = errors.New("no config found") - return - } - configName := confMap["Agent"]["FUSEVolumeName"][0] - configFilePath := fmt.Sprintf("%v/%v.conf", ConfigPath, configName) - fmt.Printf("saving config to %v\n", configFilePath) - confMap.DumpConfMapToFile(configFilePath, os.ModePerm) - - return nil -} - -func getUserInput() (response string, err error) { - scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - if err = scanner.Err(); err != nil { - log.Println("Error reading standard input:", err) - return - } - response = scanner.Text() - return - } - err = fmt.Errorf("Error retrieving user input") - return -} - -func getValueFromUser(title string, text string, currentValue string) (response string, err error) { - fmt.Printf("** Changing %v **", title) - if len(text) > 0 { - fmt.Printf("\n\t%v", text) - } - fmt.Printf("\n\nCurrent Value: %v\nNew Value: ", currentValue) - response, err = getUserInput() - if err != nil { - log.Println("Error retrieving user input", err) - } - return -} diff --git a/pfsagentConfig/validators.go b/pfsagentConfig/validators.go deleted file mode 100644 index 4f7cb6ea..00000000 --- a/pfsagentConfig/validators.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package pfsagentConfig - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" -) - -// ValidateAuthURL checks that an AUTH URL pointing to a SwiftStack cluster -// func ValidateAuthURL(urlToBeTested string) error { -// return nil -// } - -// ValidateAccess runs all -func ValidateAccess() (errorType int, err error) { - mySwiftParams := new(SwiftParams) - mySwiftParams.User = confMap["Agent"]["SwiftAuthUser"][0] - mySwiftParams.Key = confMap["Agent"]["SwiftAuthKey"][0] - mySwiftParams.Account = confMap["Agent"]["SwiftAccountName"][0] - mySwiftParams.AuthURL = confMap["Agent"]["SwiftAuthURL"][0] - - err = validateURL(mySwiftParams) - if nil != err { - errorType = typeAuthURL - return - } - _, err = validateCredentails(mySwiftParams) - if nil != err { - errorType = typeCredentails - return - } - err = validateAccount(mySwiftParams) - if nil != err { - errorType = typeAccount - return - } - return -1, nil -} - -func validateURL(mySwiftParams *SwiftParams) error { - parts := strings.Split(mySwiftParams.AuthURL, "/") - if len(parts) < 5 { - return fmt.Errorf(` -Auth URL should be of the form: -:///auth/v -where - - protocol is either 'http' or 'https' - - url is the public address of the cluster (such as 'swiftcluster.com' or '192.168.2.100') - - API version is usually in the format of 1.0, 2.0 etc. -`) - } - protocol := parts[0] - if strings.Index(protocol, "http") != 0 && strings.Index(protocol, "https") != 0 { - return fmt.Errorf("Auth URL Should Start With http:// Or https://\n\tYour Protocol is %v", protocol) - } - if parts[3] != "auth" { - return fmt.Errorf("A Valid Auth URL Should Have The Word 'auth' After The Address.\n\tExample: https://swiftcluster.com/auth/v1.0") - } - if strings.Index(parts[4], "v") != 0 || len(parts[4]) != 4 { - return fmt.Errorf("A Valid Auth URL Should Have The Protocol Version After The Term 'auth'.\n\tExample: https://swiftcluster.com/auth/v1.0") - } - address := strings.Split(parts[2], "/")[0] - url := fmt.Sprintf("%v//%v", protocol, address) - response, generalAccessErr := http.Get(url) - if nil != generalAccessErr { - return fmt.Errorf("Error Connecting To The Cluster's URL, Which Probably Means The URL Has Errors.\n\tDetected Cluster URL: %v\n\tError: %v", url, generalAccessErr) - } - infoURL := fmt.Sprintf("%v/info", url) - response, httpErr := http.Get(infoURL) - if nil != httpErr { - return fmt.Errorf("Error Connecting To The Cluster's 'info' URL (%v), Though The URL Itself (%v) Is Reponding.\n\tPlease Check Your URL\n\tError: %v", infoURL, url, httpErr) - } - defer response.Body.Close() - infoBody, bodyReadErr := ioutil.ReadAll(response.Body) - if nil != bodyReadErr { - return fmt.Errorf("Error Reading Response From %v\n\tError: %v", infoURL, bodyReadErr) - } - var infoInterface interface{} - jsonErr := json.Unmarshal(infoBody, &infoInterface) - if nil != jsonErr { - return fmt.Errorf("Error Parsing Info From %v.\nBody Text: %v\n\tError: %v", infoURL, string(infoBody), jsonErr) - } - validJSON := infoInterface.(map[string]interface{}) - if nil == validJSON["swift"] { - return fmt.Errorf("Error Finding The Term 'swift' In Parsed JSON\n\tError: %v", validJSON) - } - - authClient := &http.Client{} - authTokenRequest, authTokenErr := http.NewRequest("GET", mySwiftParams.AuthURL, nil) - if nil != authTokenErr { - fmt.Printf("Error Creting Auth Request:\n%v\n", authTokenErr) - } - authURLResponse, authURLErr := authClient.Do(authTokenRequest) - if nil != authURLErr { - return fmt.Errorf("Error Executing Auth URL Request:\n\tURL: %v\n\tError: %v", mySwiftParams.AuthURL, authURLErr) - } - if authURLResponse.StatusCode != 401 { - return fmt.Errorf("Error Response From Auth URL. Get On Auth URL With No User/Key Should Result In Status '401 Unauthorize'\n\tStatus code: %v", authURLResponse.Status) - } - - return nil -} - -func validateCredentails(mySwiftParams *SwiftParams) (authToken string, err error) { - authToken, err = GetAuthToken(mySwiftParams) - if nil != err { - return - } - mySwiftParams.AuthToken = authToken - return -} - -func validateAccount(mySwiftParams *SwiftParams) error { - headers, headErr := GetAccountHeaders(mySwiftParams) - fmt.Printf("\n\nvalidation succeeded? %v\n\n", nil == headErr) - if nil != headErr { - return headErr - } - if len(headers) == 0 { - return fmt.Errorf("Though The Request Succeeded There Were No Headers Returned") - } - return nil -} diff --git a/pfsagentd/.gitignore b/pfsagentd/.gitignore deleted file mode 100644 index d287cd3e..00000000 --- a/pfsagentd/.gitignore +++ /dev/null @@ -1 +0,0 @@ -debug diff --git a/pfsagentd/LeaseManagement.uml b/pfsagentd/LeaseManagement.uml deleted file mode 100644 index fc11ebba..00000000 --- a/pfsagentd/LeaseManagement.uml +++ /dev/null @@ -1,57 +0,0 @@ -@startuml - -scale 2400 height -skinparam StateFontSize 32 -skinparam StateAttributeFontSize 28 -skinparam ArrowFontSize 28 - -state None : No leases granted -state SharedLeaseGrantedRecently : One or more SharedLeases\ngranted not long ago -state SharedLeaseGrantedLongAgo : One or more SharedLeases\ngranted for some time -state ExclusiveLeaseGrantedRecently : One ExclusiveLease\ngranted not long ago -state ExclusiveLeaseGrantedLongAgo : One ExclusiveLease\ngranted for some time -state SharedLeaseReleasing : SharedLease holders asked to release\nto enable granting ExclusiveLease -state SharedLeaseExpired : SharedLease holder failed to release -state ExclusiveLeaseDemoting : ExclusiveLease holder asked to demote\nto enable granting SharedLeases -state ExclusiveLeaseReleasing : ExclusiveLease holder asked to release\nto enable granting ExclusiveLease -state ExclusiveLeaseExpired : ExclusiveLease holder failed to demote or release - -[*] --> None - -None --> SharedLeaseGrantedRecently : "1st SharedLease\nrequest" -None --> ExclusiveLeaseGrantedRecently : "ExclusiveLease\nrequest" - -SharedLeaseGrantedRecently --> SharedLeaseGrantedRecently : Subsequent SharedLease request -SharedLeaseGrantedRecently --> SharedLeaseGrantedLongAgo : "Minimum SharedLease\nhold time exceeded" -SharedLeaseGrantedRecently --> ExclusiveLeaseGrantedRecently : "Last SharedLock release\n(ExclusiveLease pending)" -SharedLeaseGrantedRecently --> None : "Last SharedLock release\n(nothing pending)" - -SharedLeaseGrantedLongAgo --> SharedLeaseGrantedRecently : Subsequent SharedLease request -SharedLeaseGrantedLongAgo --> SharedLeaseReleasing : ExclusiveLease request pending -SharedLeaseGrantedLongAgo --> None : Last SharedLock release - -SharedLeaseReleasing --> SharedLeaseReleasing : "Non-last SharedLease\nrelease" -SharedLeaseReleasing --> ExclusiveLeaseGrantedRecently : "Last SharedLease\nrelease" -SharedLeaseReleasing --> SharedLeaseExpired : "Maximum SharedLease\nrelease time exceeded" - -SharedLeaseExpired --> ExclusiveLeaseGrantedRecently : \"Expired SharedLeases\nimplicitly released" - -ExclusiveLeaseGrantedRecently --> ExclusiveLeaseGrantedLongAgo : "Minimum ExclusiveLease\nhold time exceeded" -ExclusiveLeaseGrantedRecently --> SharedLeaseGrantedRecently : "ExclusiveLease release\n(SharedLease pending)" -ExclusiveLeaseGrantedRecently --> ExclusiveLeaseGrantedRecently : "ExclusiveLease release\n(ExclusiveLease pending)" -ExclusiveLeaseGrantedRecently --> None : "ExclusiveLease release\n(nothing pending)" - -ExclusiveLeaseGrantedLongAgo --> ExclusiveLeaseDemoting : SharedLease request pending -ExclusiveLeaseGrantedLongAgo --> ExclusiveLeaseReleasing : ExclusiveLease request pending -ExclusiveLeaseGrantedLongAgo --> None: "ExclusiveLease\nrelease" - -ExclusiveLeaseDemoting --> SharedLeaseGrantedRecently : "ExclusiveLease\ndemote or release" -ExclusiveLeaseDemoting --> ExclusiveLeaseExpired : "Maximum ExclusiveLease\ndemote time exceeded" - -ExclusiveLeaseReleasing --> ExclusiveLeaseGrantedRecently : "ExclusiveLease\nrelease" -ExclusiveLeaseReleasing --> ExclusiveLeaseExpired : "Maximum ExclusiveLease\nrelease time exceeded" - -ExclusiveLeaseExpired --> SharedLeaseGrantedRecently : "Expired ExclusiveLease\nimplicitly released\n(SharedLease pending)" -ExclusiveLeaseExpired --> ExclusiveLeaseGrantedRecently : "Expired ExclusiveLease\nimplicitly released\n(ExclusiveLease pending)" - -@enduml diff --git a/pfsagentd/LeaseRequestClient.uml b/pfsagentd/LeaseRequestClient.uml deleted file mode 100644 index 9e254ab7..00000000 --- a/pfsagentd/LeaseRequestClient.uml +++ /dev/null @@ -1,44 +0,0 @@ -@startuml - -scale 800 width - -state None : No lease held -state SharedLeaseRequested : Awaiting ProxyFS to grant either\nSharedLease or ExclusiveLease -state SharedLeaseGranted : Able to grant multiple SharedLocks -state SharedLeasePromoting : Awaiting ProxyFS to grant ExclusiveLease\nor be forced to release SharedLease -state PreparingToReleaseSharedLease : Draining SharedLocks and invalidating\nCached Attributes and ExtentMap -state SharedLeaseReleasing : Awaiting ProxyFS to acknowledge\nSharedLease has been released -state ExclusiveLeaseRequested : Awaiting ProxyFS to\ngrant ExclusiveLease -state ExclusiveLeaseGranted : Able to grant multiple\nSharedLocks or one ExclusiveLock -state PreparingToReleaseExclusiveLease : Draining Locks, invalidating Cached\nAttributes and ExtentMap, and\nflushing in-flight LogSegment PUTs -state ExclusiveLeaseReleasing : Awaiting ProxyFS to acknowledge\nExclusiveLease has been released -state PreparingToDemoteExclusiveLease : Flushing in-flight LogSegment PUTs -state ExclusiveLeaseDemoting : Awaiting ProxyFS to acknowlege ExclusiveLease\nhas been demoted to a SharedLease - -[*] --> None - -None --> SharedLeaseRequested -None --> ExclusiveLeaseRequested - -SharedLeaseRequested --> SharedLeaseGranted -SharedLeaseRequested --> ExclusiveLeaseGranted - -SharedLeaseGranted --> PreparingToReleaseSharedLease -PreparingToReleaseSharedLease --> SharedLeaseReleasing -SharedLeaseReleasing --> None - -SharedLeaseGranted --> SharedLeasePromoting -SharedLeasePromoting --> PreparingToReleaseSharedLease -SharedLeasePromoting --> ExclusiveLeaseGranted - -ExclusiveLeaseRequested --> ExclusiveLeaseGranted - -ExclusiveLeaseGranted --> PreparingToReleaseExclusiveLease -PreparingToReleaseExclusiveLease --> ExclusiveLeaseReleasing -ExclusiveLeaseReleasing --> None - -ExclusiveLeaseGranted --> PreparingToDemoteExclusiveLease -PreparingToDemoteExclusiveLease --> ExclusiveLeaseDemoting -ExclusiveLeaseDemoting --> SharedLeaseGranted - -@enduml diff --git a/pfsagentd/LeaseRequestServer.uml b/pfsagentd/LeaseRequestServer.uml deleted file mode 100644 index 706c2ded..00000000 --- a/pfsagentd/LeaseRequestServer.uml +++ /dev/null @@ -1,47 +0,0 @@ -@startuml - -scale 800 width - -state None : No lease held -state SharedLeaseRequested : Blocked awaiting ExclusiveLease\ndemotion or release -state SharedLeaseGranted -state SharedLeaseReleasing : PFSAgent has been asked to release -state SharedLeasePromoting : SharedLease held while\nawaiting others to release -state ExclusiveLeaseRequested : Blocked awaiting all locks release -state ExclusiveLeaseGranted -state ExclusiveLeaseDemoting : PFSAgent has been asked to demote -state ExclusiveLeaseReleasing : PFSAgent has been asked to release - -[*] --> None - -None --> SharedLeaseGranted -None --> SharedLeaseRequested -None --> ExclusiveLeaseGranted -None --> ExclusiveLeaseRequested - -SharedLeaseGranted --> None -SharedLeaseGranted --> SharedLeaseReleasing -SharedLeaseGranted --> SharedLeasePromoting -SharedLeaseGranted --> ExclusiveLeaseGranted - -SharedLeaseReleasing --> None - -SharedLeaseRequested --> SharedLeaseGranted -SharedLeaseRequested --> ExclusiveLeaseGranted - -SharedLeasePromoting --> ExclusiveLeaseGranted -SharedLeasePromoting --> SharedLeaseReleasing - -ExclusiveLeaseGranted --> None -ExclusiveLeaseGranted --> SharedLeaseGranted -ExclusiveLeaseGranted --> ExclusiveLeaseDemoting -ExclusiveLeaseGranted --> ExclusiveLeaseReleasing - -ExclusiveLeaseRequested --> ExclusiveLeaseGranted - -ExclusiveLeaseDemoting --> None -ExclusiveLeaseDemoting --> SharedLeaseGranted - -ExclusiveLeaseReleasing --> None - -@enduml diff --git a/pfsagentd/LockManagement.uml b/pfsagentd/LockManagement.uml deleted file mode 100644 index a02e5d55..00000000 --- a/pfsagentd/LockManagement.uml +++ /dev/null @@ -1,29 +0,0 @@ -@startuml - -scale 800 width - -state None : No lease held -state SharedLeaseGranted : Able to grant multiple SharedLocks -state SharedLeaseRequested : Awaiting ProxyFS to grant either\nSharedLease or ExclusiveLease -state SharedLeasePromoting : Awaiting ProxyFS to grant ExclusiveLease\nor be forced to release SharedLease -state ExclusiveLeaseGranted : Able to grant multiple\nSharedLocks or one ExclusiveLock -state ExclusiveLeaseRequested : Awaiting ProxyFS to\ngrant ExclusiveLease - -[*] --> None - -None --> SharedLeaseRequested : SharedLockRequest -None --> ExclusiveLeaseRequested : ExclusiveLockRequest - -SharedLeaseRequested --> SharedLeaseGranted -SharedLeaseRequested --> ExclusiveLeaseGranted - -SharedLeaseGranted --> SharedLeaseGranted : SharedLockRequest -SharedLeaseGranted --> SharedLeasePromoting : ExclusiveLockRequest - -SharedLeasePromoting --> ExclusiveLeaseGranted -ExclusiveLeaseRequested --> ExclusiveLeaseGranted - -ExclusiveLeaseGranted --> ExclusiveLeaseGranted : SharedLockRequest -ExclusiveLeaseGranted --> ExclusiveLeaseGranted : ExclusiveLockRequest - -@enduml diff --git a/pfsagentd/LockManagementWithExclusiveLease.uml b/pfsagentd/LockManagementWithExclusiveLease.uml deleted file mode 100644 index b5a88c57..00000000 --- a/pfsagentd/LockManagementWithExclusiveLease.uml +++ /dev/null @@ -1,36 +0,0 @@ -@startuml - -scale 800 width - -state None : No locks held -state SharedLockGranted: One or more SharedLocks held -state ExclusiveLockGranted : One ExclusiveLock held -state SharedLockGrantedExclusiveLockRequested : While SharedLock held, one\nor more ExclusiveLocks requested -state ExclusiveLockGrantedLockRequested : While ExclusiveLock held,\none or more Locks requested - -[*] --> None - -None --> SharedLockGranted : First SharedLock request -None --> ExclusiveLockGranted : ExclusiveLock request - -SharedLockGranted --> SharedLockGranted : Subsequent SharedLock request -SharedLockGranted --> SharedLockGranted : Non-last SharedLock release -SharedLockGranted --> None : Last SharedLock release - -SharedLockGranted --> SharedLockGrantedExclusiveLockRequested : ExclusiveLock request - -SharedLockGrantedExclusiveLockRequested --> SharedLockGrantedExclusiveLockRequested : Lock request -SharedLockGrantedExclusiveLockRequested --> SharedLockGrantedExclusiveLockRequested : Non-last SharedLock release -SharedLockGrantedExclusiveLockRequested --> ExclusiveLockGranted : Last SharedLock release &\nonly ExclusiveLock request pending -SharedLockGrantedExclusiveLockRequested --> ExclusiveLockGrantedLockRequested : Last SharedLock release &\nmultiple Lock requests pending - -ExclusiveLockGranted --> None : ExclusiveLock release - -ExclusiveLockGranted --> ExclusiveLockGrantedLockRequested : Lock request - -ExclusiveLockGrantedLockRequested --> SharedLockGranted : ExclusiveLock release & only\nSharedLock requests pending -ExclusiveLockGrantedLockRequested --> ExclusiveLockGranted : ExclusiveLock release & only\nsingle ExclusiveLock request pending -ExclusiveLockGrantedLockRequested --> SharedLockGrantedExclusiveLockRequested : ExclusiveLock release & one or\nmore SharedLock before ExclusiveLock pending -ExclusiveLockGrantedLockRequested --> ExclusiveLockGrantedLockRequested : ExclusiveLock release & more than just\nsingle ExclusiveLock request pending - -@enduml diff --git a/pfsagentd/LockManagementWithSharedLease.uml b/pfsagentd/LockManagementWithSharedLease.uml deleted file mode 100644 index f83cf624..00000000 --- a/pfsagentd/LockManagementWithSharedLease.uml +++ /dev/null @@ -1,20 +0,0 @@ -@startuml - -scale 800 width - -state None : No locks held -state SharedLockGranted: One or more SharedLocks held -state ExclusiveLockRequested : Transition to SharedLeasePromoting - -[*] --> None - -None --> SharedLockGranted : First SharedLock request -None --> ExclusiveLockRequested : ExclusiveLock request - -SharedLockGranted --> SharedLockGranted : Subsequent SharedLock request -SharedLockGranted --> SharedLockGranted : Non-last SharedLock release -SharedLockGranted --> None : Last SharedLock release - -SharedLockGranted --> ExclusiveLockRequested : ExclusiveLock request - -@enduml diff --git a/pfsagentd/LockRequest.uml b/pfsagentd/LockRequest.uml deleted file mode 100644 index 804b587f..00000000 --- a/pfsagentd/LockRequest.uml +++ /dev/null @@ -1,15 +0,0 @@ -@startuml - -scale 800 width - -[*] --> None - -None --> SharedLockRequested -SharedLockRequested -> SharedLockGranted -SharedLockGranted -> None - -None --> ExclusiveLockRequested -ExclusiveLockRequested -> ExclusiveLockGranted -ExclusiveLockGranted -> None - -@enduml diff --git a/pfsagentd/Makefile b/pfsagentd/Makefile deleted file mode 100644 index d49ed772..00000000 --- a/pfsagentd/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfsagentd - -include ../GoMakefile diff --git a/pfsagentd/README.md b/pfsagentd/README.md deleted file mode 100644 index 85429e92..00000000 --- a/pfsagentd/README.md +++ /dev/null @@ -1,191 +0,0 @@ -# PFSAgent - -FUSE Driver for presenting ProxyFS Volumes as locally mounted file systems - -## Synopsis - -PFSAgent is a program that remotely accesses a ProxyFS Volume through -a new `PROXYFS` HTTP Method targetted at any Swift Proxy Server in a -Swift cluster that has contains the `pfs_middleware` filter. All metadata -operations tunnel through `pfs_middleware` on their way to the `ProxyFS` -instance currently managing the specified Volume (Swift Account). To -allow file reads and writes to scale out, however, PFSAgent employs -a `bypass_mode` instructing `pfs_middleware` to pass through an Object -GET or PUT HTTP Method rather than redirect it over to `ProxyFS`. - -## Setup - -There are a few distinct entities to be configured. - -### OpenStack Swift Proxy - -Each Swift Proxy Server configuration file (`proxy-server.conf`) is assumed -to already contain a `[filter:pfs]` section pointing the `pfs_middleware` at -one or more ProxyFS instances. This is what enables Swift API and S3 API -"BiModal" access to ProxyFS Volumes. A new Key:Value that is one of: -* bypass_mode = off -* bypass_mode = read-only -* bypass_mode = read-write -The Value `off` is assumed if the `bypass_mode` Key is not specified. - -### PFSAgent Daemon (pfsagentd) - -The program is supplied a configuration file in .INI format. Here is an example: -``` -[Agent] -FUSEVolumeName: CommonVolume -FUSEMountPointPath: AgentMountPoint -FUSEUnMountRetryDelay: 100ms -FUSEUnMountRetryCap: 100 -PlugInPath: pfsagentd-swift-auth-plugin -PlugInEnvName: SwiftAuthBlob -PlugInEnvValue: {"AuthURL":"http://localhost:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"} -SwiftTimeout: 10m -SwiftRetryLimit: 10 -SwiftRetryDelay: 1s -SwiftRetryDelayVariance: 25 -SwiftRetryExpBackoff: 1.4 -SwiftConnectionPoolSize: 200 -FetchExtentsFromFileOffset: 32 -FetchExtentsBeforeFileOffset: 0 -ReadCacheLineSize: 1048576 -ReadCacheLineCount: 1000 -LeaseRetryLimit: 10 -LeaseRetryDelay: 1s -LeaseRetryDelayVariance: 25 -LeaseRetryExpBackoff: 1.4 -SharedLeaseLimit: 1000 -ExclusiveLeaseLimit: 100 -ExtentMapEntryLimit: 1048576 -DirtyLogSegmentLimit: 50 -DirtyFileLimit: 50 # TODO - obsolete this -MaxFlushSize: 10485760 -MaxFlushTime: 200ms -LogFilePath: /var/log/pfsagentd.log -LogToConsole: true -TraceEnabled: false -HTTPServerIPAddr: 0.0.0.0 -HTTPServerTCPPort: 9090 -ReadDirPlusEnabled: false -XAttrEnabled: false -EntryDuration: 0s -AttrDuration: 0s -ReaddirMaxEntries: 1024 -FUSEMaxBackground: 100 -FUSECongestionThreshhold: 0 -FUSEMaxWrite: 131072 -FUSEAllowOther: true -RetryRPCPublicIPAddr: 127.0.0.1 -RetryRPCPort: 32356 -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCACertFilePath: # Defaults to /dev/null -``` - -In the above example, some important fields are as follows: -* FUSEVolumeName should be set to the ProxyFS Volume being mounted -* FUSEMountPointPath should be set to where the FUSE presentation of the Volume should appear (must pre-exist) -* PlugInPath should be set to point to the desired Swift Authorization PlugIn -* PlugInEnvName should be set to the name of the ENV variable used to pass Swift Auth secrets blob to the plug-in -* PlugInEnvValue should be set to the value of the Swift Auth secrets blob if PFSAgent should set it -* HTTPServerIPAddr should be set to the IP Address where PFSAgent should present its embedded HTTP Server -* HTTPServerTCPPort should be set to the TCP Port upon which the PFSAgent should present its embedded HTTP Server - -The balance of the settings are more related to tuning choices. Among those, the most pertinent are: -* ReadCacheLineSize specifies how much of a Swift Object is read when a read cache miss occurs -* ReadCacheLineCount specifies how many such read cache lines will be used -* MaxFlushSize specifies how frequently in terms of byte count writes are sent to new Swift Objects -* MaxFlushTime specifies how frequently in terms of time writes are sent to new Swift Objects - -Note the use of `\u002C` in the example PlugInEnvValue line above. This avoids the .INI -parser from interpreting the commas as value separators and incorrectly assuming there -are multiple values being specified. Similarly, space would be interpreted as a list -element separator so `\u0020` should be used instead. - -Each mounted ProxyFS Volume requires an instance of PFSAgent (`pfsagentd`) to run. -Hence, each ProxyFS Volume must be described by a unique configuration file as described above. - -It is convenient to employ `systemd` to launch each PFSAgent instance via unique service files. -An example service file that can be used to allow launching multiple such PFSAgent instances -(by specifying the configuration file name in the `systemd` invocation) is: -``` -[Unit] -Description=ProxyFS Agent %i -Wants=network-online.target -After=network-online.target -ConditionFileNotEmpty=/opt/ss/etc/pfsagent/%i.conf - -[Service] -ExecStart=/opt/ss/bin/pfsagentd /opt/ss/etc/pfsagent/%i.conf - -Type=simple - -# Restart the ssnoded daemon after a 2 seconds delay, in case it crashes. -Restart=always -RestartSec=2 -StartLimitInterval=11s - -[Install] -WantedBy=network-online.target -``` - -### Authentication Plug-In - -While PFSAgent targets OpenStack Swift clusters that have provisioned -ProxyFS and placed the pfs_middleware in the Swift Proxy pipeline, it -must use the normal Swift Proxy to gain access to ProxyFS LogSegments -that host FileInode data. Further, via the pfs_middleware in the Swift -Proxy pipeline, at least the Mount JSON RPC will travel through the -normal Swift Proxy to get to ProxyFS. To perform either these GETs, -PUTs, or Mount JSON RPC, the request must contain a valid X-Auth-Token. -Rather than assume normal Swift Authentifiction is used to compute -this X-Auth-Token, PFSAgent implements a plug-in mechanism whereby an -external process, the plug-in, performs the actual authentication step -and returns the X-Auth-Token to be provided as proof in each subsequent -Swift API request. Such tokens may expire, resulting in a `401 Unauthorized` -response code. Thus, this plug-in must also support token renewal or -replacement. In cases where a full re-authorization is required, the -plug-in will already have the long-lived credentials with which to repeat -the authentication as performed originally. - -#### Authentication-specific Setup - -The configuration supplied to PFSAgent (`pfsagentd`) includes, among -other keys, the following: - -``` -[Agent] -... -PlugInPath: # The path to the plug-in -PlugInEnvName: # Specifies the name of an ENV variable the plug-in should load -PlugInEnvValue: # If supplied, PFSAgent will set the ENV variable to this value -... -``` - -#### Plug-In Operation - -PFSAgent will, at start-up, launch the program indicated by [Agent]PlugInPath, -sending the value of [Agent]PlugInEnvName as Arg1. If [Agent]PlugInEnvValue is -specified, it will also ensure the ENV variable named will be set to this -value in the plug-in's process. - -At that point, and at any time subsequently receiving a byte via os.Stdin, the -plug-in will perform the authentication (or renewal) step. The result will -be used to generate two pieces of information: - -* X-Auth-Token - to be used by PFSAgent in all subsequent Swift API calls (GETs, PUTs, and Mouunt JSON RPC requests) -* StorageURL - to form the first portion of any such Swift API path - -The response (either at start-up or following reception of a -byte via os.Stdin) is sent to os.Stdout as a UTF-8 encoded JSON object: - -``` -{ - "AuthToken" : "" - "StorageURL" : "", -} -``` - -At PFSAgent termination, the plug-in should see os.Stdin close. -This should trigger the plug-in to also exit (perhaps after -cleaning up any not-to-be-persisted details of its execution). diff --git a/pfsagentd/container/build/Dockerfile b/pfsagentd/container/build/Dockerfile deleted file mode 100644 index 5319f90d..00000000 --- a/pfsagentd/container/build/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -FROM centos - -ARG GolangVersion=1.15.5 -ARG ProxyFS_Version=stable - -ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" -ENV GolangURL "https://golang.org/dl/${GolangBasename}" - -RUN yum install -y fuse gcc git make python2 tar wget -RUN ln -s /usr/bin/python2 /usr/bin/python - -WORKDIR /opt/PFSAgent - -RUN wget -nv $GolangURL -RUN tar -C /usr/local -xzf $GolangBasename - -ENV GOPATH /opt/PFSAgent/GOPATH -ENV PATH $PATH:/usr/local/go/bin:$GOPATH/bin - -RUN mkdir -p $GOPATH/src/github.com/NVIDIA -WORKDIR $GOPATH/src/github.com/NVIDIA -RUN git clone https://github.com/NVIDIA/proxyfs.git -WORKDIR $GOPATH/src/github.com/NVIDIA/proxyfs -RUN git checkout $ProxyFS_Version - -RUN make version pfsagent - -WORKDIR /opt/PFSAgent - -# To build this image: -# -# docker build \ -# [--build-arg GolangVersion=] \ -# [--build-arg ProxyFS_Version=] \ -# [-t :] . diff --git a/pfsagentd/container/insert/.gitignore b/pfsagentd/container/insert/.gitignore deleted file mode 100644 index 7bc00b86..00000000 --- a/pfsagentd/container/insert/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -Dockerfile -pfsagentd -pfsagentd-init -pfsagentd-swift-auth-plugin -pfsagent.conf_* diff --git a/pfsagentd/container/insert/MakePFSAgentDockerfile b/pfsagentd/container/insert/MakePFSAgentDockerfile deleted file mode 100755 index d9de1037..00000000 --- a/pfsagentd/container/insert/MakePFSAgentDockerfile +++ /dev/null @@ -1,206 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -function usage { - echo "usage:" - echo " $0 -h" - echo " outputs this usage and conf file format" - echo " $0 --clean" - echo " cleans current directory of generated files" - echo " $0 " - echo " derives a PFSAgent-enabled Dockerfile" - echo " based on directives in " -} - -trim() -{ - local trimmed="$1" - while [[ $trimmed == ' '* ]]; do - trimmed="${trimmed## }" - done - while [[ $trimmed == *' ' ]]; do - trimmed="${trimmed%% }" - done - echo "$trimmed" -} - -if [[ $# -eq 1 ]] -then - if [[ "$1" = "-h" ]] - then - usage - echo - echo " is in .INI format" - echo - echo "Commented lines begin with '#'" - echo - echo "The first section, [Globals], should have the following Key:Value pairs:" - echo - echo "SourceImage: " - echo "PFSAgentImage: " - echo - echo "Other sections enumerate each PFSAgent instance to configure." - echo "The name of each section should be unique, ane be neither" - echo "\"Globals\" nor \"TEMPLATE\" and enclosed in '[' and ']'." - echo - echo "Each such section (and there must be at least one of them)" - echo "should have the following Key:Value pairs:" - echo - echo "FUSEVolumeName: " - echo "FUSEMountPointPath: " - echo "PlugInEnvValue: " - echo "HTTPServerTCPPort: " - exit 0 - elif [[ "$1" = "--clean" ]] - then - rm -f pfsagentd pfsagentd-init pfsagentd-swift-auth-plugin pfsagent.conf_* Dockerfile - exit 0 - else - CONF_FILE=$1 - fi -else - usage - exit 1 -fi - -CURRENT_SECTION_NAME="" -SOURCE_IMAGE="" -PFS_AGENT_IMAGE="" - -while IFS= read -r line -do - if [[ ${#line} -ne 0 ]] - then - if [[ ${line:0:1} != "#" ]] - then - if [[ ${line:0:1} = "[" ]] - then - NEW_SECTION_NAME=${line:1:${#line}-2} - if [[ ${#NEW_SECTION_NAME} -eq 0 ]] - then - echo "Section name cannot be empty" - exit 1 - fi - if [[ $NEW_SECTION_NAME = "Globals" ]] - then - if [[ ${CURRENT_SECTION_NAME} != "" ]] - then - echo "Encountered 2nd [Globals] section" - exit 1 - fi - CURRENT_SECTION_NAME="Globals" - else - if [[ ${CURRENT_SECTION_NAME} = "" ]] - then - echo "First section must be [Globals]" - exit 1 - elif [[ ${CURRENT_SECTION_NAME} = "Globals" ]] - then - if [[ $NEW_SECTION_NAME = "TEMPLATE" ]] - then - echo "Encountered illegal section [TEMPLATE]" - exit 1 - fi - if [[ ${SOURCE_IMAGE} = "" ]] - then - echo "Missing SourceImage in [Globals] section" - exit 1 - fi - if [[ ${PFS_AGENT_IMAGE} = "" ]] - then - echo "Missing PFSAgentImage in [Globals] section" - exit 1 - fi - CONTAINER=`docker create -ti pfsagent-build:latest` - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd . - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd-init . - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd-swift-auth-plugin . - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/src/github.com/NVIDIA/proxyfs/pfsagentd/pfsagent.conf pfsagent.conf_TEMPLATE - docker container rm $CONTAINER > /dev/null - else - if [[ $NEW_SECTION_NAME = "TEMPLATE" ]] - then - echo "Encountered illegal section [TEMPLATE]" - exit 1 - fi - fi - CURRENT_SECTION_NAME=$NEW_SECTION_NAME - cp pfsagent.conf_TEMPLATE pfsagent.conf_$CURRENT_SECTION_NAME - fi - else - if [[ ${CURRENT_SECTION_NAME} == "" ]] - then - echo "Found Key:Value line before any section" - exit 1 - elif [[ ${CURRENT_SECTION_NAME} == "Globals" ]] - then - if [[ $line == "SourceImage:"* ]] - then - SOURCE_IMAGE="$(trim "${line:12}")" - elif [[ $line == "PFSAgentImage:"* ]] - then - PFS_AGENT_IMAGE="$(trim "${line:14}")" - else - echo "Encountered unexpected Key:Value line in [Globals] section: $line" - exit 1 - fi - else - echo $line >> pfsagent.conf_$CURRENT_SECTION_NAME - fi - fi - fi - fi -done < $CONF_FILE - -if [[ ${CURRENT_SECTION_NAME} = "" ]] -then - echo "Must have properly populated [Globals] section" - echo "and at least one other section" - exit 1 -fi - -if [[ ${CURRENT_SECTION_NAME} = "Globals" ]] -then - if [[ ${SOURCE_IMAGE} = "" ]] - then - echo "Missing SourceImage in [Globals] section" - exit 1 - fi - if [[ ${PFS_AGENT_IMAGE} = "" ]] - then - echo "Missing PFSAgentImage in [Globals] section" - exit 1 - fi - CONTAINER=`docker create -ti pfsagent-build:latest` - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd . - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd-init . - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd-swift-auth-plugin . - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/src/github.com/NVIDIA/proxyfs/pfsagentd/pfsagent.conf pfsagent.conf_TEMPLATE - docker container rm $CONTAINER > /dev/null -fi - -ENTRYPOINT=`docker inspect --format '{{json .Config.Entrypoint}}' ${SOURCE_IMAGE}` -CMD=`docker inspect --format '{{json .Config.Cmd}}' ${SOURCE_IMAGE}` -WORKING_DIR=`docker inspect --format '{{json .Config.WorkingDir}}' ${SOURCE_IMAGE}` - -echo "FROM ${SOURCE_IMAGE}" > Dockerfile -echo "RUN yum install -y fuse" >> Dockerfile -echo "WORKDIR /opt/PFSAgent" >> Dockerfile -echo "COPY pfsagentd ." >> Dockerfile -echo "COPY pfsagentd-init ." >> Dockerfile -echo "COPY pfsagentd-swift-auth-plugin ." >> Dockerfile -echo "COPY pfsagent.conf_* /opt/PFSAgent/" >> Dockerfile -echo "ENV PATH /opt/PFSAgent:$PATH" >> Dockerfile -echo "ENV PFSAGENT_PARENT_ENTRYPOINT '${ENTRYPOINT}'" >> Dockerfile -echo "ENV PFSAGENT_PARENT_CMD '${CMD}'" >> Dockerfile -echo "ENV PFSAGENT_PARENT_WORKING_DIR ${WORKING_DIR}" >> Dockerfile -echo "ENTRYPOINT [\"pfsagentd-init\"]" >> Dockerfile -echo "" >> Dockerfile -echo "# To run a container for this image:" >> Dockerfile -echo "#" >> Dockerfile -echo "# docker run -it --rm --device /dev/fuse --cap-add SYS_ADMIN -p 9090:9090 " >> Dockerfile diff --git a/pfsagentd/container/insert/MakePFSAgentDockerfile.conf_SAMPLE b/pfsagentd/container/insert/MakePFSAgentDockerfile.conf_SAMPLE deleted file mode 100644 index fa5bc923..00000000 --- a/pfsagentd/container/insert/MakePFSAgentDockerfile.conf_SAMPLE +++ /dev/null @@ -1,17 +0,0 @@ -# [Globals] section identifies docker container images in format -# -# Note: This section should appear before all other sections -# -[Globals] -SourceImage: centos:latest -PFSAgentImage: pfsagent-build:latest - -# [] section identifies the PFSAgent volume-specific Key:Value settings desired -# -# Note: The value of should follow the rules of a valid POSIX basename -# -[SampleVolume] -FUSEVolumeName: CommonVolume -FUSEMountPointPath: AgentMountPoint -PlugInEnvValue: {"AuthURL":"http://172.17.0.2:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"} -HTTPServerTCPPort: 9090 diff --git a/pfsagentd/file_inode.go b/pfsagentd/file_inode.go deleted file mode 100644 index 004ae4bc..00000000 --- a/pfsagentd/file_inode.go +++ /dev/null @@ -1,1938 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "container/list" - "fmt" - "io" - "net/http" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/jrpcfs" -) - -func (fileInode *fileInodeStruct) doFlushIfNecessary() { - var ( - chunkedPutContext *chunkedPutContextStruct - chunkedPutContextElement *list.Element - flushWG sync.WaitGroup - ) - - // grantedLock = fileInode.getExclusiveLock() - - if 0 == fileInode.chunkedPutList.Len() { - // No Chunked PUTs in flight... so we can just exit - // grantedLock.release() - return - } - - // At least one Chunked PUT is in flight... so we know we'll have to block later - - flushWG.Add(1) - _ = fileInode.chunkedPutFlushWaiterList.PushBack(&flushWG) - - // Now see if we need to initiate the flush - - if fileInode.flushInProgress { - // We do not need to send a flush - } else { - // No explicit flush is in progress... so make it appear so - - fileInode.flushInProgress = true - - // See if the last Chunked PUT is already flushing anyway - - chunkedPutContextElement = fileInode.chunkedPutList.Back() - chunkedPutContext = chunkedPutContextElement.Value.(*chunkedPutContextStruct) - - if chunkedPutContextStateOpen == chunkedPutContext.state { - // We need to trigger a Flush - chunkedPutContext.state = chunkedPutContextStateClosing - close(chunkedPutContext.sendChan) - } - } - - // grantedLock.release() - - // Finally, wait for the flush to complete - - flushWG.Wait() -} - -func pruneFileInodeDirtyListIfNecessary() { - var ( - fileInode *fileInodeStruct - fileInodeElement *list.Element - ) - - for { - globals.Lock() - if uint64(globals.fileInodeDirtyList.Len()) < globals.config.DirtyFileLimit { - // There is room for fileInode about to be added to the list - globals.Unlock() - return - } - fileInodeElement = globals.fileInodeDirtyList.Front() - globals.Unlock() - fileInode = fileInodeElement.Value.(*fileInodeStruct) - fileInode.doFlushIfNecessary() - } -} - -func emptyFileInodeDirtyListAndLogSegmentChan() { - var ( - fileInode *fileInodeStruct - fileInodeDirtyLogSegmentChanIndex uint64 - fileInodeElement *list.Element - ) - - for { - globals.Lock() - - if 0 == globals.fileInodeDirtyList.Len() { - // The list is now empty - - globals.Unlock() - - // And globals.config.DirtyLogSegmentLimit should be fully stocked with globals.config.DirtyLogSegmentLimit elements... so empty it also - - for fileInodeDirtyLogSegmentChanIndex = 0; fileInodeDirtyLogSegmentChanIndex < globals.config.DirtyLogSegmentLimit; fileInodeDirtyLogSegmentChanIndex++ { - _ = <-globals.fileInodeDirtyLogSegmentChan - } - - // And we are done... - - return - } - - // Kick off a flush if the first (then next) element of globals.fileInodeDirtyList - - fileInodeElement = globals.fileInodeDirtyList.Front() - - globals.Unlock() - - fileInode = fileInodeElement.Value.(*fileInodeStruct) - fileInode.doFlushIfNecessary() - } -} - -func (chunkedPutContext *chunkedPutContextStruct) sendDaemon() { - var ( - expirationDelay time.Duration - expirationTime time.Time - fileInode *fileInodeStruct - flushWaiterListElement *list.Element - nextChunkedPutContext *chunkedPutContextStruct - nextChunkedPutListElement *list.Element - sendChanOpenOrNonEmpty bool - ) - - fileInode = chunkedPutContext.fileInode - - // Kick off Chunked PUT - - chunkedPutContext.Add(1) - go chunkedPutContext.performChunkedPut() - - // Start MaxFlushTime timer - - expirationTime = time.Now().Add(globals.config.MaxFlushTime) - - // Loop awaiting chunks (that could be explicit flushes) or expirationTime - - for { - expirationDelay = expirationTime.Sub(time.Now()) - - select { - case <-time.After(expirationDelay): - // MaxFlushTime-triggered flush requested - - // grantedLock = fileInode.getExclusiveLock() - chunkedPutContext.state = chunkedPutContextStateClosing - // grantedLock.release() - - goto PerformFlush - case _, sendChanOpenOrNonEmpty = <-chunkedPutContext.sendChan: - // Send chunk to *chunkedPutContextStruct.Read() - - select { - case chunkedPutContext.wakeChan <- struct{}{}: - // We just notified Read() - default: - // We didn't need to notify Read() - } - - if !sendChanOpenOrNonEmpty { - goto PerformFlush - } - } - } - -PerformFlush: - - // Tell *chunkedPutContextStruct.Read() to finish & wait for it to finish - - close(chunkedPutContext.wakeChan) - chunkedPutContext.Wait() - - // Chunked PUT is complete - - // grantedLock = fileInode.getExclusiveLock() - - chunkedPutContext.state = chunkedPutContextStateClosed - - // But first, make sure sendChan is drained - - for { - select { - case _, sendChanOpenOrNonEmpty = <-chunkedPutContext.sendChan: - if sendChanOpenOrNonEmpty { - continue - } else { - goto EscapeSendChanDrain - } - default: - goto EscapeSendChanDrain - } - } - -EscapeSendChanDrain: - - // Can we tell ProxyFS about it and dispose of it? - - if nil == chunkedPutContext.chunkedPutListElement.Prev() { - // We can record this chunkedPutContext as having completed - - nextChunkedPutListElement = chunkedPutContext.chunkedPutListElement.Next() - - chunkedPutContext.complete() - - // Check to see subsequent chunkedPutContext's are also closed and able to be completed - - for nil != nextChunkedPutListElement { - nextChunkedPutContext = nextChunkedPutListElement.Value.(*chunkedPutContextStruct) - - if chunkedPutContextStateClosed != nextChunkedPutContext.state { - // Ran into an un-closed chunkedPutContext...so simply exit the loop - break - } - - // We can similarly reccord this chunkedPutContext as having completed - - nextChunkedPutListElement = nextChunkedPutContext.chunkedPutListElement.Next() - - nextChunkedPutContext.complete() - } - } - - if 0 == fileInode.chunkedPutList.Len() { - // Indicate flush is complete - - fileInode.flushInProgress = false - - for fileInode.chunkedPutFlushWaiterList.Len() > 0 { - flushWaiterListElement = fileInode.chunkedPutFlushWaiterList.Front() - _ = fileInode.chunkedPutFlushWaiterList.Remove(flushWaiterListElement) - flushWaiterListElement.Value.(*sync.WaitGroup).Done() - } - - // Remove globals.fileInode from globals.fileInodeDirtyList - - globals.Lock() - globals.fileInodeDirtyList.Remove(fileInode.dirtyListElement) - fileInode.dirtyListElement = nil - globals.Unlock() - } - - // grantedLock.release() -} - -func (chunkedPutContext *chunkedPutContextStruct) performChunkedPut() { - var ( - chunkedPutRequest *http.Request - containerAndObjectNames string - containerAndObjectNamesSplit []string - err error - ok bool - physPathSplit []string - provisionObjectReply *jrpcfs.ProvisionObjectReply - provisionObjectRequest *jrpcfs.ProvisionObjectRequest - statusCode int - swiftStorageURL string - ) - - provisionObjectRequest = &jrpcfs.ProvisionObjectRequest{ - MountID: globals.mountID, - } - - provisionObjectReply = &jrpcfs.ProvisionObjectReply{} - - err = globals.retryRPCClient.Send("RpcProvisionObject", provisionObjectRequest, provisionObjectReply) - if nil != err { - logFatalf("*chunkedPutContextStruct.performChunkedPut() call to Server.RpcProvisionObject failed: %v", err) - } - - physPathSplit = strings.SplitAfterN(provisionObjectReply.PhysPath, "/", 4) - containerAndObjectNames = physPathSplit[len(physPathSplit)-1] - containerAndObjectNamesSplit = strings.Split(containerAndObjectNames, "/") - chunkedPutContext.containerName = containerAndObjectNamesSplit[0] - chunkedPutContext.objectName = containerAndObjectNamesSplit[1] - - swiftStorageURL = fetchStorageURL() - - chunkedPutRequest, err = http.NewRequest(http.MethodPut, swiftStorageURL+"/"+chunkedPutContext.containerName+"/"+chunkedPutContext.objectName, chunkedPutContext) - if nil != err { - logFatalf("*chunkedPutContextStruct.performChunkedPut() call to http.NewRequest() failed: %v", err) - } - - chunkedPutRequest.Header.Add("Transfer-Encoding", "chunked") - - chunkedPutContext.pos = 0 - - _, _, ok, statusCode = doHTTPRequest(chunkedPutRequest, http.StatusCreated) - if !ok { - logFatalf("*chunkedPutContextStruct.performChunkedPut() failed with unexpected statusCode: %v", statusCode) - } - - globals.stats.LogSegmentPutBytes.Add(uint64(len(chunkedPutContext.buf))) - - chunkedPutContext.Done() -} - -func (chunkedPutContext *chunkedPutContextStruct) complete() { - var ( - curExtentAsMultiObjectExtent *multiObjectExtentStruct - curExtentAsSingleObjectExtent *singleObjectExtentStruct - curExtentAsValue sortedmap.Value - curExtentIndex int - err error - extentMapLen int - fileInode *fileInodeStruct - ok bool - wroteReply *jrpcfs.WroteReply - wroteRequest *jrpcfs.WroteRequest - ) - - fileInode = chunkedPutContext.fileInode - - extentMapLen, err = chunkedPutContext.extentMap.Len() - if nil != err { - logFatalf("*chunkedPutContextStruct.complete() failed chunkedPutContext.extentMap.Len(): %v", err) - } - - // Now that LogSegment Chunked PUT has completed, update FileInode in ProxyFS and our fileInode.extentMap - - wroteRequest = &jrpcfs.WroteRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(fileInode.InodeNumber), - }, - ContainerName: chunkedPutContext.containerName, - ObjectName: chunkedPutContext.objectName, - FileOffset: make([]uint64, extentMapLen), - ObjectOffset: make([]uint64, extentMapLen), - Length: make([]uint64, extentMapLen), - WroteTimeNs: uint64(time.Now().UnixNano()), - } - - for curExtentIndex = 0; curExtentIndex < extentMapLen; curExtentIndex++ { - _, curExtentAsValue, ok, err = chunkedPutContext.extentMap.GetByIndex(curExtentIndex) - if nil != err { - logFatalf("*chunkedPutContextStruct.complete() failed chunkedPutContext.extentMap.GetByIndex(): %v", err) - } - if !ok { - logFatalf("*chunkedPutContextStruct.complete() chunkedPutContext.extentMap.GetByIndex() returned !ok") - } - curExtentAsSingleObjectExtent = curExtentAsValue.(*singleObjectExtentStruct) - - wroteRequest.FileOffset[curExtentIndex] = curExtentAsSingleObjectExtent.fileOffset - wroteRequest.ObjectOffset[curExtentIndex] = curExtentAsSingleObjectExtent.objectOffset - wroteRequest.Length[curExtentIndex] = curExtentAsSingleObjectExtent.length - - curExtentAsMultiObjectExtent = &multiObjectExtentStruct{ - fileOffset: curExtentAsSingleObjectExtent.fileOffset, - containerName: chunkedPutContext.containerName, - objectName: chunkedPutContext.objectName, - objectOffset: curExtentAsSingleObjectExtent.objectOffset, - length: curExtentAsSingleObjectExtent.length, - } - - fileInode.updateExtentMap(curExtentAsMultiObjectExtent) - } - - wroteReply = &jrpcfs.WroteReply{} - - err = globals.retryRPCClient.Send("RpcWrote", wroteRequest, wroteReply) - if nil != err { - // logFatalf("*chunkedPutContextStruct.complete() failed Server.RpcWrote: %v", err) - logWarnf("TODO (i.e. convert to logFatalf) *chunkedPutContextStruct.complete() failed Server.RpcWrote: %v", err) - } - - // Remove this chunkedPutContext from fileInode.chunkedPutList - - _ = fileInode.chunkedPutList.Remove(chunkedPutContext.chunkedPutListElement) - - // fileInode.dereference() - - // Finally, yield our chunkedPutContext quota - - globals.fileInodeDirtyLogSegmentChan <- struct{}{} -} - -func (chunkedPutContext *chunkedPutContextStruct) Read(p []byte) (n int, err error) { - var ( - wakeChanOpenOrNonEmpty bool - ) - - _, wakeChanOpenOrNonEmpty = <-chunkedPutContext.wakeChan - - // grantedLock = chunkedPutContext.fileInode.getExclusiveLock() - - n = len(chunkedPutContext.buf) - chunkedPutContext.pos - - if n < 0 { - logFatalf("*chunkedPutContextStruct.Read() called with pos past beyond len(chunkedPutContext.buf)") - } - - if n > 0 { - // Return any unsent data in buf that will fit in p immediately - - if n > len(p) { - n = len(p) - } - - copy(p, chunkedPutContext.buf[chunkedPutContext.pos:chunkedPutContext.pos+n]) - - chunkedPutContext.pos += n - - // grantedLock.release() - - err = nil - return - } - - // grantedLock.release() - - // At this point, n == 0... do we need to send EOF? - - if wakeChanOpenOrNonEmpty { - // Nope... we were awoken to send bytes but we'd already sent them - - err = nil - return - } - - // Time to send EOF - - err = io.EOF - - return -} - -func (chunkedPutContext *chunkedPutContextStruct) Close() (err error) { - // Make sure Read() gets a chance to cleanly exit - - if chunkedPutContext.inRead { - select { - case chunkedPutContext.wakeChan <- struct{}{}: - // We just notified Read() - default: - // We didn't need to notify Read() - } - - for chunkedPutContext.inRead { - time.Sleep(chunkedPutContextExitReadPollingRate) - } - } - - // To ensure retry resends all the data, reset pos - - chunkedPutContext.pos = 0 - - err = nil - return -} - -// populateExtentMap ensures that the range [fileOffset:fileOffset+length) is covered by -// fileInode.extentMap whether or not it is needed. Extents won't be needed if there is -// some fileInode.chunkedPutListElement that logically supercedes it. -// -func (fileInode *fileInodeStruct) populateExtentMap(fileOffset uint64, length uint64) (err error) { - var ( - curExtent *multiObjectExtentStruct - curExtentAsValue sortedmap.Value - curExtentIndex int - curFileOffset uint64 - exhausted bool - ok bool - ) - - if nil == fileInode.extentMap { - // Create an empty ExtentMap... and perform initial population - - fileInode.extentMap = sortedmap.NewLLRBTree(sortedmap.CompareUint64, fileInode) - - exhausted, err = fileInode.populateExtentMapHelper(fileOffset) - if nil != err { - fileInode.extentMap = nil - return - } - if exhausted { - return - } - } - - // Handle cases where [fileOffset:fileOffset+length) references beyond FileSize - - if fileOffset >= fileInode.cachedStat.Size { - // The entire [fileOffset:fileOffset+length) lies beyond FileSize... so just return - - err = nil - return - } - if (fileOffset + length) > fileInode.cachedStat.Size { - // Truncate length since ExtentMap doesn't contain any extent beyond FileSize - - length = fileInode.cachedStat.Size - fileOffset - } - -Restart: - - curFileOffset = fileOffset - - curExtentIndex, _, err = fileInode.extentMap.BisectLeft(curFileOffset) - if nil != err { - logFatalf("populateExtentMap() couldn't fetch extent [Case 1]: %v", err) - } - - // Note it is possible for curExtentIndex == -1 if no extents are at or precede fileOffset - - for curFileOffset < (fileOffset + length) { - _, curExtentAsValue, ok, err = fileInode.extentMap.GetByIndex(curExtentIndex) - if nil != err { - logFatalf("populateExtentMap() couldn't fetch extent [Case 2]: %v", err) - } - - if !ok { - // No extent at curExtentIndex - so populate from here if possible and restart - - exhausted, err = fileInode.populateExtentMapHelper(curFileOffset) - if nil != err { - fileInode.extentMap = nil - return - } - if exhausted { - return - } - goto Restart - } - - curExtent, ok = curExtentAsValue.(*multiObjectExtentStruct) - if !ok { - logFatalf("populateExtentMap() couldn't fetch extent [Case 3]: %v", err) - } - - if curFileOffset < curExtent.fileOffset { - // Next extent starts after curFileOffset - so populate the hole if possible and restart - - exhausted, err = fileInode.populateExtentMapHelper(curFileOffset) - if nil != err { - fileInode.extentMap = nil - return - } - if exhausted { - return - } - goto Restart - } - - if curFileOffset >= (curExtent.fileOffset + curExtent.length) { - // Handle case where BisectLeft pointed at an extent before fileOffset - // and this extent ends before fileOffset - so populate from there if possible and restart - - exhausted, err = fileInode.populateExtentMapHelper(curExtent.fileOffset + curExtent.length) - if nil != err { - fileInode.extentMap = nil - return - } - if exhausted { - return - } - goto Restart - } - - // Advance to next extent to check for contiguity - - curFileOffset = curExtent.fileOffset + curExtent.length - curExtentIndex++ - } - - return -} - -// populateExtentMapHelper fetches an ExtentMapChunk anchored by fileOffset and inserts -// it into fileInode.extentMap using updateExtentMap(). -// -// If returned exhausted is true, there are no more extentMapEntry's to populate -// -func (fileInode *fileInodeStruct) populateExtentMapHelper(fileOffset uint64) (exhausted bool, err error) { - var ( - curExtent *multiObjectExtentStruct - curExtentAsValue sortedmap.Value - curExtentMapChunkIndex int - curFileOffset uint64 - extentMapEntry *inode.ExtentMapEntryStruct - extentMapLength int - fetchExtentMapChunkReply *jrpcfs.FetchExtentMapChunkReply - fetchExtentMapChunkRequest *jrpcfs.FetchExtentMapChunkRequest - ok bool - ) - - fetchExtentMapChunkRequest = &jrpcfs.FetchExtentMapChunkRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(fileInode.InodeNumber), - }, - FileOffset: fileOffset, - MaxEntriesFromFileOffset: int64(globals.config.FetchExtentsFromFileOffset), - MaxEntriesBeforeFileOffset: int64(globals.config.FetchExtentsBeforeFileOffset), - } - - fetchExtentMapChunkReply = &jrpcfs.FetchExtentMapChunkReply{} - - err = globals.retryRPCClient.Send("RpcFetchExtentMapChunk", fetchExtentMapChunkRequest, fetchExtentMapChunkReply) - if nil != err { - return - } - - if 0 == len(fetchExtentMapChunkReply.ExtentMapEntry) { - exhausted = true - return - } - - exhausted = false - - curFileOffset = fetchExtentMapChunkReply.FileOffsetRangeStart - - for curExtentMapChunkIndex = range fetchExtentMapChunkReply.ExtentMapEntry { - extentMapEntry = &fetchExtentMapChunkReply.ExtentMapEntry[curExtentMapChunkIndex] - - if curFileOffset < extentMapEntry.FileOffset { - // We need to insert a preceeding zero-fill extent first - - curExtent = &multiObjectExtentStruct{ - fileOffset: curFileOffset, - containerName: "", - objectName: "", - objectOffset: 0, - length: extentMapEntry.FileOffset - curFileOffset, - } - - fileInode.updateExtentMap(curExtent) - } - - // Insert the actual extent - - curExtent = &multiObjectExtentStruct{ - fileOffset: extentMapEntry.FileOffset, - containerName: extentMapEntry.ContainerName, - objectName: extentMapEntry.ObjectName, - objectOffset: extentMapEntry.LogSegmentOffset, - length: extentMapEntry.Length, - } - - fileInode.updateExtentMap(curExtent) - - curFileOffset = extentMapEntry.FileOffset + extentMapEntry.Length - } - - if curFileOffset < fetchExtentMapChunkReply.FileOffsetRangeEnd { - // We need to insert a trailing zero-fill extent - - curExtent = &multiObjectExtentStruct{ - fileOffset: curFileOffset, - containerName: "", - objectName: "", - objectOffset: 0, - length: fetchExtentMapChunkReply.FileOffsetRangeEnd - curFileOffset, - } - - _, err = fileInode.extentMap.Put(curExtent.fileOffset, curExtent) - if nil != err { - logFatalf("populateExtentMap() couldn't insert zero-fill extent [Case 2]: %v", err) - } - } - - // Finally, we need to trim, as necessary, excess extents - - extentMapLength, err = fileInode.extentMap.Len() - if nil != err { - logFatalf("populateExtentMap() couldn't get number of extents: %v", err) - } - - for { - if 0 == extentMapLength { - // Handle case where we have no extents left at all - - return - } - - _, curExtentAsValue, ok, err = fileInode.extentMap.GetByIndex(extentMapLength - 1) - if nil != err { - logFatalf("populateExtentMap() couldn't get last extent [Case 1]: %v", err) - } - if !ok { - logFatalf("populateExtentMap() couldn't get last extent [Case 2]") - } - - curExtent, ok = curExtentAsValue.(*multiObjectExtentStruct) - if !ok { - logFatalf("populateExtentMap() couldn't get last extent [Case 3]") - } - - if (curExtent.fileOffset + curExtent.length) <= fetchExtentMapChunkReply.FileSize { - // Last extent does not extend beyond FileSize... so we are done - - return - } - - if curExtent.fileOffset < fetchExtentMapChunkReply.FileSize { - // Last extent crossed FileSize boundary... truncate it and we are done - - curExtent.length = fileInode.cachedStat.Size - curExtent.fileOffset - - return - } - - // Last extent completely beyond FileSize... just delete it and loop - - fileInode.extentMap.DeleteByIndex(extentMapLength - 1) - - extentMapLength-- - } -} - -// updateExtentMap is called to update the ExtentMap and, as necessary, the FileSize with -// the supplied multiObjectExtent. This func is used during fetching of ExtentMap chunks -// by *fileInodeStruct.populateExtentMapHelper() and at completion of a Chunked PUT -// by *chunkedPutContextStruct.complete(). -// -func (fileInode *fileInodeStruct) updateExtentMap(newExtent *multiObjectExtentStruct) { - var ( - curExtent *multiObjectExtentStruct - curExtentAsValue sortedmap.Value - curExtentIndex int - curExtentLostLength uint64 - err error - found bool - ok bool - prevExtent *multiObjectExtentStruct - prevExtentAsValue sortedmap.Value - prevExtentIndex int - prevExtentNewLength uint64 - splitExtent *multiObjectExtentStruct - ) - - if nil == fileInode.extentMap { - // Create an empty ExtentMap - - fileInode.extentMap = sortedmap.NewLLRBTree(sortedmap.CompareUint64, fileInode) - } - - // Locate prevExtent (if any) - - prevExtentIndex, found, err = fileInode.extentMap.BisectLeft(newExtent.fileOffset) - if nil != err { - logFatalf("updateExtentMap() couldn't find prevExtent [Case 1]: %v", err) - } - - if found { - // Make prevExtentIndex truly point to the previous (and non-overlapping) extent (if any) - - prevExtentIndex-- - } else { - if prevExtentIndex < 0 { - // No prevExtent exists... so it cannot need to be split - } else { - // A prevExtent exists... but does it need to be split? - - _, prevExtentAsValue, ok, err = fileInode.extentMap.GetByIndex(prevExtentIndex) - if nil != err { - logFatalf("updateExtentMap() couldn't find prevExtent [Case 2]: %v", err) - } - if !ok { - logFatalf("updateExtentMap() couldn't find prevExtent [Case 3]") - } - prevExtent, ok = prevExtentAsValue.(*multiObjectExtentStruct) - if !ok { - logFatalf("updateExtentMap() couldn't find prevExtent [Case 4]") - } - - if (prevExtent.fileOffset + prevExtent.length) > newExtent.fileOffset { - // Existing prevExtent does overlap... so we need to split it now - - prevExtentNewLength = newExtent.fileOffset - prevExtent.fileOffset - - splitExtent = &multiObjectExtentStruct{ - fileOffset: newExtent.fileOffset, - containerName: prevExtent.containerName, - objectName: prevExtent.objectName, - objectOffset: prevExtent.objectOffset + prevExtentNewLength, - length: prevExtent.length - prevExtentNewLength, - } - - prevExtent.length = prevExtentNewLength - - ok, err = fileInode.extentMap.Put(splitExtent.fileOffset, splitExtent) - if nil != err { - logFatalf("updateExtentMap() couldn't split prevExtent [Case 1]: %v", err) - } - if !ok { - logFatalf("updateExtentMap() couldn't split prevExtent [Case 2]") - } - } else { - // Existing prevExtent does not overlap - } - } - } - - // Now loop thru extents after prevExtent replaced (partially or fully) by newExtent - - curExtentIndex = prevExtentIndex + 1 - - for { - _, curExtentAsValue, ok, err = fileInode.extentMap.GetByIndex(curExtentIndex) - if nil != err { - logFatalf("updateExtentMap() couldn't find curExtent [Case 1]: %v", err) - } - - if !ok { - // Simple case where we walked off the end of the ExtentMap - - break - } - - curExtent, ok = curExtentAsValue.(*multiObjectExtentStruct) - if !ok { - logFatalf("updateExtentMap() couldn't find curExtent [Case 2]: %v", err) - } - - if (newExtent.fileOffset + newExtent.length) <= curExtent.fileOffset { - // Simple case where we walked to an Extent that start after newExtent - - break - } - - // At this point, we know we are either going to split curExtent - // or replace it... so we must remove it from the ExtentMap regardless - // (since splitting curExtent would change it's fileOffset Key) - - ok, err = fileInode.extentMap.DeleteByIndex(curExtentIndex) - if nil != err { - logFatalf("updateExtentMap() couldn't delete curExtent [Case 1]: %v", err) - } - if !ok { - logFatalf("updateExtentMap() couldn't delete curExtent [Case 2]: %v", err) - } - - if (curExtent.fileOffset + curExtent.length) > (newExtent.fileOffset + newExtent.length) { - // We need to split curExtent because newExtent only partially replaces it - - curExtentLostLength = (newExtent.fileOffset + newExtent.length) - curExtent.fileOffset - - splitExtent = &multiObjectExtentStruct{ - fileOffset: curExtent.fileOffset + curExtentLostLength, - containerName: curExtent.containerName, - objectName: curExtent.objectName, - objectOffset: curExtent.objectOffset + curExtentLostLength, - length: curExtent.length - curExtentLostLength, - } - - ok, err = fileInode.extentMap.Put(splitExtent.fileOffset, splitExtent) - if nil != err { - logFatalf("updateExtentMap() couldn't split curExtent [Case 1]: %v", err) - } - if !ok { - logFatalf("updateExtentMap() couldn't split curExtent [Case 2]") - } - - // We also know that we are done scanning - - break - } - } - - // We can finally insert newExtent without fear of colliding with existing extents - - ok, err = fileInode.extentMap.Put(newExtent.fileOffset, newExtent) - if nil != err { - logFatalf("updateExtentMap() couldn't insert newExtent [Case 1]: %v", err) - } - if !ok { - logFatalf("updateExtentMap() couldn't insert newExtent [Case 2]") - } - - if (newExtent.fileOffset + newExtent.length) > fileInode.cachedStat.Size { - fileInode.cachedStat.Size = newExtent.fileOffset + newExtent.length - } -} - -// getReadPlan returns a slice of extents and their span (to aid in the make([]byte,) call needed -// by the caller to provision the slice into which they will copy the extents). Each extent will -// be one of three types: -// -// singleObjectExtentWithLinkStruct - a reference to a portion of a LogSegment being written by a chunkedPutContextStruct -// multiObjectExtentStruct - a reference to a portion of a LogSegment described by a fileInodeStruct.extentMap -// multiObjectExtentStruct - a description of a zero-filled extent (.objectName == "") -// -func (fileInode *fileInodeStruct) getReadPlan(fileOffset uint64, length uint64) (readPlan []interface{}, readPlanSpan uint64) { - var ( - chunkedPutContext *chunkedPutContextStruct - chunkedPutContextAsElement *list.Element - curExtentAsValue sortedmap.Value - curExtentIndex int - curFileOffset uint64 - curMultiObjectExtent *multiObjectExtentStruct - err error - multiObjectReadPlanStep *multiObjectExtentStruct - ok bool - remainingLength uint64 - ) - - // First assemble readPlan based upon fileInode.extentMap - - readPlan = make([]interface{}, 0, 1) - - curFileOffset = fileOffset - remainingLength = length - - curExtentIndex, _, err = fileInode.extentMap.BisectLeft(fileOffset) - if nil != err { - logFatalf("getReadPlan() couldn't find curExtent: %v", err) - } - - for remainingLength > 0 { - _, curExtentAsValue, ok, err = fileInode.extentMap.GetByIndex(curExtentIndex) - if nil != err { - logFatalf("getReadPlan() couldn't find curExtent [Case 1]: %v", err) - } - - if !ok { - // Crossed EOF - stop here - - break - } - - curMultiObjectExtent, ok = curExtentAsValue.(*multiObjectExtentStruct) - if !ok { - logFatalf("getReadPlan() couldn't find curExtent [Case 2]: %v", err) - } - - if (curMultiObjectExtent.fileOffset + curMultiObjectExtent.length) <= curFileOffset { - // curExtent ends at or before curFileOffset - stop here - - break - } - - multiObjectReadPlanStep = &multiObjectExtentStruct{ - fileOffset: curFileOffset, - containerName: curMultiObjectExtent.containerName, - objectName: curMultiObjectExtent.objectName, // May be == "" - objectOffset: curMultiObjectExtent.objectOffset + (curFileOffset - curMultiObjectExtent.fileOffset), - length: curMultiObjectExtent.length - (curFileOffset - curMultiObjectExtent.fileOffset), - } - - if remainingLength < multiObjectReadPlanStep.length { - // This is the last readPlanStep and needs to be truncated - - multiObjectReadPlanStep.length = remainingLength - } - - if 0 == multiObjectReadPlanStep.length { - // Reached EOF - stop here - - break - } - - readPlan = append(readPlan, multiObjectReadPlanStep) - - curFileOffset += multiObjectReadPlanStep.length - remainingLength -= multiObjectReadPlanStep.length - - curExtentIndex++ - } - - // Compute tentative readPlanSpan - - if 0 == len(readPlan) { - readPlanSpan = 0 - } else { - multiObjectReadPlanStep = readPlan[len(readPlan)-1].(*multiObjectExtentStruct) - readPlanSpan = (multiObjectReadPlanStep.fileOffset + multiObjectReadPlanStep.length) - fileOffset - } - - // But we must apply, in order, any changes due to chunkedPutContextStruct's - - chunkedPutContextAsElement = fileInode.chunkedPutList.Front() - for nil != chunkedPutContextAsElement { - chunkedPutContext = chunkedPutContextAsElement.Value.(*chunkedPutContextStruct) - readPlan, readPlanSpan = chunkedPutContext.getReadPlanHelper(fileOffset, length, readPlan) - chunkedPutContextAsElement = chunkedPutContextAsElement.Next() - } - - // And we are done... - - return -} - -func (chunkedPutContext *chunkedPutContextStruct) getReadPlanHelper(fileOffset uint64, length uint64, inReadPlan []interface{}) (outReadPlan []interface{}, outReadPlanSpan uint64) { - var ( - curFileOffset uint64 - err error - found bool - inReadPlanStepAsInterface interface{} - inReadPlanStepAsMultiObjectExtent *multiObjectExtentStruct - inReadPlanStepAsSingleObjectExtent *singleObjectExtentWithLinkStruct - inReadPlanStepFileOffset uint64 - inReadPlanStepLength uint64 - ok bool - outReadPlanStepAsInterface interface{} - outReadPlanStepAsMultiObjectExtent *multiObjectExtentStruct - outReadPlanStepAsSingleObjectExtent *singleObjectExtentWithLinkStruct - overlapExtent *singleObjectExtentStruct - overlapExtentAsValue sortedmap.Value - overlapExtentWithLink *singleObjectExtentWithLinkStruct - overlapExtentIndex int - postExtent *singleObjectExtentStruct - postExtentAsValue sortedmap.Value - postExtentIndex int - postOverlapLength uint64 - preExtent *singleObjectExtentStruct - preExtentAsValue sortedmap.Value - preExtentIndex int - preOverlapLength uint64 - remainingLength uint64 - wasMultiObjectReadPlanStep bool - ) - - outReadPlan = make([]interface{}, 0, len(inReadPlan)) - - for _, inReadPlanStepAsInterface = range inReadPlan { - // Compute overlap with chunkedPutContext.extentMap - - inReadPlanStepAsMultiObjectExtent, wasMultiObjectReadPlanStep = inReadPlanStepAsInterface.(*multiObjectExtentStruct) - if wasMultiObjectReadPlanStep { - inReadPlanStepAsSingleObjectExtent = nil - inReadPlanStepFileOffset = inReadPlanStepAsMultiObjectExtent.fileOffset - inReadPlanStepLength = inReadPlanStepAsMultiObjectExtent.length - } else { - inReadPlanStepAsMultiObjectExtent = nil - inReadPlanStepAsSingleObjectExtent = inReadPlanStepAsInterface.(*singleObjectExtentWithLinkStruct) - inReadPlanStepFileOffset = inReadPlanStepAsSingleObjectExtent.fileOffset - inReadPlanStepLength = inReadPlanStepAsSingleObjectExtent.length - } - - preExtentIndex, found, err = chunkedPutContext.extentMap.BisectLeft(inReadPlanStepFileOffset) - if nil != err { - logFatalf("getReadPlanHelper() couldn't find preExtentIndex: %v", err) - } - if found { - // Back up preExtentIndex... we know previous extent (if any) doesn't overlap - preExtentIndex-- - } else { - // But preExtentIndex might point to extent overlapping - if 0 <= preExtentIndex { - _, preExtentAsValue, _, err = chunkedPutContext.extentMap.GetByIndex(preExtentIndex) - if nil != err { - logFatalf("getReadPlanHelper() couldn't fetch preExtent: %v", err) - } - preExtent = preExtentAsValue.(*singleObjectExtentStruct) - if (preExtent.fileOffset + preExtent.length) > inReadPlanStepFileOffset { - preExtentIndex-- - } - } - } - postExtentIndex, _, err = chunkedPutContext.extentMap.BisectRight(inReadPlanStepFileOffset + inReadPlanStepLength) - if nil != err { - logFatalf("getReadPlanHelper() couldn't find postExtentIndex [Case 1]: %v", err) - } - - if 1 == (postExtentIndex - preExtentIndex) { - // No overlap... replicate inReadPlanStep as is - - outReadPlan = append(outReadPlan, inReadPlanStepAsInterface) - - continue - } - - // Apply overlapping extents from chunkedPutContext.extentMap with inReadPlanStep - - curFileOffset = inReadPlanStepFileOffset - remainingLength = inReadPlanStepLength - - for overlapExtentIndex = preExtentIndex + 1; overlapExtentIndex < postExtentIndex; overlapExtentIndex++ { - _, overlapExtentAsValue, _, err = chunkedPutContext.extentMap.GetByIndex(overlapExtentIndex) - if nil != err { - logFatalf("getReadPlanHelper() couldn't find overlapExtentIndex: %v", err) - } - overlapExtent = overlapExtentAsValue.(*singleObjectExtentStruct) - - if overlapExtent.fileOffset < curFileOffset { - preOverlapLength = curFileOffset - overlapExtent.fileOffset - } else { - preOverlapLength = 0 - } - if (overlapExtent.fileOffset + overlapExtent.length) > (curFileOffset + remainingLength) { - postOverlapLength = (overlapExtent.fileOffset + overlapExtent.length) - (curFileOffset + remainingLength) - } else { - postOverlapLength = 0 - } - - overlapExtentWithLink = &singleObjectExtentWithLinkStruct{ - fileOffset: overlapExtent.fileOffset + preOverlapLength, - objectOffset: overlapExtent.objectOffset + preOverlapLength, - length: overlapExtent.length - (preOverlapLength + postOverlapLength), - chunkedPutContext: chunkedPutContext, - } - - if curFileOffset < overlapExtentWithLink.fileOffset { - // Append non-overlapped portion of inReadPlanStep preceeding overlapExtentWithLink - - if wasMultiObjectReadPlanStep { - outReadPlanStepAsMultiObjectExtent = &multiObjectExtentStruct{ - fileOffset: curFileOffset, - containerName: inReadPlanStepAsMultiObjectExtent.containerName, - objectName: inReadPlanStepAsMultiObjectExtent.objectName, - objectOffset: inReadPlanStepAsMultiObjectExtent.objectOffset + (curFileOffset - inReadPlanStepAsMultiObjectExtent.fileOffset), - length: overlapExtentWithLink.fileOffset - curFileOffset, - } - - outReadPlan = append(outReadPlan, outReadPlanStepAsMultiObjectExtent) - - curFileOffset += outReadPlanStepAsMultiObjectExtent.length - remainingLength -= outReadPlanStepAsMultiObjectExtent.length - } else { - outReadPlanStepAsSingleObjectExtent = &singleObjectExtentWithLinkStruct{ - fileOffset: curFileOffset, - objectOffset: inReadPlanStepAsSingleObjectExtent.objectOffset + (curFileOffset - inReadPlanStepAsSingleObjectExtent.fileOffset), - length: overlapExtentWithLink.fileOffset - curFileOffset, - chunkedPutContext: inReadPlanStepAsSingleObjectExtent.chunkedPutContext, - } - - outReadPlan = append(outReadPlan, outReadPlanStepAsSingleObjectExtent) - - curFileOffset += outReadPlanStepAsSingleObjectExtent.length - remainingLength -= outReadPlanStepAsSingleObjectExtent.length - } - } - - // Append overlapExtentWithLink - - outReadPlan = append(outReadPlan, overlapExtentWithLink) - - curFileOffset += overlapExtentWithLink.length - remainingLength -= overlapExtentWithLink.length - } - - if 0 < remainingLength { - // Append non-overlapped trailing portion of inReadPlanStep in outReadPlan - - if wasMultiObjectReadPlanStep { - outReadPlanStepAsMultiObjectExtent = &multiObjectExtentStruct{ - fileOffset: curFileOffset, - containerName: inReadPlanStepAsMultiObjectExtent.containerName, - objectName: inReadPlanStepAsMultiObjectExtent.objectName, - objectOffset: inReadPlanStepAsMultiObjectExtent.objectOffset + (curFileOffset - inReadPlanStepAsMultiObjectExtent.fileOffset), - length: remainingLength, - } - - outReadPlan = append(outReadPlan, outReadPlanStepAsMultiObjectExtent) - } else { - outReadPlanStepAsSingleObjectExtent = &singleObjectExtentWithLinkStruct{ - fileOffset: curFileOffset, - objectOffset: inReadPlanStepAsSingleObjectExtent.objectOffset + (curFileOffset - inReadPlanStepAsSingleObjectExtent.fileOffset), - length: remainingLength, - chunkedPutContext: inReadPlanStepAsSingleObjectExtent.chunkedPutContext, - } - - outReadPlan = append(outReadPlan, outReadPlanStepAsSingleObjectExtent) - } - } - } - - // Compute tentative outReadPlanSpan - - if 0 == len(outReadPlan) { - outReadPlanSpan = 0 - } else { - outReadPlanStepAsInterface = outReadPlan[len(outReadPlan)-1] - outReadPlanStepAsMultiObjectExtent, wasMultiObjectReadPlanStep = outReadPlanStepAsInterface.(*multiObjectExtentStruct) - if wasMultiObjectReadPlanStep { - outReadPlanSpan = (outReadPlanStepAsMultiObjectExtent.fileOffset + outReadPlanStepAsMultiObjectExtent.length) - fileOffset - } else { - outReadPlanStepAsSingleObjectExtent = outReadPlanStepAsInterface.(*singleObjectExtentWithLinkStruct) - outReadPlanSpan = (outReadPlanStepAsSingleObjectExtent.fileOffset + outReadPlanStepAsSingleObjectExtent.length) - fileOffset - } - } - - if outReadPlanSpan == length { - return - } - - // inReadPlan was limited by incoming FileSize... can we extend it? - - curFileOffset = fileOffset + outReadPlanSpan - - postExtentIndex, found, err = chunkedPutContext.extentMap.BisectLeft(curFileOffset) - if nil != err { - logFatalf("getReadPlanHelper() couldn't find postExtentIndex [Case 2]: %v", err) - } - if found { - // We know this extent, if it exists, does not overlap - - _, postExtentAsValue, ok, err = chunkedPutContext.extentMap.GetByIndex(postExtentIndex) - if nil != err { - logFatalf("getReadPlanHelper() couldn't find postExtent [Case 1]: %v", err) - } - if !ok { - return - } - - postExtent = postExtentAsValue.(*singleObjectExtentStruct) - } else { - // So this extent, if it exists, must overlap... and possibly extend beyond - - _, postExtentAsValue, ok, err = chunkedPutContext.extentMap.GetByIndex(postExtentIndex) - if nil != err { - logFatalf("getReadPlanHelper() couldn't find postExtent [Case 2]: %v", err) - } - - if ok { - overlapExtent = postExtentAsValue.(*singleObjectExtentStruct) - - if (overlapExtent.fileOffset + overlapExtent.length) > curFileOffset { - // Create a postExtent equivalent to the non-overlapping tail of overlapExtent - - postExtent = &singleObjectExtentStruct{ - fileOffset: curFileOffset, - objectOffset: overlapExtent.objectOffset + (curFileOffset - overlapExtent.fileOffset), - length: overlapExtent.length - (curFileOffset - overlapExtent.fileOffset), - } - } else { - // Create a zero-length postExtent instead - - postExtent = &singleObjectExtentStruct{ - fileOffset: curFileOffset, - objectOffset: 0, - length: 0, - } - } - } else { - // Create a zero-length postExtent instead - - postExtent = &singleObjectExtentStruct{ - fileOffset: curFileOffset, - objectOffset: 0, - length: 0, - } - } - } - - // Now enter a loop until either outReadPlanSpan reaches length or we reach FileSize - // Each loop iteration, postExtent either starts at or after curFileSize (requiring zero-fill) - - for { - if 0 < postExtent.length { - if postExtent.fileOffset > curFileOffset { - // We must "zero-fill" to MIN(postExtent.fileOffset, fileInode.cachedStat.Size) - - if postExtent.fileOffset >= (fileOffset + length) { - // postExtent starts beyond fileOffset+length, so just append zero-fill step & return - outReadPlanStepAsSingleObjectExtent = &singleObjectExtentWithLinkStruct{ - fileOffset: curFileOffset, - objectOffset: 0, - length: length - outReadPlanSpan, - chunkedPutContext: nil, - } - - outReadPlan = append(outReadPlan, outReadPlanStepAsSingleObjectExtent) - outReadPlanSpan = length - - return - } - - // postExtent starts after curFileOffset but before fileOffset+length, so insert zero-fill step first - - outReadPlanStepAsSingleObjectExtent = &singleObjectExtentWithLinkStruct{ - fileOffset: curFileOffset, - objectOffset: 0, - length: postExtent.fileOffset - curFileOffset, - chunkedPutContext: nil, - } - - outReadPlan = append(outReadPlan, outReadPlanStepAsSingleObjectExtent) - - curFileOffset += outReadPlanStepAsSingleObjectExtent.length - outReadPlanSpan += outReadPlanStepAsSingleObjectExtent.length - } - - // Now append a step for some or all of postExtent - - if (postExtent.fileOffset + postExtent.length) >= (fileOffset + length) { - // postExtent will take us to (and beyond) fileOffset+length, so insert proper portion & return - - outReadPlanStepAsSingleObjectExtent = &singleObjectExtentWithLinkStruct{ - fileOffset: postExtent.fileOffset, - objectOffset: postExtent.objectOffset, - length: (fileOffset + length) - postExtent.fileOffset, - chunkedPutContext: chunkedPutContext, - } - - outReadPlan = append(outReadPlan, outReadPlanStepAsSingleObjectExtent) - outReadPlanSpan = length - - return - } - - // The entire postExtent will "fit"... and not exhaust fileOffset+length - - outReadPlanStepAsSingleObjectExtent = &singleObjectExtentWithLinkStruct{ - fileOffset: postExtent.fileOffset, - objectOffset: postExtent.objectOffset, - length: postExtent.length, - chunkedPutContext: chunkedPutContext, - } - - outReadPlan = append(outReadPlan, outReadPlanStepAsSingleObjectExtent) - - curFileOffset += outReadPlanStepAsSingleObjectExtent.length - outReadPlanSpan += outReadPlanStepAsSingleObjectExtent.length - } - - // Index to next postExtent - - postExtentIndex++ - - _, postExtentAsValue, ok, err = chunkedPutContext.extentMap.GetByIndex(postExtentIndex) - if nil != err { - logFatalf("getReadPlanHelper() couldn't find postExtent [Case 3]: %v", err) - } - if !ok { - return - } - - postExtent = postExtentAsValue.(*singleObjectExtentStruct) - } -} - -func fetchLogSegmentCacheLine(containerName string, objectName string, offset uint64) (logSegmentCacheElement *logSegmentCacheElementStruct) { - var ( - err error - getRequest *http.Request - logSegmentCacheElementGetEndime time.Time - logSegmentCacheElementGetStartTime time.Time - logSegmentCacheElementKey logSegmentCacheElementKeyStruct - logSegmentCacheElementToEvict *logSegmentCacheElementStruct - logSegmentCacheElementToEvictKey logSegmentCacheElementKeyStruct - logSegmentCacheElementToEvictLRUElement *list.Element - logSegmentEnd uint64 - logSegmentStart uint64 - ok bool - statusCode int - swiftStorageURL string - url string - ) - - // Compute Key for LogSegment Cache lookup - - logSegmentCacheElementKey.logSegmentNumber, err = strconv.ParseUint(objectName, 16, 64) - if nil != err { - logWarnf("fetchLogSegmentCacheLine() passed un-parseable objectName: \"%s\" (err: %v)", objectName, err) - logSegmentCacheElement = nil - return - } - - logSegmentCacheElementKey.cacheLineTag = offset / globals.config.ReadCacheLineSize - - // Perform lookup - - globals.Lock() - - logSegmentCacheElement, ok = globals.logSegmentCacheMap[logSegmentCacheElementKey] - - if ok { - // Found it... so indicate cache hit & move it to MRU end of LRU - - _ = atomic.AddUint64(&globals.metrics.ReadCacheHits, 1) - - globals.logSegmentCacheLRU.MoveToBack(logSegmentCacheElement.cacheLRUElement) - - globals.Unlock() - - // It may not be here yet, so wait for it to be - - logSegmentCacheElement.Wait() - - // GET may have failed... LogSegment disappeared - - if logSegmentCacheElementStateGetFailed == logSegmentCacheElement.state { - logInfof("LogSegment %s/%s at Offset 0x%016X not available", containerName, objectName, logSegmentCacheElementKey.cacheLineTag*globals.config.ReadCacheLineSize) - logSegmentCacheElement = nil - } - - // In any case, we can now return - - return - } - - // Cache miss - - _ = atomic.AddUint64(&globals.metrics.ReadCacheMisses, 1) - - // Make room for new LogSegment Cache Line if necessary - - for uint64(globals.logSegmentCacheLRU.Len()) >= globals.config.ReadCacheLineCount { - logSegmentCacheElementToEvictLRUElement = globals.logSegmentCacheLRU.Front() - logSegmentCacheElementToEvict = logSegmentCacheElementToEvictLRUElement.Value.(*logSegmentCacheElementStruct) - logSegmentCacheElementToEvictKey.logSegmentNumber, err = strconv.ParseUint(logSegmentCacheElementToEvict.objectName, 16, 64) - logSegmentCacheElementToEvictKey.cacheLineTag = logSegmentCacheElementToEvict.startingOffset / globals.config.ReadCacheLineSize - if nil != err { - logFatalf("fetchLogSegmentCacheLine() evicting hit un-parseable objectName: \"%s\" (err: %v)", logSegmentCacheElementToEvict.objectName, err) - } - delete(globals.logSegmentCacheMap, logSegmentCacheElementToEvictKey) - globals.logSegmentCacheLRU.Remove(logSegmentCacheElementToEvictLRUElement) - } - - // Create new LogSegment Cache Line...as yet unfilled - - logSegmentCacheElement = &logSegmentCacheElementStruct{ - state: logSegmentCacheElementStateGetIssued, - containerName: containerName, - objectName: objectName, - startingOffset: logSegmentCacheElementKey.cacheLineTag * globals.config.ReadCacheLineSize, - } - - logSegmentCacheElement.Add(1) - - // Make it findable while we fill it (to avoid multiple copies) - - globals.logSegmentCacheMap[logSegmentCacheElementKey] = logSegmentCacheElement - logSegmentCacheElement.cacheLRUElement = globals.logSegmentCacheLRU.PushBack(logSegmentCacheElement) - - globals.Unlock() - - // Issue GET for it - - swiftStorageURL = fetchStorageURL() - - url = swiftStorageURL + "/" + containerName + "/" + objectName - - logSegmentStart = logSegmentCacheElementKey.cacheLineTag * globals.config.ReadCacheLineSize - logSegmentEnd = logSegmentStart + globals.config.ReadCacheLineSize - 1 - - getRequest, err = http.NewRequest(http.MethodGet, url, nil) - if nil != err { - logFatalf("unable to create GET http.Request (,%s,): %v", url) - } - - getRequest.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", logSegmentStart, logSegmentEnd)) - - logSegmentCacheElementGetStartTime = time.Now() - - _, logSegmentCacheElement.buf, ok, statusCode = doHTTPRequest(getRequest, http.StatusOK, http.StatusPartialContent) - if !ok { - logFatalf("fetchLogSegmentCacheLine() failed with unexpected statusCode: %v", statusCode) - } - - logSegmentCacheElementGetEndime = time.Now() - - globals.Lock() - - if ok { - logSegmentCacheElement.state = logSegmentCacheElementStateGetSuccessful - - globals.stats.LogSegmentGetUsec.Add(uint64(logSegmentCacheElementGetEndime.Sub(logSegmentCacheElementGetStartTime) / time.Microsecond)) - } else { - logSegmentCacheElement.state = logSegmentCacheElementStateGetFailed - - // Remove it from the LogSegment Cache as well - - delete(globals.logSegmentCacheMap, logSegmentCacheElementKey) - globals.logSegmentCacheLRU.Remove(logSegmentCacheElement.cacheLRUElement) - } - - globals.Unlock() - - // Signal any (other) waiters GET completed (either successfully or not) before returning - - logSegmentCacheElement.Done() - - return -} - -func (chunkedPutContext *chunkedPutContextStruct) mergeSingleObjectExtent(newExtent *singleObjectExtentStruct) { - var ( - curExtent *singleObjectExtentStruct - curExtentAsValue sortedmap.Value - curExtentIndex int - err error - extentMapLen int - found bool - ok bool - postLength uint64 - preLength uint64 - splitExtent *singleObjectExtentStruct - ) - - // See if we can simply extend last element - - extentMapLen, err = chunkedPutContext.extentMap.Len() - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to Len(): %v", err) - } - - if 0 < extentMapLen { - _, curExtentAsValue, _, err = chunkedPutContext.extentMap.GetByIndex(extentMapLen - 1) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to GetByIndex() [Case 1]: %v", err) - } - curExtent = curExtentAsValue.(*singleObjectExtentStruct) - - if (curExtent.fileOffset + curExtent.length) == newExtent.fileOffset { - if (curExtent.objectOffset + curExtent.length) == newExtent.objectOffset { - // Simply extend curExtent (coalescing newExtent into it) - - curExtent.length += newExtent.length - - return - } - } - } - - // See if newExtent collides with a first curExtent - - curExtentIndex, found, err = chunkedPutContext.extentMap.BisectLeft(newExtent.fileOffset) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to BisectLeft(): %v", err) - } - - if found { - // curExtent exists and starts precisely at newExtent.fileOffset... fully overlapped by newExtent? - - _, curExtentAsValue, _, err = chunkedPutContext.extentMap.GetByIndex(curExtentIndex) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to GetByIndex() [Case 2]: %v", err) - } - curExtent = curExtentAsValue.(*singleObjectExtentStruct) - - if (curExtent.fileOffset + curExtent.length) <= (newExtent.fileOffset + newExtent.length) { - // curExtent fully overlapped by newExtent... just drop it - - _, err = chunkedPutContext.extentMap.DeleteByIndex(curExtentIndex) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to DeleteByIndex() [Case 1]: %v", err) - } - - // curExtentIndex left pointing to subsequent extent (if any) for loop below - } else { - // curExtent is overlapped "on the left" by newExtent... so tuncate and move curExtent - - postLength = (curExtent.fileOffset + curExtent.length) - (newExtent.fileOffset + newExtent.length) - - _, err = chunkedPutContext.extentMap.DeleteByIndex(curExtentIndex) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to DeleteByIndex() curExtent [Case 2]: %v", err) - } - - splitExtent = &singleObjectExtentStruct{ - fileOffset: newExtent.fileOffset + newExtent.length, - objectOffset: curExtent.objectOffset + preLength + newExtent.length, - length: postLength, - } - - _, err = chunkedPutContext.extentMap.Put(splitExtent.fileOffset, splitExtent) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to Put() splitExtent [Case 1]: %v", err) - } - - // From here, we know we can just insert newExtent and we are done - - _, err = chunkedPutContext.extentMap.Put(newExtent.fileOffset, newExtent) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to Put() newExtent [Case 1]: %v", err) - } - - return - } - } else { // !found - if 0 > curExtentIndex { - // curExtent does not exist (so cannot overlap)... so set curExtentIndex to point to first extent (if any) for loop below - - curExtentIndex = 0 - } else { // 0 <= curExtentIndex - // curExtent exists and starts strictly before newExtent.fileOffset... any overlap with newExtent? - - _, curExtentAsValue, _, err = chunkedPutContext.extentMap.GetByIndex(curExtentIndex) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to GetByIndex() [Case 3]: %v", err) - } - curExtent = curExtentAsValue.(*singleObjectExtentStruct) - - if (curExtent.fileOffset + curExtent.length) > newExtent.fileOffset { - // curExtent definitely collides with newExtent... can we just truncate or do we need to split - - preLength = newExtent.fileOffset - curExtent.fileOffset - - if (curExtent.fileOffset + curExtent.length) <= (newExtent.fileOffset + newExtent.length) { - // curExtent ends at or before newExtent ends... so simply truncate curExtent - - curExtent.length = preLength - - // Set curExtentIndex to point to following extent (if any) for loop below - - curExtentIndex++ - } else { - // curExtent is overlapped "in the middle" by newExtent... so split curExtent "around" newExtent - - postLength = (curExtent.fileOffset + curExtent.length) - (newExtent.fileOffset + newExtent.length) - - curExtent.length = preLength - - splitExtent = &singleObjectExtentStruct{ - fileOffset: newExtent.fileOffset + newExtent.length, - objectOffset: curExtent.objectOffset + preLength + newExtent.length, - length: postLength, - } - - _, err = chunkedPutContext.extentMap.Put(splitExtent.fileOffset, splitExtent) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to Put() splitExtent [Case 2]: %v", err) - } - - // From here, we know we can just insert newExtent and we are done - - _, err = chunkedPutContext.extentMap.Put(newExtent.fileOffset, newExtent) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to Put() newExtent [Case 2]: %v", err) - } - - return - } - } else { - // curExtent does not overlap newExtent... set curExtentIndex to point to following extent (in any) for loop below - - curExtentIndex++ - } - } - } - - // At this point, the special case of the first extent starting at or before newExtent has been - // cleared from overlapping with newExtent... so now we have to loop from curExtentIndex looking - // for additional extents to either delete entirely or truncate "on the left" in order to ensure - // that no other extents overlap with newExtent - - for { - _, curExtentAsValue, ok, err = chunkedPutContext.extentMap.GetByIndex(curExtentIndex) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to GetByIndex() [Case 4]: %v", err) - } - - if !ok { - // No more extents, so we know we are done removing overlapped extents - - break - } - - curExtent = curExtentAsValue.(*singleObjectExtentStruct) - - if curExtent.fileOffset >= (newExtent.fileOffset + newExtent.length) { - // This and all subsequent extents are "beyond" newExtent so cannot overlap - - break - } - - if (curExtent.fileOffset + curExtent.length) <= (newExtent.fileOffset + newExtent.length) { - // curExtent completely overlapped by newExtent... so simply delete it - - _, err = chunkedPutContext.extentMap.DeleteByIndex(curExtentIndex) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to DeleteByIndex() [Case 3]: %v", err) - } - - // curExtentIndex left pointing to subsequent extent (if any) for next loop iteration - } else { - // curExtent is overlapped "on the left" by newExtent... so truncate and move curExtent - - postLength = (curExtent.fileOffset + curExtent.length) - (newExtent.fileOffset + newExtent.length) - - _, err = chunkedPutContext.extentMap.DeleteByIndex(curExtentIndex) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to DeleteByIndex() curExtent [Case 4]: %v", err) - } - - splitExtent = &singleObjectExtentStruct{ - fileOffset: newExtent.fileOffset + newExtent.length, - objectOffset: curExtent.objectOffset + (curExtent.length - postLength), - length: postLength, - } - - _, err = chunkedPutContext.extentMap.Put(splitExtent.fileOffset, splitExtent) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to Put() splitExtent [Case 3]: %v", err) - } - - // From here, we know we can just insert newExtent and we are done - - _, err = chunkedPutContext.extentMap.Put(newExtent.fileOffset, newExtent) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to Put() newExtent [Case 3]: %v", err) - } - - return - } - } - - // Having ensured there are no overlapping extents, it is safe to insert newExtent - - _, err = chunkedPutContext.extentMap.Put(newExtent.fileOffset, newExtent) - if nil != err { - logFatalf("mergeSingleObjectExtent() failed to Put() newExtent [Case 4]: %v", err) - } -} - -func pruneExtentMap(extentMap sortedmap.LLRBTree, newSize uint64) { - var ( - err error - extentAsMultiObjectExtent *multiObjectExtentStruct - extentAsSingleObjectExtent *singleObjectExtentStruct - extentAsSingleObjectExtentWithLink *singleObjectExtentWithLinkStruct - extentAsValue sortedmap.Value - index int - ok bool - ) - - // Nothing to do if extentMap hasn't been populated yet - - if nil == extentMap { - return - } - - // First, destroy any extents starting at or beyond newSize - - index, _, err = extentMap.BisectRight(newSize) - if nil != err { - logFatalf("extentMap.BisectLeft() failed: %v", err) - } - - ok = true - for ok { - ok, err = extentMap.DeleteByIndex(index) - if nil != err { - logFatalf("extentMap.DeleteByIndex(index+1) failed: %v", err) - } - } - - // Next, back up and look at (new) tailing extent - - index-- - - if 0 > index { - // No extents remain in extentmap, so just return empty extentMap - - return - } - - // See if trailing extent map entries need to be truncated - - _, extentAsValue, _, err = extentMap.GetByIndex(index) - if nil != err { - logFatalf("extentMap.GetByIndex(index) failed: %v", err) - } - - extentAsMultiObjectExtent, ok = extentAsValue.(*multiObjectExtentStruct) - if ok { - if (extentAsMultiObjectExtent.fileOffset + extentAsMultiObjectExtent.length) > newSize { - extentAsMultiObjectExtent.length = newSize - extentAsMultiObjectExtent.fileOffset - } - } else { - extentAsSingleObjectExtent, ok = extentAsValue.(*singleObjectExtentStruct) - if ok { - if (extentAsSingleObjectExtent.fileOffset + extentAsSingleObjectExtent.length) > newSize { - extentAsSingleObjectExtent.length = newSize - extentAsSingleObjectExtent.fileOffset - } - } else { - extentAsSingleObjectExtentWithLink, ok = extentAsValue.(*singleObjectExtentWithLinkStruct) - if ok { - if (extentAsSingleObjectExtentWithLink.fileOffset + extentAsSingleObjectExtentWithLink.length) > newSize { - extentAsSingleObjectExtentWithLink.length = newSize - extentAsSingleObjectExtentWithLink.fileOffset - } - } else { - logFatalf("extentAsValue.(*{multi|single|single}ObjectExtent{||WithLink}Struct) returned !ok") - } - } - } - - return -} - -// DumpKey formats the Key (multiObjectExtentStruct.fileOffset) for fileInodeStruct.ExtentMap -func (fileInode *fileInodeStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - var ( - keyAsU64 uint64 - ok bool - ) - - keyAsU64, ok = key.(uint64) - if ok { - keyAsString = fmt.Sprintf("0x%016X", keyAsU64) - } else { - err = fmt.Errorf("Failure of *fileInodeStruct.DumpKey(%v)", key) - } - - return -} - -// DumpKey formats the Value (multiObjectExtentStruct) for fileInodeStruct.ExtentMap -func (fileInode *fileInodeStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - var ( - ok bool - valueAsMultiObjextExtentStruct *multiObjectExtentStruct - ) - - valueAsMultiObjextExtentStruct, ok = value.(*multiObjectExtentStruct) - if ok { - valueAsString = fmt.Sprintf( - "{fileOffset:0x%016X,containerName:%s,objectName:%s,objectOffset:0x%016X,length:0x%016X}", - valueAsMultiObjextExtentStruct.fileOffset, - valueAsMultiObjextExtentStruct.containerName, - valueAsMultiObjextExtentStruct.objectName, - valueAsMultiObjextExtentStruct.objectOffset, - valueAsMultiObjextExtentStruct.length) - } else { - err = fmt.Errorf("Failure of *fileInodeStruct.DumpValue(%v)", value) - } - - return -} - -// DumpKey formats the Key (singleObjectExtentStruct.fileOffset) for chunkedPutContextStruct.ExtentMap -func (chunkedPutContext *chunkedPutContextStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - var ( - keyAsU64 uint64 - ok bool - ) - - keyAsU64, ok = key.(uint64) - if ok { - keyAsString = fmt.Sprintf("0x%016X", keyAsU64) - } else { - err = fmt.Errorf("Failure of *chunkedPutContextStruct.DumpKey(%v)", key) - } - - return -} - -// DumpKey formats the Value (singleObjectExtentStruct) for chunkedPutContextStruct.ExtentMap -func (chunkedPutContext *chunkedPutContextStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - var ( - ok bool - valueAsSingleObjectExtentStruct *singleObjectExtentStruct - ) - - valueAsSingleObjectExtentStruct, ok = value.(*singleObjectExtentStruct) - if ok { - valueAsString = fmt.Sprintf( - "{fileOffset:0x%016X,objectOffset:0x%016X,length:0x%016X}", - valueAsSingleObjectExtentStruct.fileOffset, - valueAsSingleObjectExtentStruct.objectOffset, - valueAsSingleObjectExtentStruct.length) - } else { - err = fmt.Errorf("Failure of *chunkedPutContextStruct.DumpValue(%v)", value) - } - - return -} - -func dumpExtentMap(extentMap sortedmap.LLRBTree) { - var ( - err error - extentAsMultiObjectExtent *multiObjectExtentStruct - extentAsSingleObjectExtent *singleObjectExtentStruct - extentAsSingleObjectExtentWithLink *singleObjectExtentWithLinkStruct - extentAsValue sortedmap.Value - extentIndex int - extentMapLen int - ok bool - ) - - extentMapLen, err = extentMap.Len() - if nil != err { - logFatalf("dumpExtentMap() doing extentMap.Len() failed: %v", err) - } - - for extentIndex = 0; extentIndex < extentMapLen; extentIndex++ { - _, extentAsValue, ok, err = extentMap.GetByIndex(extentIndex) - if nil != err { - logFatalf("dumpExtentMap() doing extentMap.GetByIndex() failed: %v", err) - } - if !ok { - logFatalf("dumpExtentMap() doing extentMap.GetByIndex() returned !ok") - } - - extentAsMultiObjectExtent, ok = extentAsValue.(*multiObjectExtentStruct) - if ok { - logInfof(" MultiObjectExtent: %+v", extentAsMultiObjectExtent) - } else { - extentAsSingleObjectExtent, ok = extentAsValue.(*singleObjectExtentStruct) - if ok { - logInfof(" SingleObjectExtent: %+v", extentAsSingleObjectExtent) - } else { - extentAsSingleObjectExtentWithLink, ok = extentAsValue.(*singleObjectExtentWithLinkStruct) - if ok { - logInfof(" SingleObjectExtentWithLink: %+v [containerName:%s objectName:%s]", extentAsSingleObjectExtentWithLink, extentAsSingleObjectExtentWithLink.chunkedPutContext.containerName, extentAsSingleObjectExtentWithLink.chunkedPutContext.objectName) - } else { - logFatalf("dumpExtentMap() doing extentAsValue.(*{multi|single|single}ObjectExtent{||WithLink}Struct) returned !ok") - } - } - } - } -} - -func (fileInode *fileInodeStruct) dumpExtentMaps() { - var ( - chunkedPutContext *chunkedPutContextStruct - chunkedPutContextElement *list.Element - chunkedPutContextIndex uint64 - ok bool - ) - - logInfof("FileInode @%p ExtentMap:", fileInode) - - dumpExtentMap(fileInode.extentMap) - - chunkedPutContextIndex = 0 - chunkedPutContextElement = fileInode.chunkedPutList.Front() - - for nil != chunkedPutContextElement { - chunkedPutContext, ok = chunkedPutContextElement.Value.(*chunkedPutContextStruct) - if !ok { - logFatalf("dumpExtentMaps() doing chunkedPutContextElement.Value.(*chunkedPutContextStruct) returned !ok") - } - - logInfof(" ChunkedPutContext #%v @%p ExtentMap [containerName:%s objectName:%s]:", chunkedPutContextIndex, chunkedPutContext, chunkedPutContext.containerName, chunkedPutContext.objectName) - - dumpExtentMap(chunkedPutContext.extentMap) - - chunkedPutContextIndex++ - chunkedPutContextElement = chunkedPutContextElement.Next() - } -} diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go deleted file mode 100644 index 00c64b52..00000000 --- a/pfsagentd/fission.go +++ /dev/null @@ -1,2575 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "container/list" - "fmt" - "log" - "os" - "sync/atomic" - "syscall" - "time" - - "github.com/NVIDIA/fission" - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/jrpcfs" -) - -const ( - attrBlockSize = uint32(512) - - initOutFlagsMaskReadDirPlusDisabled = uint32(0) | - fission.InitFlagsAsyncRead | - fission.InitFlagsFileOps | - fission.InitFlagsAtomicOTrunc | - fission.InitFlagsBigWrites | - fission.InitFlagsAutoInvalData | - fission.InitFlagsParallelDirops | - fission.InitFlagsMaxPages | - fission.InitFlagsExplicitInvalData - - initOutFlagsMaskReadDirPlusEnabled = initOutFlagsMaskReadDirPlusDisabled | - fission.InitFlagsDoReadDirPlus | - fission.InitFlagsReaddirplusAuto - - pfsagentFuseSubtype = "PFSAgent" -) - -func performMountFUSE() { - var ( - err error - ) - - globals.fissionVolume = fission.NewVolume( - globals.config.FUSEVolumeName, // volumeName string - globals.config.FUSEMountPointPath, // mountpointDirPath string - pfsagentFuseSubtype, // fuseSubtype string - globals.config.FUSEMaxWrite, // initOutMaxWrite uint32 - globals.config.FUSEAllowOther, // allowOther bool - &globals, // callbacks fission.Callbacks - newLogger(), // logger *log.Logger - globals.fissionErrChan) // errChan chan error - - err = globals.fissionVolume.DoMount() - if nil != err { - log.Fatalf("fissionVolume.DoMount() failed: %v", err) - } -} - -func performUnmountFUSE() { - var ( - err error - ) - - err = globals.fissionVolume.DoUnmount() - if nil != err { - log.Fatalf("fissionVolume.DoUnmount() failed: %v", err) - } -} - -func convertErrToErrno(err error, defaultErrno syscall.Errno) (errno syscall.Errno) { - var ( - convertErr error - possibleErrno syscall.Errno - ) - - _, convertErr = fmt.Sscanf(err.Error(), "errno: %v", &possibleErrno) - if nil == convertErr { - errno = possibleErrno - } else { - errno = defaultErrno - } - - return -} - -func fixAttrSizes(attr *fission.Attr) { - if syscall.S_IFREG == (attr.Mode & syscall.S_IFMT) { - attr.Blocks = attr.Size + (uint64(attrBlockSize) - 1) - attr.Blocks /= uint64(attrBlockSize) - attr.BlkSize = attrBlockSize - } else { - attr.Size = 0 - attr.Blocks = 0 - attr.BlkSize = 0 - } -} - -func nsToUnixTime(ns uint64) (sec uint64, nsec uint32) { - sec = ns / 1e9 - nsec = uint32(ns - (sec * 1e9)) - return -} - -func unixTimeToNs(sec uint64, nsec uint32) (ns uint64) { - ns = (sec * 1e9) + uint64(nsec) - return -} - -func (fileInode *fileInodeStruct) ensureCachedStatPopulatedWhileLocked() (err error) { - var ( - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - ) - - if nil == fileInode.cachedStat { - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(fileInode.InodeNumber), - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - return - } - - fileInode.cachedStat = getStatReply - } - - err = nil - return -} - -func (dummy *globalsStruct) DoLookup(inHeader *fission.InHeader, lookupIn *fission.LookupIn) (lookupOut *fission.LookupOut, errno syscall.Errno) { - var ( - aTimeNSec uint32 - aTimeSec uint64 - cTimeNSec uint32 - cTimeSec uint64 - err error - fileInode *fileInodeStruct - lookupPlusReply *jrpcfs.LookupPlusReply - lookupPlusRequest *jrpcfs.LookupPlusRequest - mTimeNSec uint32 - mTimeSec uint64 - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoLookup_calls, 1) - - lookupPlusRequest = &jrpcfs.LookupPlusRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - Basename: string(lookupIn.Name[:]), - } - - lookupPlusReply = &jrpcfs.LookupPlusReply{} - - err = globals.retryRPCClient.Send("RpcLookupPlus", lookupPlusRequest, lookupPlusReply) - if nil != err { - errno = convertErrToErrno(err, syscall.ENOENT) - return - } - - fileInode = lockInodeIfExclusiveLeaseGranted(inode.InodeNumber(lookupPlusReply.InodeNumber)) - if nil != fileInode { - if nil == fileInode.cachedStat { - // Might as well cache the Stat portion of lookupPlusReply in fileInode.cachedStat - // - // Note: This is a convenient, and unlikely, optimization that won't be done in - // the SharedLease case... but that is also an unlikely case. - - fileInode.cachedStat = &jrpcfs.StatStruct{ - CTimeNs: lookupPlusReply.CTimeNs, - CRTimeNs: lookupPlusReply.CRTimeNs, - MTimeNs: lookupPlusReply.MTimeNs, - ATimeNs: lookupPlusReply.ATimeNs, - Size: lookupPlusReply.Size, - NumLinks: lookupPlusReply.NumLinks, - StatInodeNumber: lookupPlusReply.StatInodeNumber, - FileMode: lookupPlusReply.FileMode, - UserID: lookupPlusReply.UserID, - GroupID: lookupPlusReply.GroupID, - } - } else { - // Update lookupPlusReply from fileInode.cachedStat - - lookupPlusReply.CTimeNs = fileInode.cachedStat.CTimeNs - lookupPlusReply.CRTimeNs = fileInode.cachedStat.CRTimeNs - lookupPlusReply.MTimeNs = fileInode.cachedStat.MTimeNs - lookupPlusReply.ATimeNs = fileInode.cachedStat.ATimeNs - lookupPlusReply.Size = fileInode.cachedStat.Size - lookupPlusReply.NumLinks = fileInode.cachedStat.NumLinks - lookupPlusReply.StatInodeNumber = fileInode.cachedStat.StatInodeNumber - lookupPlusReply.FileMode = fileInode.cachedStat.FileMode - lookupPlusReply.UserID = fileInode.cachedStat.UserID - lookupPlusReply.GroupID = fileInode.cachedStat.GroupID - } - - fileInode.unlock(false) - } - - aTimeSec, aTimeNSec = nsToUnixTime(lookupPlusReply.ATimeNs) - mTimeSec, mTimeNSec = nsToUnixTime(lookupPlusReply.MTimeNs) - cTimeSec, cTimeNSec = nsToUnixTime(lookupPlusReply.CTimeNs) - - lookupOut = &fission.LookupOut{ - EntryOut: fission.EntryOut{ - NodeID: uint64(lookupPlusReply.InodeNumber), - Generation: 0, - EntryValidSec: globals.entryValidSec, - AttrValidSec: globals.attrValidSec, - EntryValidNSec: globals.entryValidNSec, - AttrValidNSec: globals.attrValidNSec, - Attr: fission.Attr{ - Ino: uint64(lookupPlusReply.InodeNumber), - Size: lookupPlusReply.Size, - Blocks: 0, // fixAttrSizes() will compute this - ATimeSec: aTimeSec, - MTimeSec: mTimeSec, - CTimeSec: cTimeSec, - ATimeNSec: aTimeNSec, - MTimeNSec: mTimeNSec, - CTimeNSec: cTimeNSec, - Mode: lookupPlusReply.FileMode, - NLink: uint32(lookupPlusReply.NumLinks), - UID: lookupPlusReply.UserID, - GID: lookupPlusReply.GroupID, - RDev: 0, - BlkSize: 0, // fixAttrSizes() will set this - Padding: 0, - }, - }, - } - - fixAttrSizes(&lookupOut.EntryOut.Attr) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoForget(inHeader *fission.InHeader, forgetIn *fission.ForgetIn) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoForget_calls, 1) - return -} - -func (dummy *globalsStruct) DoGetAttr(inHeader *fission.InHeader, getAttrIn *fission.GetAttrIn) (getAttrOut *fission.GetAttrOut, errno syscall.Errno) { - var ( - aTimeNSec uint32 - aTimeSec uint64 - cTimeNSec uint32 - cTimeSec uint64 - err error - fileInode *fileInodeStruct - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - mTimeNSec uint32 - mTimeSec uint64 - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoGetAttr_calls, 1) - - fileInode = lockInodeWithSharedLease(inode.InodeNumber(inHeader.NodeID)) - - if nil == fileInode.cachedStat { - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - fileInode.cachedStat = getStatReply - } - - aTimeSec, aTimeNSec = nsToUnixTime(fileInode.cachedStat.ATimeNs) - mTimeSec, mTimeNSec = nsToUnixTime(fileInode.cachedStat.MTimeNs) - cTimeSec, cTimeNSec = nsToUnixTime(fileInode.cachedStat.CTimeNs) - - getAttrOut = &fission.GetAttrOut{ - AttrValidSec: globals.attrValidSec, - AttrValidNSec: globals.attrValidNSec, - Dummy: 0, - Attr: fission.Attr{ - Ino: inHeader.NodeID, - Size: fileInode.cachedStat.Size, - Blocks: 0, // fixAttrSizes() will compute this - ATimeSec: aTimeSec, - MTimeSec: mTimeSec, - CTimeSec: cTimeSec, - ATimeNSec: aTimeNSec, - MTimeNSec: mTimeNSec, - CTimeNSec: cTimeNSec, - Mode: fileInode.cachedStat.FileMode, - NLink: uint32(fileInode.cachedStat.NumLinks), - UID: fileInode.cachedStat.UserID, - GID: fileInode.cachedStat.GroupID, - RDev: 0, - BlkSize: 0, // fixAttrSizes() will set this - Padding: 0, - }, - } - - fileInode.unlock(false) - - fixAttrSizes(&getAttrOut.Attr) - - errno = 0 - return -} - -func setMode(nodeID uint64, mode uint32) (errno syscall.Errno) { - var ( - chmodReply *jrpcfs.Reply - chmodRequest *jrpcfs.ChmodRequest - err error - ) - - chmodRequest = &jrpcfs.ChmodRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(nodeID), - }, - FileMode: mode & uint32(os.ModePerm), - } - - chmodReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcChmod", chmodRequest, chmodReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - errno = 0 - return -} - -func setUIDAndOrGID(nodeID uint64, settingUID bool, uid uint32, settingGID bool, gid uint32) (errno syscall.Errno) { - var ( - chownReply *jrpcfs.Reply - chownRequest *jrpcfs.ChownRequest - err error - ) - - chownRequest = &jrpcfs.ChownRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(nodeID), - }, - } - - if settingUID { - chownRequest.UserID = int32(uid) - } else { - chownRequest.UserID = -1 - } - - if settingGID { - chownRequest.GroupID = int32(gid) - } else { - chownRequest.GroupID = -1 - } - - chownReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcChown", chownRequest, chownReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - errno = 0 - return -} - -func (fileInode *fileInodeStruct) setSize(nodeID uint64, size uint64) (errno syscall.Errno) { - var ( - chunkedPutContext *chunkedPutContextStruct - chunkedPutContextElement *list.Element - err error - ok bool - resizeReply *jrpcfs.Reply - resizeRequest *jrpcfs.ResizeRequest - ) - - fileInode.cachedStat.Size = size - - pruneExtentMap(fileInode.extentMap, size) - - chunkedPutContextElement = fileInode.chunkedPutList.Front() - - for nil != chunkedPutContextElement { - chunkedPutContext, ok = chunkedPutContextElement.Value.(*chunkedPutContextStruct) - if !ok { - logFatalf("chunkedPutContextElement.Value.(*chunkedPutContextStruct) returned !ok") - } - - pruneExtentMap(chunkedPutContext.extentMap, size) - - chunkedPutContextElement = chunkedPutContextElement.Next() - } - - resizeRequest = &jrpcfs.ResizeRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(nodeID), - }, - NewSize: size, - } - - resizeReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcResize", resizeRequest, resizeReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - errno = 0 - return -} - -func setMTimeAndOrATime(nodeID uint64, settingMTime bool, settingMTimeNow bool, mTimeSec uint64, mTimeNSec uint32, settingATime bool, settingATimeNow bool, aTimeSec uint64, aTimeNSec uint32) (errno syscall.Errno) { - var ( - err error - setTimeReply *jrpcfs.Reply - setTimeRequest *jrpcfs.SetTimeRequest - timeNow time.Time - ) - - setTimeRequest = &jrpcfs.SetTimeRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(nodeID), - }, - StatStruct: jrpcfs.StatStruct{ - MTimeNs: uint64(0), // Updated below if settingMTime - ATimeNs: uint64(0), // Updated below if settingATime - }, - } - - timeNow = time.Now() - - if settingMTime { - if settingMTimeNow { - setTimeRequest.MTimeNs = uint64(timeNow.UnixNano()) - } else { - setTimeRequest.MTimeNs = unixTimeToNs(mTimeSec, mTimeNSec) - } - } - if settingATime { - if settingATimeNow { - setTimeRequest.ATimeNs = uint64(timeNow.UnixNano()) - } else { - setTimeRequest.ATimeNs = unixTimeToNs(aTimeSec, aTimeNSec) - } - } - - setTimeReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcSetTime", setTimeRequest, setTimeReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fission.SetAttrIn) (setAttrOut *fission.SetAttrOut, errno syscall.Errno) { - var ( - aTimeNSec uint32 - aTimeSec uint64 - cTimeNSec uint32 - cTimeSec uint64 - err error - fileInode *fileInodeStruct - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - mTimeNSec uint32 - mTimeSec uint64 - settingATime bool - settingATimeNow bool - settingGID bool - settingMode bool - settingMTime bool - settingMTimeAndOrATime bool - settingMTimeNow bool - settingSize bool - settingUID bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoSetAttr_calls, 1) - - if setAttrIn.Valid != (setAttrIn.Valid & (fission.SetAttrInValidMode | fission.SetAttrInValidUID | fission.SetAttrInValidGID | fission.SetAttrInValidSize | fission.SetAttrInValidATime | fission.SetAttrInValidMTime | fission.SetAttrInValidFH | fission.SetAttrInValidATimeNow | fission.SetAttrInValidMTimeNow | fission.SetAttrInValidLockOwner)) { - errno = syscall.ENOSYS - return - } - - settingMode = (0 != (setAttrIn.Valid & fission.SetAttrInValidMode)) - - settingUID = (0 != (setAttrIn.Valid & fission.SetAttrInValidUID)) - settingGID = (0 != (setAttrIn.Valid & fission.SetAttrInValidGID)) - - settingSize = (0 != (setAttrIn.Valid & fission.SetAttrInValidSize)) - - settingMTime = (0 != (setAttrIn.Valid & fission.SetAttrInValidMTime)) || (0 != (setAttrIn.Valid & fission.SetAttrInValidMTimeNow)) - settingATime = (0 != (setAttrIn.Valid & fission.SetAttrInValidATime)) || (0 != (setAttrIn.Valid & fission.SetAttrInValidATimeNow)) - - settingMTimeNow = settingMTime && (0 != (setAttrIn.Valid & fission.SetAttrInValidMTimeNow)) - settingATimeNow = settingATime && (0 != (setAttrIn.Valid & fission.SetAttrInValidATimeNow)) - - settingMTimeAndOrATime = settingATime || settingMTime - - // TODO: Verify we can accept but ignore fission.SetAttrInValidFH in setAttrIn.Valid - // TODO: Verify we can accept but ignore fission.SetAttrInValidLockOwner in setAttrIn.Valid - - // Now perform requested setAttrIn.Mode operations - - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(inHeader.NodeID)) - - fileInode.doFlushIfNecessary() - - err = fileInode.ensureCachedStatPopulatedWhileLocked() - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - if settingMode { - errno = setMode(inHeader.NodeID, setAttrIn.Mode) - if 0 != errno { - fileInode.unlock(true) - return - } - } - - if settingUID || settingGID { - errno = setUIDAndOrGID(inHeader.NodeID, settingUID, setAttrIn.UID, settingGID, setAttrIn.GID) - if 0 != errno { - fileInode.unlock(true) - return - } - } - - if settingSize { - errno = fileInode.setSize(inHeader.NodeID, setAttrIn.Size) - if 0 != errno { - fileInode.unlock(true) - return - } - } - - if settingMTimeAndOrATime { - errno = setMTimeAndOrATime(inHeader.NodeID, settingMTime, settingMTimeNow, setAttrIn.MTimeSec, setAttrIn.MTimeNSec, settingATime, settingATimeNow, setAttrIn.ATimeSec, setAttrIn.ATimeNSec) - if 0 != errno { - fileInode.unlock(true) - return - } - } - - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - fileInode.cachedStat = getStatReply - - aTimeSec, aTimeNSec = nsToUnixTime(fileInode.cachedStat.ATimeNs) - mTimeSec, mTimeNSec = nsToUnixTime(fileInode.cachedStat.MTimeNs) - cTimeSec, cTimeNSec = nsToUnixTime(fileInode.cachedStat.CTimeNs) - - setAttrOut = &fission.SetAttrOut{ - AttrValidSec: globals.attrValidSec, - AttrValidNSec: globals.attrValidNSec, - Dummy: 0, - Attr: fission.Attr{ - Ino: inHeader.NodeID, - Size: fileInode.cachedStat.Size, - Blocks: 0, // fixAttrSizes() will compute this - ATimeSec: aTimeSec, - MTimeSec: mTimeSec, - CTimeSec: cTimeSec, - ATimeNSec: aTimeNSec, - MTimeNSec: mTimeNSec, - CTimeNSec: cTimeNSec, - Mode: fileInode.cachedStat.FileMode, - NLink: uint32(fileInode.cachedStat.NumLinks), - UID: fileInode.cachedStat.UserID, - GID: fileInode.cachedStat.GroupID, - RDev: 0, - BlkSize: 0, // fixAttrSizes() will set this - Padding: 0, - }, - } - - fileInode.unlock(false) - - fixAttrSizes(&setAttrOut.Attr) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoReadLink(inHeader *fission.InHeader) (readLinkOut *fission.ReadLinkOut, errno syscall.Errno) { - var ( - err error - readSymlinkReply *jrpcfs.ReadSymlinkReply - readSymlinkRequest *jrpcfs.ReadSymlinkRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoReadLink_calls, 1) - - readSymlinkRequest = &jrpcfs.ReadSymlinkRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - } - - readSymlinkReply = &jrpcfs.ReadSymlinkReply{} - - err = globals.retryRPCClient.Send("RpcReadSymlink", readSymlinkRequest, readSymlinkReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - readLinkOut = &fission.ReadLinkOut{ - Data: []byte(readSymlinkReply.Target), - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoSymLink(inHeader *fission.InHeader, symLinkIn *fission.SymLinkIn) (symLinkOut *fission.SymLinkOut, errno syscall.Errno) { - var ( - aTimeNSec uint32 - aTimeSec uint64 - cTimeNSec uint32 - cTimeSec uint64 - err error - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - mTimeNSec uint32 - mTimeSec uint64 - symlinkReply *jrpcfs.InodeReply - symlinkRequest *jrpcfs.SymlinkRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoSymLink_calls, 1) - - symlinkRequest = &jrpcfs.SymlinkRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - Basename: string(symLinkIn.Name[:]), - Target: string(symLinkIn.Data[:]), - UserID: int32(inHeader.UID), - GroupID: int32(inHeader.GID), - } - - symlinkReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcSymlink", symlinkRequest, symlinkReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: symlinkReply.InodeNumber, - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - aTimeSec, aTimeNSec = nsToUnixTime(getStatReply.ATimeNs) - mTimeSec, mTimeNSec = nsToUnixTime(getStatReply.MTimeNs) - cTimeSec, cTimeNSec = nsToUnixTime(getStatReply.CTimeNs) - - symLinkOut = &fission.SymLinkOut{ - EntryOut: fission.EntryOut{ - NodeID: uint64(symlinkReply.InodeNumber), - Generation: 0, - EntryValidSec: globals.entryValidSec, - AttrValidSec: globals.attrValidSec, - EntryValidNSec: globals.entryValidNSec, - AttrValidNSec: globals.attrValidNSec, - Attr: fission.Attr{ - Ino: uint64(symlinkReply.InodeNumber), - Size: 0, - Blocks: 0, - ATimeSec: aTimeSec, - MTimeSec: mTimeSec, - CTimeSec: cTimeSec, - ATimeNSec: aTimeNSec, - MTimeNSec: mTimeNSec, - CTimeNSec: cTimeNSec, - Mode: getStatReply.FileMode, - NLink: uint32(getStatReply.NumLinks), - UID: getStatReply.UserID, - GID: getStatReply.GroupID, - RDev: 0, - BlkSize: 0, - Padding: 0, - }, - }, - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoMkNod(inHeader *fission.InHeader, mkNodIn *fission.MkNodIn) (mkNodOut *fission.MkNodOut, errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoMkNod_calls, 1) - errno = syscall.ENOSYS - return -} - -func (dummy *globalsStruct) DoMkDir(inHeader *fission.InHeader, mkDirIn *fission.MkDirIn) (mkDirOut *fission.MkDirOut, errno syscall.Errno) { - var ( - aTimeNSec uint32 - aTimeSec uint64 - cTimeNSec uint32 - cTimeSec uint64 - err error - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - mkdirReply *jrpcfs.InodeReply - mkdirRequest *jrpcfs.MkdirRequest - mTimeNSec uint32 - mTimeSec uint64 - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoMkDir_calls, 1) - - mkdirRequest = &jrpcfs.MkdirRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - Basename: string(mkDirIn.Name[:]), - UserID: int32(inHeader.UID), - GroupID: int32(inHeader.GID), - FileMode: mkDirIn.Mode & uint32(os.ModePerm), - } - - mkdirReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcMkdir", mkdirRequest, mkdirReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: mkdirReply.InodeNumber, - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - aTimeSec, aTimeNSec = nsToUnixTime(getStatReply.ATimeNs) - mTimeSec, mTimeNSec = nsToUnixTime(getStatReply.MTimeNs) - cTimeSec, cTimeNSec = nsToUnixTime(getStatReply.CTimeNs) - - mkDirOut = &fission.MkDirOut{ - EntryOut: fission.EntryOut{ - NodeID: uint64(mkdirReply.InodeNumber), - Generation: 0, - EntryValidSec: globals.entryValidSec, - AttrValidSec: globals.attrValidSec, - EntryValidNSec: globals.entryValidNSec, - AttrValidNSec: globals.attrValidNSec, - Attr: fission.Attr{ - Ino: uint64(mkdirReply.InodeNumber), - Size: 0, - Blocks: 0, - ATimeSec: aTimeSec, - MTimeSec: mTimeSec, - CTimeSec: cTimeSec, - ATimeNSec: aTimeNSec, - MTimeNSec: mTimeNSec, - CTimeNSec: cTimeNSec, - Mode: getStatReply.FileMode, - NLink: uint32(getStatReply.NumLinks), - UID: getStatReply.UserID, - GID: getStatReply.GroupID, - RDev: 0, - BlkSize: 0, - Padding: 0, - }, - }, - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoUnlink(inHeader *fission.InHeader, unlinkIn *fission.UnlinkIn) (errno syscall.Errno) { - var ( - err error - fileInode *fileInodeStruct - lookupReply *jrpcfs.InodeReply - lookupRequest *jrpcfs.LookupRequest - unlinkReply *jrpcfs.Reply - unlinkRequest *jrpcfs.UnlinkRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoUnlink_calls, 1) - - lookupRequest = &jrpcfs.LookupRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - Basename: string(unlinkIn.Name[:]), - } - - lookupReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcLookup", lookupRequest, lookupReply) - if nil != err { - errno = convertErrToErrno(err, syscall.ENOENT) - return - } - - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(lookupReply.InodeNumber)) - - // Make sure potentially file inode didn't move before we were able to ExclusiveLease it - - lookupReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcLookup", lookupRequest, lookupReply) - if (nil != err) || (fileInode.InodeNumber != inode.InodeNumber(lookupReply.InodeNumber)) { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.ENOENT) - return - } - - fileInode.doFlushIfNecessary() - - unlinkRequest = &jrpcfs.UnlinkRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - Basename: string(unlinkIn.Name[:]), - } - - unlinkReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcUnlink", unlinkRequest, unlinkReply) - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - fileInode.unlock(false) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoRmDir(inHeader *fission.InHeader, rmDirIn *fission.RmDirIn) (errno syscall.Errno) { - var ( - err error - unlinkReply *jrpcfs.Reply - unlinkRequest *jrpcfs.UnlinkRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoRmDir_calls, 1) - - unlinkRequest = &jrpcfs.UnlinkRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - Basename: string(rmDirIn.Name[:]), - } - - unlinkReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcRmdir", unlinkRequest, unlinkReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoRename(inHeader *fission.InHeader, renameIn *fission.RenameIn) (errno syscall.Errno) { - var ( - destroyReply *jrpcfs.Reply - destroyRequest *jrpcfs.DestroyRequest - err error - fileInode *fileInodeStruct - lookupReply *jrpcfs.InodeReply - lookupRequest *jrpcfs.LookupRequest - moveReply *jrpcfs.MoveReply - moveRequest *jrpcfs.MoveRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoRename_calls, 1) - - lookupRequest = &jrpcfs.LookupRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(renameIn.NewDir), - }, - Basename: string(renameIn.NewName[:]), - } - - lookupReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcLookup", lookupRequest, lookupReply) - if nil == err { - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(lookupReply.InodeNumber)) - - // Make sure potentially file inode didn't move before we were able to ExclusiveLease it - - lookupReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcLookup", lookupRequest, lookupReply) - if (nil != err) || (fileInode.InodeNumber != inode.InodeNumber(lookupReply.InodeNumber)) { - fileInode.unlock(true) - fileInode = nil - } else { - fileInode.doFlushIfNecessary() - } - } else { - fileInode = nil - } - - moveRequest = &jrpcfs.MoveRequest{ - MountID: globals.mountID, - SrcDirInodeNumber: int64(inHeader.NodeID), - SrcBasename: string(renameIn.OldName[:]), - DstDirInodeNumber: int64(renameIn.NewDir), - DstBasename: string(renameIn.NewName[:]), - } - - moveReply = &jrpcfs.MoveReply{} - - err = globals.retryRPCClient.Send("RpcMove", moveRequest, moveReply) - if nil == err { - errno = 0 - } else { - errno = convertErrToErrno(err, syscall.EIO) - } - - if nil != fileInode { - fileInode.unlock(false) - } - - if 0 != moveReply.ToDestroyInodeNumber { - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(moveReply.ToDestroyInodeNumber)) - - fileInode.doFlushIfNecessary() - - destroyRequest = &jrpcfs.DestroyRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: moveReply.ToDestroyInodeNumber, - }, - } - - destroyReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcDestroy", destroyRequest, destroyReply) - if nil != err { - logWarnf("RpcDestroy(InodeHandle: %#v) failed: err", err) - } - - fileInode.unlock(false) - } - - return -} - -func (dummy *globalsStruct) DoLink(inHeader *fission.InHeader, linkIn *fission.LinkIn) (linkOut *fission.LinkOut, errno syscall.Errno) { - var ( - aTimeNSec uint32 - aTimeSec uint64 - cTimeNSec uint32 - cTimeSec uint64 - err error - fileInode *fileInodeStruct - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - linkReply *jrpcfs.Reply - linkRequest *jrpcfs.LinkRequest - mTimeNSec uint32 - mTimeSec uint64 - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoLink_calls, 1) - - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(linkIn.OldNodeID)) - - fileInode.doFlushIfNecessary() - - linkRequest = &jrpcfs.LinkRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - Basename: string(linkIn.Name[:]), - TargetInodeNumber: int64(linkIn.OldNodeID), - } - - linkReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcLink", linkRequest, linkReply) - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(linkIn.OldNodeID), - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - fileInode.cachedStat = getStatReply - - aTimeSec, aTimeNSec = nsToUnixTime(fileInode.cachedStat.ATimeNs) - mTimeSec, mTimeNSec = nsToUnixTime(fileInode.cachedStat.MTimeNs) - cTimeSec, cTimeNSec = nsToUnixTime(fileInode.cachedStat.CTimeNs) - - linkOut = &fission.LinkOut{ - EntryOut: fission.EntryOut{ - NodeID: linkIn.OldNodeID, - Generation: 0, - EntryValidSec: globals.entryValidSec, - AttrValidSec: globals.attrValidSec, - EntryValidNSec: globals.entryValidNSec, - AttrValidNSec: globals.attrValidNSec, - Attr: fission.Attr{ - Ino: linkIn.OldNodeID, - Size: fileInode.cachedStat.Size, - Blocks: 0, // fixAttrSizes() will compute this - ATimeSec: aTimeSec, - MTimeSec: mTimeSec, - CTimeSec: cTimeSec, - ATimeNSec: aTimeNSec, - MTimeNSec: mTimeNSec, - CTimeNSec: cTimeNSec, - Mode: fileInode.cachedStat.FileMode, - NLink: uint32(fileInode.cachedStat.NumLinks), - UID: fileInode.cachedStat.UserID, - GID: fileInode.cachedStat.GroupID, - RDev: 0, - BlkSize: 0, // fixAttrSizes() will set this - Padding: 0, - }, - }, - } - - fileInode.unlock(false) - - fixAttrSizes(&linkOut.EntryOut.Attr) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoOpen(inHeader *fission.InHeader, openIn *fission.OpenIn) (openOut *fission.OpenOut, errno syscall.Errno) { - var ( - err error - fhSet fhSetType - fileInode *fileInodeStruct - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - ok bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoOpen_calls, 1) - - if 0 != (openIn.Flags & fission.FOpenRequestTRUNC) { - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(inHeader.NodeID)) - fileInode.doFlushIfNecessary() - } else { - fileInode = lockInodeWithSharedLease(inode.InodeNumber(inHeader.NodeID)) - } - - if nil == fileInode.cachedStat { - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - fileInode.cachedStat = getStatReply - } - - if syscall.S_IFREG != (fileInode.cachedStat.FileMode & syscall.S_IFMT) { - fileInode.unlock(true) - errno = syscall.EINVAL - return - } - - globals.Lock() - - globals.lastFH++ - - globals.fhToOpenRequestMap[globals.lastFH] = openIn.Flags - globals.fhToInodeNumberMap[globals.lastFH] = inHeader.NodeID - - fhSet, ok = globals.inodeNumberToFHMap[inHeader.NodeID] - if !ok { - fhSet = make(fhSetType) - } - fhSet[globals.lastFH] = struct{}{} - globals.inodeNumberToFHMap[inHeader.NodeID] = fhSet - - openOut = &fission.OpenOut{ - FH: globals.lastFH, - OpenFlags: 0, - Padding: 0, - } - - globals.Unlock() - - if 0 != (openIn.Flags & fission.FOpenRequestTRUNC) { - errno = fileInode.setSize(inHeader.NodeID, 0) - if 0 != errno { - fileInode.unlock(true) - return - } - } - - fileInode.unlock(false) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.ReadIn) (readOut *fission.ReadOut, errno syscall.Errno) { - var ( - curObjectOffset uint64 - err error - fhInodeNumber uint64 - fileInode *fileInodeStruct - fOpenRequestFlags uint32 - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - logSegmentCacheElement *logSegmentCacheElementStruct - logSegmentCacheElementBufEndingPosition uint64 - logSegmentCacheElementBufRemainingLen uint64 - logSegmentCacheElementBufSelectedLen uint64 - logSegmentCacheElementBufStartingPosition uint64 - ok bool - readPlan []interface{} - readPlanSpan uint64 - readPlanStepAsInterface interface{} - readPlanStepAsMultiObjectExtentStruct *multiObjectExtentStruct - readPlanStepAsSingleObjectExtentWithLink *singleObjectExtentWithLinkStruct - readPlanStepRemainingLength uint64 - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoRead_calls, 1) - globals.stats.FUSEDoReadBytes.Add(uint64(readIn.Size)) - - fileInode = lockInodeWithSharedLease(inode.InodeNumber(inHeader.NodeID)) - - if nil == fileInode.cachedStat { - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - fileInode.cachedStat = getStatReply - } - - globals.Lock() - - fOpenRequestFlags, ok = globals.fhToOpenRequestMap[readIn.FH] - if !ok { - logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToOpenRequestMap)", inHeader.NodeID, readIn.FH) - } - - fhInodeNumber, ok = globals.fhToInodeNumberMap[readIn.FH] - if !ok { - logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToInodeNumberMap)", inHeader.NodeID, readIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoRead(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, readIn.FH, fhInodeNumber) - } - - globals.Unlock() - - if 0 != (fOpenRequestFlags & fission.FOpenRequestWRONLY) { - fileInode.unlock(true) - errno = syscall.EINVAL - return - } - - // fileInode = referenceFileInode(inode.InodeNumber(inHeader.NodeID)) - if nil == fileInode { - logFatalf("DoRead(NodeID=%v,FH=%v) called for non-FileInode", inHeader.NodeID, readIn.FH) - } - // defer fileInode.dereference() - - err = fileInode.ensureCachedStatPopulatedWhileLocked() - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - // grantedLock = fileInode.getSharedLock() - // defer grantedLock.release() - - err = fileInode.populateExtentMap(uint64(readIn.Offset), uint64(readIn.Size)) - if nil != err { - fileInode.unlock(true) - errno = convertErrToErrno(err, syscall.EIO) - return - } - - readPlan, readPlanSpan = fileInode.getReadPlan(uint64(readIn.Offset), uint64(readIn.Size)) - - if (nil == readPlan) || (0 == readPlanSpan) { - readOut = &fission.ReadOut{ - Data: make([]byte, 0), - } - } else { - readOut = &fission.ReadOut{ - Data: make([]byte, 0, readPlanSpan), - } - - for _, readPlanStepAsInterface = range readPlan { - switch readPlanStepAsInterface.(type) { - case *multiObjectExtentStruct: - readPlanStepAsMultiObjectExtentStruct = readPlanStepAsInterface.(*multiObjectExtentStruct) - - if "" == readPlanStepAsMultiObjectExtentStruct.objectName { - // Zero-fill for readPlanStep.length - - readOut.Data = append(readOut.Data, make([]byte, readPlanStepAsMultiObjectExtentStruct.length)...) - } else { - // Fetch LogSegment data... possibly crossing LogSegmentCacheLine boundaries - - curObjectOffset = readPlanStepAsMultiObjectExtentStruct.objectOffset - readPlanStepRemainingLength = readPlanStepAsMultiObjectExtentStruct.length - - for readPlanStepRemainingLength > 0 { - logSegmentCacheElement = fetchLogSegmentCacheLine(readPlanStepAsMultiObjectExtentStruct.containerName, readPlanStepAsMultiObjectExtentStruct.objectName, curObjectOffset) - - if logSegmentCacheElementStateGetFailed == logSegmentCacheElement.state { - fileInode.unlock(true) - errno = syscall.EIO - return - } - - logSegmentCacheElementBufStartingPosition = curObjectOffset - logSegmentCacheElement.startingOffset - logSegmentCacheElementBufRemainingLen = uint64(len(logSegmentCacheElement.buf)) - logSegmentCacheElementBufStartingPosition - - if logSegmentCacheElementBufRemainingLen <= readPlanStepRemainingLength { - logSegmentCacheElementBufSelectedLen = logSegmentCacheElementBufRemainingLen - } else { - logSegmentCacheElementBufSelectedLen = readPlanStepRemainingLength - } - - logSegmentCacheElementBufEndingPosition = logSegmentCacheElementBufStartingPosition + logSegmentCacheElementBufSelectedLen - - readOut.Data = append(readOut.Data, logSegmentCacheElement.buf[logSegmentCacheElementBufStartingPosition:logSegmentCacheElementBufEndingPosition]...) - - curObjectOffset += logSegmentCacheElementBufSelectedLen - readPlanStepRemainingLength -= logSegmentCacheElementBufSelectedLen - } - } - case *singleObjectExtentWithLinkStruct: - readPlanStepAsSingleObjectExtentWithLink = readPlanStepAsInterface.(*singleObjectExtentWithLinkStruct) - - if nil == readPlanStepAsSingleObjectExtentWithLink.chunkedPutContext { - // Zero-fill for readPlanStep.length - - readOut.Data = append(readOut.Data, make([]byte, readPlanStepAsSingleObjectExtentWithLink.length)...) - } else { - // Fetch LogSegment data... from readPlanStepAsSingleObjectExtentWithLink.chunkedPutContextStruct - - _ = atomic.AddUint64(&globals.metrics.LogSegmentPUTReadHits, 1) - - readOut.Data = append(readOut.Data, readPlanStepAsSingleObjectExtentWithLink.chunkedPutContext.buf[readPlanStepAsSingleObjectExtentWithLink.objectOffset:readPlanStepAsSingleObjectExtentWithLink.objectOffset+readPlanStepAsSingleObjectExtentWithLink.length]...) - } - default: - logFatalf("getReadPlan() returned an invalid readPlanStep: %v", readPlanStepAsInterface) - } - } - } - - fileInode.unlock(false) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoRead_bytes, uint64(len(readOut.Data))) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission.WriteIn) (writeOut *fission.WriteOut, errno syscall.Errno) { - var ( - actualOffset uint64 - chunkedPutContext *chunkedPutContextStruct - chunkedPutContextElement *list.Element - err error - fhInodeNumber uint64 - fileInode *fileInodeStruct - fOpenRequestFlags uint32 - ok bool - singleObjectExtent *singleObjectExtentStruct - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoWrite_calls, 1) - - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(inHeader.NodeID)) - - globals.Lock() - - fOpenRequestFlags, ok = globals.fhToOpenRequestMap[writeIn.FH] - if !ok { - logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToOpenRequestMap)", inHeader.NodeID, writeIn.FH) - } - - fhInodeNumber, ok = globals.fhToInodeNumberMap[writeIn.FH] - if !ok { - logFatalf("DoWrite(NodeID=%v,FH=%v) called for unknown FH (in fhToInodeNumberMap)", inHeader.NodeID, writeIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoWrite(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, writeIn.FH, fhInodeNumber) - } - - globals.Unlock() - - if 0 != (fOpenRequestFlags & fission.FOpenRequestRDONLY) { - fileInode.unlock(true) - errno = syscall.EINVAL - return - } - - // fileInode = referenceFileInode(inode.InodeNumber(inHeader.NodeID)) - if nil == fileInode { - logFatalf("DoWrite(NodeID=%v,FH=%v) called for non-FileInode", inHeader.NodeID, writeIn.FH) - } - - err = fileInode.ensureCachedStatPopulatedWhileLocked() - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - if 0 == (fOpenRequestFlags & fission.FOpenRequestAPPEND) { - actualOffset = writeIn.Offset - } else { - actualOffset = fileInode.cachedStat.Size - } - - // Grab quota to start a fresh chunkedPutContext before calling fileInode.getExclusiveLock() - // since, in the pathologic case where only fileInode has any outstanding chunkedPutContext's, - // they can only be complete()'d while holding fileInode.getExclusiveLock() and we would - // deadlock. - - _ = <-globals.fileInodeDirtyLogSegmentChan - - // grantedLock = fileInode.getExclusiveLock() - - if 0 == fileInode.chunkedPutList.Len() { - // No chunkedPutContext is present (so none can be open), so open one - - _ = atomic.AddUint64(&globals.metrics.LogSegmentPUTs, 1) - - chunkedPutContext = &chunkedPutContextStruct{ - buf: make([]byte, 0), - fileInode: fileInode, - state: chunkedPutContextStateOpen, - sendChan: make(chan struct{}, 1), - wakeChan: make(chan struct{}, 1), - inRead: false, - flushRequested: false, - } - - chunkedPutContext.extentMap = sortedmap.NewLLRBTree(sortedmap.CompareUint64, chunkedPutContext) - chunkedPutContext.chunkedPutListElement = fileInode.chunkedPutList.PushBack(chunkedPutContext) - - // fileInode.reference() - - pruneFileInodeDirtyListIfNecessary() - - globals.Lock() - fileInode.dirtyListElement = globals.fileInodeDirtyList.PushBack(fileInode) - globals.Unlock() - - go chunkedPutContext.sendDaemon() - } else { - globals.Lock() - globals.fileInodeDirtyList.MoveToBack(fileInode.dirtyListElement) - globals.Unlock() - - chunkedPutContextElement = fileInode.chunkedPutList.Back() - chunkedPutContext = chunkedPutContextElement.Value.(*chunkedPutContextStruct) - - if chunkedPutContextStateOpen == chunkedPutContext.state { - // Use this most recent (and open) chunkedPutContext... so we can give back our chunkedPutContext quota - - globals.fileInodeDirtyLogSegmentChan <- struct{}{} - } else { - // Most recent chunkedPutContext is closed, so open a new one - - _ = atomic.AddUint64(&globals.metrics.LogSegmentPUTs, 1) - - chunkedPutContext = &chunkedPutContextStruct{ - buf: make([]byte, 0), - fileInode: fileInode, - state: chunkedPutContextStateOpen, - sendChan: make(chan struct{}, 1), - wakeChan: make(chan struct{}, 1), - inRead: false, - flushRequested: false, - } - - chunkedPutContext.extentMap = sortedmap.NewLLRBTree(sortedmap.CompareUint64, chunkedPutContext) - chunkedPutContext.chunkedPutListElement = fileInode.chunkedPutList.PushBack(chunkedPutContext) - - // fileInode.reference() - - go chunkedPutContext.sendDaemon() - } - } - - singleObjectExtent = &singleObjectExtentStruct{ - fileOffset: actualOffset, - objectOffset: uint64(len(chunkedPutContext.buf)), - length: uint64(len(writeIn.Data)), - } - - chunkedPutContext.mergeSingleObjectExtent(singleObjectExtent) - - if (singleObjectExtent.fileOffset + singleObjectExtent.length) > fileInode.cachedStat.Size { - fileInode.cachedStat.Size = singleObjectExtent.fileOffset + singleObjectExtent.length - } - - chunkedPutContext.buf = append(chunkedPutContext.buf, writeIn.Data...) - - select { - case chunkedPutContext.sendChan <- struct{}{}: - // We just notified sendDaemon() - default: - // We didn't need to notify sendDaemon() - } - - if uint64(len(chunkedPutContext.buf)) >= globals.config.MaxFlushSize { - // Time to do a Flush - chunkedPutContext.state = chunkedPutContextStateClosing - close(chunkedPutContext.sendChan) - } - - // grantedLock.release() - - // fileInode.dereference() - - fileInode.unlock(false) - - writeOut = &fission.WriteOut{ - Size: uint32(len(writeIn.Data)), - Padding: 0, - } - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoWrite_bytes, uint64(writeOut.Size)) - globals.stats.FUSEDoWriteBytes.Add(uint64(writeOut.Size)) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoStatFS(inHeader *fission.InHeader) (statFSOut *fission.StatFSOut, errno syscall.Errno) { - var ( - err error - statVFSRequest *jrpcfs.StatVFSRequest - statVFSReply *jrpcfs.StatVFS - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoStatFS_calls, 1) - - statVFSRequest = &jrpcfs.StatVFSRequest{ - MountID: globals.mountID, - } - - statVFSReply = &jrpcfs.StatVFS{} - - err = globals.retryRPCClient.Send("RpcStatVFS", statVFSRequest, statVFSReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - statFSOut = &fission.StatFSOut{ - KStatFS: fission.KStatFS{ - Blocks: statVFSReply.TotalBlocks, - BFree: statVFSReply.FreeBlocks, - BAvail: statVFSReply.AvailBlocks, - Files: statVFSReply.TotalInodes, - FFree: statVFSReply.FreeInodes, - BSize: uint32(statVFSReply.BlockSize), - NameLen: uint32(statVFSReply.MaxFilenameLen), - FRSize: uint32(statVFSReply.FragmentSize), - Padding: 0, - Spare: [6]uint32{0, 0, 0, 0, 0, 0}, - }, - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoRelease(inHeader *fission.InHeader, releaseIn *fission.ReleaseIn) (errno syscall.Errno) { - var ( - fhInodeNumber uint64 - fhSet fhSetType - ok bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoRelease_calls, 1) - - globals.Lock() - - _, ok = globals.fhToOpenRequestMap[releaseIn.FH] - if !ok { - logFatalf("DoRelease(NodeID=%v,FH=%v) called for unknown FH (fhToOpenRequestMap)", inHeader.NodeID, releaseIn.FH) - } - - delete(globals.fhToOpenRequestMap, releaseIn.FH) - - fhInodeNumber, ok = globals.fhToInodeNumberMap[releaseIn.FH] - if !ok { - logFatalf("DoRelease(NodeID=%v,FH=%v) called for unknown FH (fhToInodeNumberMap)", inHeader.NodeID, releaseIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoRelease(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, releaseIn.FH, fhInodeNumber) - } - - delete(globals.fhToInodeNumberMap, releaseIn.FH) - - fhSet, ok = globals.inodeNumberToFHMap[inHeader.NodeID] - if !ok { - logFatalf("DoRelease(NodeID=%v,FH=%v) called for unknown NodeID", inHeader.NodeID, releaseIn.FH) - } - - _, ok = fhSet[releaseIn.FH] - if !ok { - logFatalf("DoRelease(NodeID=%v,FH=%v) called for FH missing from fhSet: %v", inHeader.NodeID, releaseIn.FH, fhSet) - } - - delete(fhSet, releaseIn.FH) - - if 0 == len(fhSet) { - delete(globals.inodeNumberToFHMap, inHeader.NodeID) - } else { - globals.inodeNumberToFHMap[inHeader.NodeID] = fhSet - } - - globals.Unlock() - - errno = 0 - return -} - -func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission.FSyncIn) (errno syscall.Errno) { - var ( - fhInodeNumber uint64 - fileInode *fileInodeStruct - ok bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoFSync_calls, 1) - - globals.Lock() - - _, ok = globals.fhToOpenRequestMap[fSyncIn.FH] - if !ok { - logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToOpenRequestMap)", inHeader.NodeID, fSyncIn.FH) - } - - fhInodeNumber, ok = globals.fhToInodeNumberMap[fSyncIn.FH] - if !ok { - logFatalf("DoFSync(NodeID=%v,FH=%v) called for unknown FH (in fhToInodeNumberMap)", inHeader.NodeID, fSyncIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoFSync(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, fSyncIn.FH, fhInodeNumber) - } - - globals.Unlock() - - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(inHeader.NodeID)) - // fileInode = referenceFileInode(inode.InodeNumber(inHeader.NodeID)) - if nil == fileInode { - logFatalf("DoFSync(NodeID=%v,FH=%v) called for non-FileInode", inHeader.NodeID, fSyncIn.FH) - } - - fileInode.doFlushIfNecessary() - - // fileInode.dereference() - fileInode.unlock(false) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoSetXAttr(inHeader *fission.InHeader, setXAttrIn *fission.SetXAttrIn) (errno syscall.Errno) { - var ( - err error - setXAttrReply *jrpcfs.Reply - setXAttrRequest *jrpcfs.SetXAttrRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoSetXAttr_calls, 1) - - if !globals.config.XAttrEnabled { - errno = syscall.ENOSYS - return - } - - setXAttrRequest = &jrpcfs.SetXAttrRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - AttrName: string(setXAttrIn.Name[:]), - AttrValue: setXAttrIn.Data[:], - AttrFlags: fs.SetXAttrCreateOrReplace, - } - - setXAttrReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcSetXAttr", setXAttrRequest, setXAttrReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoSetXAttr_bytes, uint64(len(setXAttrIn.Data))) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoGetXAttr(inHeader *fission.InHeader, getXAttrIn *fission.GetXAttrIn) (getXAttrOut *fission.GetXAttrOut, errno syscall.Errno) { - var ( - err error - getXAttrReply *jrpcfs.GetXAttrReply - getXAttrRequest *jrpcfs.GetXAttrRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoGetXAttr_calls, 1) - - if !globals.config.XAttrEnabled { - errno = syscall.ENOSYS - return - } - - getXAttrRequest = &jrpcfs.GetXAttrRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - AttrName: string(getXAttrIn.Name[:]), - } - - getXAttrReply = &jrpcfs.GetXAttrReply{} - - err = globals.retryRPCClient.Send("RpcGetXAttr", getXAttrRequest, getXAttrReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - if 0 == getXAttrIn.Size { - getXAttrOut = &fission.GetXAttrOut{ - Size: uint32(len(getXAttrReply.AttrValue)), - Padding: 0, - Data: make([]byte, 0), - } - errno = 0 - return - } - - if uint32(len(getXAttrReply.AttrValue)) > getXAttrIn.Size { - errno = syscall.ERANGE - return - } - - getXAttrOut = &fission.GetXAttrOut{ - Size: uint32(len(getXAttrReply.AttrValue)), - Padding: 0, - Data: getXAttrReply.AttrValue, - } - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoGetXAttr_bytes, uint64(getXAttrOut.Size)) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoListXAttr(inHeader *fission.InHeader, listXAttrIn *fission.ListXAttrIn) (listXAttrOut *fission.ListXAttrOut, errno syscall.Errno) { - var ( - err error - listXAttrReply *jrpcfs.ListXAttrReply - listXAttrRequest *jrpcfs.ListXAttrRequest - totalSize uint32 - xAttrIndex int - xAttrName string - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoListXAttr_calls, 1) - - if !globals.config.XAttrEnabled { - errno = syscall.ENOSYS - return - } - - listXAttrRequest = &jrpcfs.ListXAttrRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - } - - listXAttrReply = &jrpcfs.ListXAttrReply{} - - err = globals.retryRPCClient.Send("RpcListXAttr", listXAttrRequest, listXAttrReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - totalSize = 0 - - for _, xAttrName = range listXAttrReply.AttrNames { - totalSize += uint32(len(xAttrName) + 1) - } - - if 0 == listXAttrIn.Size { - listXAttrOut = &fission.ListXAttrOut{ - Size: totalSize, - Padding: 0, - Name: make([][]byte, 0), - } - errno = 0 - return - } - - listXAttrOut = &fission.ListXAttrOut{ - Size: totalSize, // unnecessary - Padding: 0, - Name: make([][]byte, len(listXAttrReply.AttrNames)), - } - - for xAttrIndex, xAttrName = range listXAttrReply.AttrNames { - listXAttrOut.Name[xAttrIndex] = []byte(xAttrName) - } - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoListXAttr_names, uint64(len(listXAttrOut.Name))) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoRemoveXAttr(inHeader *fission.InHeader, removeXAttrIn *fission.RemoveXAttrIn) (errno syscall.Errno) { - var ( - err error - removeXAttrReply *jrpcfs.Reply - removeXAttrRequest *jrpcfs.RemoveXAttrRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoRemoveXAttr_calls, 1) - - if !globals.config.XAttrEnabled { - errno = syscall.ENOSYS - return - } - - removeXAttrRequest = &jrpcfs.RemoveXAttrRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - AttrName: string(removeXAttrIn.Name[:]), - } - - removeXAttrReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcRemoveXAttr", removeXAttrRequest, removeXAttrReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoFlush(inHeader *fission.InHeader, flushIn *fission.FlushIn) (errno syscall.Errno) { - var ( - fhInodeNumber uint64 - ok bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoFlush_calls, 1) - - globals.Lock() - - _, ok = globals.fhToOpenRequestMap[flushIn.FH] - if !ok { - logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToOpenRequestMap)", inHeader.NodeID, flushIn.FH) - } - - fhInodeNumber, ok = globals.fhToInodeNumberMap[flushIn.FH] - if !ok { - logFatalf("DoFlush(NodeID=%v,FH=%v) called for unknown FH (in fhToInodeNumberMap)", inHeader.NodeID, flushIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoFlush(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, flushIn.FH, fhInodeNumber) - } - - globals.Unlock() - - errno = 0 - return -} - -func (dummy *globalsStruct) DoInit(inHeader *fission.InHeader, initIn *fission.InitIn) (initOut *fission.InitOut, errno syscall.Errno) { - var ( - initOutFlags uint32 - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoInit_calls, 1) - - if globals.config.ReadDirPlusEnabled { - initOutFlags = initOutFlagsMaskReadDirPlusEnabled - } else { - initOutFlags = initOutFlagsMaskReadDirPlusDisabled - } - - initOut = &fission.InitOut{ - Major: initIn.Major, - Minor: initIn.Minor, - MaxReadAhead: initIn.MaxReadAhead, - Flags: initOutFlags, - MaxBackground: globals.config.FUSEMaxBackground, - CongestionThreshhold: globals.config.FUSECongestionThreshhold, - MaxWrite: globals.config.FUSEMaxWrite, - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoOpenDir(inHeader *fission.InHeader, openDirIn *fission.OpenDirIn) (openDirOut *fission.OpenDirOut, errno syscall.Errno) { - var ( - err error - fhSet fhSetType - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - ok bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoOpenDir_calls, 1) - - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - if syscall.S_IFDIR != (getStatReply.FileMode & syscall.S_IFMT) { - errno = syscall.ENOTDIR - return - } - - globals.Lock() - - globals.lastFH++ - - globals.fhToInodeNumberMap[globals.lastFH] = inHeader.NodeID - - fhSet, ok = globals.inodeNumberToFHMap[inHeader.NodeID] - if !ok { - fhSet = make(fhSetType) - } - fhSet[globals.lastFH] = struct{}{} - globals.inodeNumberToFHMap[inHeader.NodeID] = fhSet - - openDirOut = &fission.OpenDirOut{ - FH: globals.lastFH, - OpenFlags: 0, - Padding: 0, - } - - globals.Unlock() - - errno = 0 - return -} - -func (dummy *globalsStruct) DoReadDir(inHeader *fission.InHeader, readDirIn *fission.ReadDirIn) (readDirOut *fission.ReadDirOut, errno syscall.Errno) { - var ( - curSize uint32 - dirEntIndex uint64 - dirEntNameLenAligned uint32 - dirEnt fission.DirEnt - dirEntSize uint32 - dirEntry *jrpcfs.DirEntry - err error - fhInodeNumber uint64 - maxEntries uint64 - numEntries uint64 - ok bool - readdirByLocRequest *jrpcfs.ReaddirByLocRequest - readdirReply *jrpcfs.ReaddirReply - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoReadDir_calls, 1) - - globals.Lock() - - fhInodeNumber, ok = globals.fhToInodeNumberMap[readDirIn.FH] - if !ok { - logFatalf("DoReadDir(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, readDirIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoReadDir(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, readDirIn.FH, fhInodeNumber) - } - - globals.Unlock() - - maxEntries = (uint64(readDirIn.Size) + fission.DirEntFixedPortionSize + 1 - 1) / (fission.DirEntFixedPortionSize + 1) - - readdirByLocRequest = &jrpcfs.ReaddirByLocRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - MaxEntries: maxEntries, - PrevDirEntLocation: int64(readDirIn.Offset) - 1, - } - - readdirReply = &jrpcfs.ReaddirReply{} - - err = globals.retryRPCClient.Send("RpcReaddirByLoc", readdirByLocRequest, readdirReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - numEntries = uint64(len(readdirReply.DirEnts)) - - readDirOut = &fission.ReadDirOut{ - DirEnt: make([]fission.DirEnt, 0, numEntries), - } - - curSize = 0 - - for dirEntIndex = 0; dirEntIndex < numEntries; dirEntIndex++ { - dirEntry = &readdirReply.DirEnts[dirEntIndex] - - dirEntNameLenAligned = (uint32(len(dirEntry.Basename)) + (fission.DirEntAlignment - 1)) & ^uint32(fission.DirEntAlignment-1) - dirEntSize = fission.DirEntFixedPortionSize + dirEntNameLenAligned - - if (curSize + dirEntSize) > readDirIn.Size { - break - } - - dirEnt = fission.DirEnt{ - Ino: uint64(dirEntry.InodeNumber), - Off: uint64(dirEntry.NextDirLocation), - NameLen: uint32(len(dirEntry.Basename)), // unnecessary - Type: uint32(dirEntry.FileType), - Name: []byte(dirEntry.Basename), - } - - readDirOut.DirEnt = append(readDirOut.DirEnt, dirEnt) - - curSize += dirEntSize - } - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoReadDir_entries, uint64(len(readDirOut.DirEnt))) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoReleaseDir(inHeader *fission.InHeader, releaseDirIn *fission.ReleaseDirIn) (errno syscall.Errno) { - var ( - fhInodeNumber uint64 - fhSet fhSetType - ok bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoReleaseDir_calls, 1) - - globals.Lock() - - fhInodeNumber, ok = globals.fhToInodeNumberMap[releaseDirIn.FH] - if !ok { - logFatalf("DoReleaseDir(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, releaseDirIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoReleaseDir(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, releaseDirIn.FH, fhInodeNumber) - } - - delete(globals.fhToInodeNumberMap, releaseDirIn.FH) - - fhSet, ok = globals.inodeNumberToFHMap[inHeader.NodeID] - if !ok { - logFatalf("DoReleaseDir(NodeID=%v,FH=%v) called for unknown NodeID", inHeader.NodeID, releaseDirIn.FH) - } - - _, ok = fhSet[releaseDirIn.FH] - if !ok { - logFatalf("DoReleaseDir(NodeID=%v,FH=%v) called for FH missing from fhSet: %v", inHeader.NodeID, releaseDirIn.FH, fhSet) - } - - delete(fhSet, releaseDirIn.FH) - - if 0 == len(fhSet) { - delete(globals.inodeNumberToFHMap, inHeader.NodeID) - } else { - globals.inodeNumberToFHMap[inHeader.NodeID] = fhSet - } - - globals.Unlock() - - errno = 0 - return -} - -func (dummy *globalsStruct) DoFSyncDir(inHeader *fission.InHeader, fSyncDirIn *fission.FSyncDirIn) (errno syscall.Errno) { - var ( - fhInodeNumber uint64 - ok bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoFSyncDir_calls, 1) - - globals.Lock() - - fhInodeNumber, ok = globals.fhToInodeNumberMap[fSyncDirIn.FH] - if !ok { - logFatalf("DoFSync(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, fSyncDirIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoFSync(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, fSyncDirIn.FH, fhInodeNumber) - } - - globals.Unlock() - - errno = 0 - return -} - -func (dummy *globalsStruct) DoGetLK(inHeader *fission.InHeader, getLKIn *fission.GetLKIn) (getLKOut *fission.GetLKOut, errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoGetLK_calls, 1) - errno = syscall.ENOSYS - return -} -func (dummy *globalsStruct) DoSetLK(inHeader *fission.InHeader, setLKIn *fission.SetLKIn) (errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoSetLK_calls, 1) - errno = syscall.ENOSYS - return -} -func (dummy *globalsStruct) DoSetLKW(inHeader *fission.InHeader, setLKWIn *fission.SetLKWIn) (errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoSetLKW_calls, 1) - errno = syscall.ENOSYS - return -} - -func (dummy *globalsStruct) DoAccess(inHeader *fission.InHeader, accessIn *fission.AccessIn) (errno syscall.Errno) { - var ( - err error - accessReply *jrpcfs.Reply - accessRequest *jrpcfs.AccessRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoAccess_calls, 1) - - accessRequest = &jrpcfs.AccessRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - UserID: int32(inHeader.UID), - GroupID: int32(inHeader.GID), - AccessMode: accessIn.Mask, - } - - accessReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcAccess", accessRequest, accessReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - errno = 0 - return -} - -func (dummy *globalsStruct) DoCreate(inHeader *fission.InHeader, createIn *fission.CreateIn) (createOut *fission.CreateOut, errno syscall.Errno) { - var ( - aTimeNSec uint32 - aTimeSec uint64 - cTimeNSec uint32 - cTimeSec uint64 - createReply *jrpcfs.InodeReply - createRequest *jrpcfs.CreateRequest - err error - fhSet fhSetType - getStatReply *jrpcfs.StatStruct - getStatRequest *jrpcfs.GetStatRequest - mTimeNSec uint32 - mTimeSec uint64 - ok bool - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoCreate_calls, 1) - - createRequest = &jrpcfs.CreateRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - Basename: string(createIn.Name[:]), - UserID: int32(inHeader.UID), - GroupID: int32(inHeader.GID), - FileMode: createIn.Mode & uint32(os.ModePerm), - } - - createReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcCreate", createRequest, createReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - getStatRequest = &jrpcfs.GetStatRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: createReply.InodeNumber, - }, - } - - getStatReply = &jrpcfs.StatStruct{} - - err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - aTimeSec, aTimeNSec = nsToUnixTime(getStatReply.ATimeNs) - mTimeSec, mTimeNSec = nsToUnixTime(getStatReply.MTimeNs) - cTimeSec, cTimeNSec = nsToUnixTime(getStatReply.CTimeNs) - - globals.Lock() - - globals.lastFH++ - - globals.fhToOpenRequestMap[globals.lastFH] = createIn.Flags - globals.fhToInodeNumberMap[globals.lastFH] = uint64(createReply.InodeNumber) - - fhSet, ok = globals.inodeNumberToFHMap[uint64(createReply.InodeNumber)] - if !ok { - fhSet = make(fhSetType) - } - fhSet[globals.lastFH] = struct{}{} - globals.inodeNumberToFHMap[uint64(createReply.InodeNumber)] = fhSet - - createOut = &fission.CreateOut{ - EntryOut: fission.EntryOut{ - NodeID: uint64(createReply.InodeNumber), - Generation: 0, - EntryValidSec: globals.entryValidSec, - AttrValidSec: globals.attrValidSec, - EntryValidNSec: globals.entryValidNSec, - AttrValidNSec: globals.attrValidNSec, - Attr: fission.Attr{ - Ino: uint64(createReply.InodeNumber), - Size: getStatReply.Size, - Blocks: 0, // fixAttrSizes() will compute this - ATimeSec: aTimeSec, - MTimeSec: mTimeSec, - CTimeSec: cTimeSec, - ATimeNSec: aTimeNSec, - MTimeNSec: mTimeNSec, - CTimeNSec: cTimeNSec, - Mode: getStatReply.FileMode, - NLink: uint32(getStatReply.NumLinks), - UID: getStatReply.UserID, - GID: getStatReply.GroupID, - RDev: 0, - BlkSize: 0, // fixAttrSizes() will set this - Padding: 0, - }, - }, - FH: globals.lastFH, - OpenFlags: fission.FOpenResponseDirectIO, - Padding: 0, - } - - globals.Unlock() - - fixAttrSizes(&createOut.Attr) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoInterrupt(inHeader *fission.InHeader, interruptIn *fission.InterruptIn) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoInterrupt_calls, 1) - return -} - -func (dummy *globalsStruct) DoBMap(inHeader *fission.InHeader, bMapIn *fission.BMapIn) (bMapOut *fission.BMapOut, errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoBMap_calls, 1) - errno = syscall.ENOSYS - return -} - -func (dummy *globalsStruct) DoDestroy(inHeader *fission.InHeader) (errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoDestroy_calls, 1) - errno = syscall.ENOSYS - return -} - -func (dummy *globalsStruct) DoPoll(inHeader *fission.InHeader, pollIn *fission.PollIn) (pollOut *fission.PollOut, errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoPoll_calls, 1) - errno = syscall.ENOSYS - return -} - -func (dummy *globalsStruct) DoBatchForget(inHeader *fission.InHeader, batchForgetIn *fission.BatchForgetIn) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoBatchForget_calls, 1) - _ = atomic.AddUint64(&globals.metrics.FUSE_DoBatchForget_nodes, uint64(len(batchForgetIn.Forget))) - return -} - -func (dummy *globalsStruct) DoFAllocate(inHeader *fission.InHeader, fAllocateIn *fission.FAllocateIn) (errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoFAllocate_calls, 1) - errno = syscall.ENOSYS - return -} - -func (dummy *globalsStruct) DoReadDirPlus(inHeader *fission.InHeader, readDirPlusIn *fission.ReadDirPlusIn) (readDirPlusOut *fission.ReadDirPlusOut, errno syscall.Errno) { - var ( - aTimeNSec uint32 - aTimeSec uint64 - cTimeNSec uint32 - cTimeSec uint64 - curSize uint32 - dirEntIndex uint64 - dirEntNameLenAligned uint32 - dirEntPlus fission.DirEntPlus - dirEntPlusSize uint32 - dirEntry *jrpcfs.DirEntry - err error - fhInodeNumber uint64 - mTimeNSec uint32 - mTimeSec uint64 - maxEntries uint64 - numEntries uint64 - ok bool - readdirPlusByLocRequest *jrpcfs.ReaddirPlusByLocRequest - readdirPlusReply *jrpcfs.ReaddirPlusReply - statStruct *jrpcfs.StatStruct - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoReadDirPlus_calls, 1) - - globals.Lock() - - fhInodeNumber, ok = globals.fhToInodeNumberMap[readDirPlusIn.FH] - if !ok { - logFatalf("DoReadDirPlus(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, readDirPlusIn.FH) - } - if fhInodeNumber != inHeader.NodeID { - logFatalf("DoReadDirPlus(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, readDirPlusIn.FH, fhInodeNumber) - } - - globals.Unlock() - - maxEntries = (uint64(readDirPlusIn.Size) + fission.DirEntPlusFixedPortionSize + 1 - 1) / (fission.DirEntPlusFixedPortionSize + 1) - - readdirPlusByLocRequest = &jrpcfs.ReaddirPlusByLocRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inHeader.NodeID), - }, - MaxEntries: maxEntries, - PrevDirEntLocation: int64(readDirPlusIn.Offset) - 1, - } - - readdirPlusReply = &jrpcfs.ReaddirPlusReply{} - - err = globals.retryRPCClient.Send("RpcReaddirPlusByLoc", readdirPlusByLocRequest, readdirPlusReply) - if nil != err { - errno = convertErrToErrno(err, syscall.EIO) - return - } - - numEntries = uint64(len(readdirPlusReply.DirEnts)) - if numEntries != uint64(len(readdirPlusReply.StatEnts)) { - logFatalf("DoReadDirPlus(NodeID=%v,FH=%v) fetched mismatched number of DirEnts (%v) & StatEnts (%v)", inHeader.NodeID, readDirPlusIn.FH, len(readdirPlusReply.DirEnts), len(readdirPlusReply.StatEnts)) - } - - readDirPlusOut = &fission.ReadDirPlusOut{ - DirEntPlus: make([]fission.DirEntPlus, 0, numEntries), - } - - curSize = 0 - - for dirEntIndex = 0; dirEntIndex < numEntries; dirEntIndex++ { - dirEntry = &readdirPlusReply.DirEnts[dirEntIndex] - statStruct = &readdirPlusReply.StatEnts[dirEntIndex] - - dirEntNameLenAligned = (uint32(len(dirEntry.Basename)) + (fission.DirEntAlignment - 1)) & ^uint32(fission.DirEntAlignment-1) - dirEntPlusSize = fission.DirEntPlusFixedPortionSize + dirEntNameLenAligned - - if (curSize + dirEntPlusSize) > readDirPlusIn.Size { - break - } - - aTimeSec, aTimeNSec = nsToUnixTime(statStruct.ATimeNs) - mTimeSec, mTimeNSec = nsToUnixTime(statStruct.MTimeNs) - cTimeSec, cTimeNSec = nsToUnixTime(statStruct.CTimeNs) - - dirEntPlus = fission.DirEntPlus{ - EntryOut: fission.EntryOut{ - NodeID: uint64(dirEntry.InodeNumber), - Generation: 0, - EntryValidSec: globals.entryValidSec, - AttrValidSec: globals.attrValidSec, - EntryValidNSec: globals.entryValidNSec, - AttrValidNSec: globals.attrValidNSec, - Attr: fission.Attr{ - Ino: uint64(dirEntry.InodeNumber), - Size: statStruct.Size, - Blocks: 0, // fixAttrSizes() will compute this - ATimeSec: aTimeSec, - MTimeSec: mTimeSec, - CTimeSec: cTimeSec, - ATimeNSec: aTimeNSec, - MTimeNSec: mTimeNSec, - CTimeNSec: cTimeNSec, - Mode: statStruct.FileMode, - NLink: uint32(statStruct.NumLinks), - UID: statStruct.UserID, - GID: statStruct.GroupID, - RDev: 0, - BlkSize: 0, // fixAttrSizes() will set this - Padding: 0, - }, - }, - DirEnt: fission.DirEnt{ - Ino: uint64(dirEntry.InodeNumber), - Off: uint64(dirEntry.NextDirLocation), - NameLen: uint32(len(dirEntry.Basename)), // unnecessary - Type: uint32(dirEntry.FileType), - Name: []byte(dirEntry.Basename), - }, - } - - fixAttrSizes(&dirEntPlus.EntryOut.Attr) - - readDirPlusOut.DirEntPlus = append(readDirPlusOut.DirEntPlus, dirEntPlus) - - curSize += dirEntPlusSize - } - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoReadDirPlus_entries, uint64(len(readDirPlusOut.DirEntPlus))) - - errno = 0 - return -} - -func (dummy *globalsStruct) DoRename2(inHeader *fission.InHeader, rename2In *fission.Rename2In) (errno syscall.Errno) { - var ( - destroyReply *jrpcfs.Reply - destroyRequest *jrpcfs.DestroyRequest - err error - fileInode *fileInodeStruct - lookupReply *jrpcfs.InodeReply - lookupRequest *jrpcfs.LookupRequest - moveReply *jrpcfs.MoveReply - moveRequest *jrpcfs.MoveRequest - ) - - _ = atomic.AddUint64(&globals.metrics.FUSE_DoRename2_calls, 1) - - lookupRequest = &jrpcfs.LookupRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(rename2In.NewDir), - }, - Basename: string(rename2In.NewName[:]), - } - - lookupReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcLookup", lookupRequest, lookupReply) - if nil == err { - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(lookupReply.InodeNumber)) - - // Make sure potentially file inode didn't move before we were able to ExclusiveLease it - - lookupReply = &jrpcfs.InodeReply{} - - err = globals.retryRPCClient.Send("RpcLookup", lookupRequest, lookupReply) - if (nil != err) || (fileInode.InodeNumber != inode.InodeNumber(lookupReply.InodeNumber)) { - fileInode.unlock(true) - fileInode = nil - } else { - fileInode.doFlushIfNecessary() - } - } else { - fileInode = nil - } - - moveRequest = &jrpcfs.MoveRequest{ - MountID: globals.mountID, - SrcDirInodeNumber: int64(inHeader.NodeID), - SrcBasename: string(rename2In.OldName[:]), - DstDirInodeNumber: int64(rename2In.NewDir), - DstBasename: string(rename2In.NewName[:]), - } - - moveReply = &jrpcfs.MoveReply{} - - err = globals.retryRPCClient.Send("RpcMove", moveRequest, moveReply) - if nil == err { - errno = 0 - } else { - errno = convertErrToErrno(err, syscall.EIO) - } - - if nil != fileInode { - fileInode.unlock(false) - } - - if 0 != moveReply.ToDestroyInodeNumber { - fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(moveReply.ToDestroyInodeNumber)) - - fileInode.doFlushIfNecessary() - - destroyRequest = &jrpcfs.DestroyRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: moveReply.ToDestroyInodeNumber, - }, - } - - destroyReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcDestroy", destroyRequest, destroyReply) - if nil != err { - logWarnf("RpcDestroy(InodeHandle: %#v) failed: err", err) - } - - fileInode.unlock(false) - } - - return -} - -func (dummy *globalsStruct) DoLSeek(inHeader *fission.InHeader, lSeekIn *fission.LSeekIn) (lSeekOut *fission.LSeekOut, errno syscall.Errno) { - _ = atomic.AddUint64(&globals.metrics.FUSE_DoLSeek_calls, 1) - errno = syscall.ENOSYS - return -} diff --git a/pfsagentd/functional_test.go b/pfsagentd/functional_test.go deleted file mode 100644 index 14c96d0d..00000000 --- a/pfsagentd/functional_test.go +++ /dev/null @@ -1,389 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "io/ioutil" - "math" - "math/rand" - "os" - "testing" -) - -var ( - testFile *os.File - testFileContents []byte - testFilePath string - testFileLastByte byte -) - -func TestSimpleWriteReadClose(t *testing.T) { - const ( - testFileName = "testSimpleWriteReadCloseFileName" - totalBytes int64 = 100 - randSeed int64 = 0x0123456789ABCDEF - ) - var ( - err error - readBackBuf []byte - writtenBuf []byte - ) - - testSetup(t) - - testFilePath = globals.config.FUSEMountPointPath + "/" + testFileName - - testFile, err = os.Create(testFilePath) - if nil != err { - t.Fatalf("os.Create(\"%s\") failed: %v", testFilePath, err) - } - - writtenBuf = make([]byte, totalBytes) - - rand.Seed(randSeed) - - _, _ = rand.Read(writtenBuf) - - _, err = testFile.WriteAt(writtenBuf, 0) - if nil != err { - t.Fatalf("testFile.WriteAt(writtenBuf, 0) failed: %v", err) - } - - readBackBuf = make([]byte, totalBytes) - - _, err = testFile.ReadAt(readBackBuf, 0) - if nil != err { - t.Fatalf("testFile.ReadAt(readBackBuf, 0) after Sync() failed: %v", err) - } - - if 0 != bytes.Compare(writtenBuf, readBackBuf) { - t.Fatalf("bytes.Compare(writtenBuf, readBackBuf) reports un-equal buf's") - } - - err = testFile.Close() - if nil != err { - t.Fatalf("testFile.Close() failed: %v", err) - } - - err = os.Remove(testFilePath) - if nil != err { - t.Fatalf("os.Remove(testFilePath) failed: %v", err) - } - - testTeardown(t) -} - -func TestRandomOverwrites(t *testing.T) { - const ( - testFileName = "testRandomeOverwritesFileName" - totalBytes int64 = 1000 - minWriteSize int64 = 1 - maxWriteSize int64 = 10 - numWrites int64 = 10000 - randSeed int64 = 0x0123456789ABCDEF - ) - var ( - err error - readBackBuf []byte - toWriteBuf []byte - toWriteBufLen int64 - toWriteOffset int64 - writeNumber int64 - writtenBuf []byte - ) - - testSetup(t) - - testFilePath = globals.config.FUSEMountPointPath + "/" + testFileName - - testFile, err = os.Create(testFilePath) - if nil != err { - t.Fatalf("os.Create(\"%s\") failed: %v", testFilePath, err) - } - - writtenBuf = make([]byte, totalBytes) - - rand.Seed(randSeed) - - _, _ = rand.Read(writtenBuf) - - _, err = testFile.WriteAt(writtenBuf, 0) - if nil != err { - t.Fatalf("testFile.WriteAt(writtenBuf, 0) failed: %v", err) - } - - toWriteBuf = make([]byte, maxWriteSize) - - for writeNumber = 0; writeNumber < numWrites; writeNumber++ { - toWriteBufLen = minWriteSize + rand.Int63n(maxWriteSize-minWriteSize+1) - toWriteOffset = rand.Int63n(totalBytes - toWriteBufLen + 1) - _, _ = rand.Read(toWriteBuf[:toWriteBufLen]) - - _, err = testFile.WriteAt(toWriteBuf[:toWriteBufLen], toWriteOffset) - if nil != err { - t.Fatalf("testFile.WriteAt(toWriteBuf[:toWriteBufLen], toWriteOffset) after Create() failed: %v", err) - } - - _ = copy(writtenBuf[toWriteOffset:(toWriteOffset+toWriteBufLen)], toWriteBuf[:toWriteBufLen]) - } - - readBackBuf = make([]byte, totalBytes) - - _, err = testFile.ReadAt(readBackBuf, 0) - if nil != err { - t.Fatalf("testFile.ReadAt(readBackBuf, 0) before Sync() failed: %v", err) - } - - if 0 != bytes.Compare(writtenBuf, readBackBuf) { - t.Fatalf("bytes.Compare(writtenBuf, readBackBuf) reports un-equal buf's before Sync()") - } - - err = testFile.Sync() - if nil != err { - t.Fatalf("testFile.Sync() after write pass 1 failed: %v", err) - } - - _, err = testFile.ReadAt(readBackBuf, 0) - if nil != err { - t.Fatalf("testFile.ReadAt(readBackBuf, 0) after Sync() failed: %v", err) - } - - if 0 != bytes.Compare(writtenBuf, readBackBuf) { - t.Fatalf("bytes.Compare(writtenBuf, readBackBuf) reports un-equal buf's after Sync()") - } - - err = testFile.Close() - if nil != err { - t.Fatalf("testFile.Close() after Sync() failed: %v", err) - } - - testFile, err = os.OpenFile(testFilePath, os.O_RDWR, 0) - if nil != err { - t.Fatalf("os.OpenFile(\"%s\",,) failed: %v", testFilePath, err) - } - - _, err = testFile.ReadAt(readBackBuf, 0) - if nil != err { - t.Fatalf("testFile.ReadAt(readBackBuf, 0) after Open() failed: %v", err) - } - - if 0 != bytes.Compare(writtenBuf, readBackBuf) { - t.Fatalf("bytes.Compare(writtenBuf, readBackBuf) reports un-equal buf's after Sync()") - } - - for writeNumber = 0; writeNumber < numWrites; writeNumber++ { - toWriteBufLen = minWriteSize + rand.Int63n(maxWriteSize-minWriteSize+1) - toWriteOffset = rand.Int63n(totalBytes - toWriteBufLen + 1) - _, _ = rand.Read(toWriteBuf[:toWriteBufLen]) - - _, err = testFile.WriteAt(toWriteBuf[:toWriteBufLen], toWriteOffset) - if nil != err { - t.Fatalf("testFile.WriteAt(toWriteBuf[:toWriteBufLen], toWriteOffset) after Open() failed: %v", err) - } - - _ = copy(writtenBuf[toWriteOffset:(toWriteOffset+toWriteBufLen)], toWriteBuf[:toWriteBufLen]) - } - - _, err = testFile.ReadAt(readBackBuf, 0) - if nil != err { - t.Fatalf("testFile.ReadAt(readBackBuf, 0) after overwrite failed: %v", err) - } - - if 0 != bytes.Compare(writtenBuf, readBackBuf) { - t.Fatalf("bytes.Compare(writtenBuf, readBackBuf) reports un-equal buf's after overwrite") - } - - err = testFile.Sync() - if nil != err { - t.Fatalf("testFile.Sync() after write pass 2 failed: %v", err) - } - - err = testFile.Close() - if nil != err { - t.Fatalf("testFile.Close() after Open() failed: %v", err) - } - - err = os.Remove(testFilePath) - if nil != err { - t.Fatalf("os.Remove(testFilePath) failed: %v", err) - } - - testTeardown(t) -} - -func optionallyReopenTestFile(t *testing.T, prefix string, reopenBeforeEachWrite bool, step string) { - var ( - err error - ) - - err = testFile.Close() - if nil != err { - t.Fatalf("%s: testFile.Close() before \"%s\" failed: %v", prefix, step, err) - } - - testFile, err = os.OpenFile(testFilePath, os.O_RDWR, 0) - if nil != err { - t.Fatalf("%s: os.OpenFile(\"%s\",,) before \"%s\" failed: %v", prefix, testFilePath, step, err) - } -} - -func writeNextByteSlice(t *testing.T, prefix string, offset int, toWriteLen int, step string) { - var ( - err error - extension int - index int - toWriteBuf []byte - ) - - extension = (offset + toWriteLen) - len(testFileContents) - - if 0 < extension { - testFileContents = append(testFileContents, make([]byte, extension)...) - } - - toWriteBuf = make([]byte, toWriteLen) - - for index = 0; index < toWriteLen; index++ { - if math.MaxUint8 == testFileLastByte { - testFileLastByte = 1 - } else { - testFileLastByte++ - } - toWriteBuf[index] = testFileLastByte - testFileContents[offset+index] = testFileLastByte - } - - _, err = testFile.WriteAt(toWriteBuf, int64(offset)) - if nil != err { - t.Fatalf("%s: testFile.WriteAt(,0) for \"%s\" failed: %v", prefix, step, err) - } -} - -func optionallyFlushTestFile(t *testing.T, prefix string, flushAfterEachWrite bool, step string) { - var ( - err error - ) - - if flushAfterEachWrite { - err = testFile.Sync() - if nil != err { - t.Fatalf("%s: testFile.Sync() for \"%s\" failed: %v", prefix, step, err) - } - } -} - -func verifyTestFileContents(t *testing.T, prefix string, step string) { - var ( - err error - readBackBuf []byte - ) - - _, err = testFile.Seek(0, os.SEEK_SET) - if nil != err { - t.Fatalf("%s: testFile.Seek(0, os.SEEK_SET) for \"%s\" failed: %v", prefix, step, err) - } - readBackBuf, err = ioutil.ReadAll(testFile) - if nil != err { - t.Fatalf("%s: ioutil.ReadAll(testFile) for \"%s\" failed: %v", prefix, step, err) - } - if 0 != bytes.Compare(readBackBuf, testFileContents) { - t.Fatalf("%s: bytes.Compare(readBackBuf, testFileContents) for \"%s\" reports unequal bufs", prefix, step) - } -} - -func testExhaustiveOverwrites(t *testing.T, prefix string, reopenBeforeEachWrite bool, flushAfterEachWrite bool) { - // Non-overlapping extent - - optionallyReopenTestFile(t, prefix, reopenBeforeEachWrite, "Non-overlapping extent") - writeNextByteSlice(t, prefix, 0, 3, "Non-overlapping extent") - optionallyFlushTestFile(t, prefix, flushAfterEachWrite, "Non-overlapping extent") - verifyTestFileContents(t, prefix, "Non-overlapping extent") - - // Overlapping existing extent precisely - - optionallyReopenTestFile(t, prefix, reopenBeforeEachWrite, "Overlapping existing extent precisely") - writeNextByteSlice(t, prefix, 0, 3, "Overlapping existing extent precisely") - optionallyFlushTestFile(t, prefix, flushAfterEachWrite, "Overlapping existing extent precisely") - verifyTestFileContents(t, prefix, "Overlapping existing extent precisely") - - // Overlapping the right of an existing extent - - optionallyReopenTestFile(t, prefix, reopenBeforeEachWrite, "Overlapping the right of an existing extent") - writeNextByteSlice(t, prefix, 2, 3, "Overlapping the right of an existing extent") - optionallyFlushTestFile(t, prefix, flushAfterEachWrite, "Overlapping the right of an existing extent") - verifyTestFileContents(t, prefix, "Overlapping the right of an existing extent") - - // Overlapping the middle of an existing extent - - optionallyReopenTestFile(t, prefix, reopenBeforeEachWrite, "Overlapping the middle of an existing extent") - writeNextByteSlice(t, prefix, 1, 3, "Overlapping the middle of an existing extent") - optionallyFlushTestFile(t, prefix, flushAfterEachWrite, "Overlapping the middle of an existing extent") - verifyTestFileContents(t, prefix, "Overlapping the middle of an existing extent") - - // Overlapping the left of an existing extent - - optionallyReopenTestFile(t, prefix, reopenBeforeEachWrite, "Overlapping the left of an existing extent") - writeNextByteSlice(t, prefix, 8, 3, "Overlapping the left of an existing extent") - optionallyFlushTestFile(t, prefix, flushAfterEachWrite, "Overlapping the left of an existing extent") - verifyTestFileContents(t, prefix, "Overlapping the left of an existing extent") - writeNextByteSlice(t, prefix, 6, 3, "Overlapping the left of an existing extent") - optionallyFlushTestFile(t, prefix, flushAfterEachWrite, "Overlapping the left of an existing extent") - verifyTestFileContents(t, prefix, "Overlapping the left of an existing extent") - - // Overlapping one existing extent on the right, another existing extent entirely, & a 3rd existing extent on the left - - optionallyReopenTestFile(t, prefix, reopenBeforeEachWrite, "Overlapping one existing extent on the right, another existing extent entirely, & a 3rd existing extent on the left") - writeNextByteSlice(t, prefix, 12, 3, "Overlapping one existing extent on the right, another existing extent entirely, & a 3rd existing extent on the left") - optionallyFlushTestFile(t, prefix, flushAfterEachWrite, "Overlapping one existing extent on the right, another existing extent entirely, & a 3rd existing extent on the left") - verifyTestFileContents(t, prefix, "Overlapping one existing extent on the right, another existing extent entirely, & a 3rd existing extent on the left") - writeNextByteSlice(t, prefix, 1, 13, "Overlapping one existing extent on the right, another existing extent entirely, & a 3rd existing extent on the left") - optionallyFlushTestFile(t, prefix, flushAfterEachWrite, "Overlapping one existing extent on the right, another existing extent entirely, & a 3rd existing extent on the left") - verifyTestFileContents(t, prefix, "Overlapping one existing extent on the right, another existing extent entirely, & a 3rd existing extent on the left") -} - -func TestExhaustiveOverwrites(t *testing.T) { - const ( - testFileName = "testExhaustiveOverwritesFileName" - ) - var ( - err error - ) - - testSetup(t) - - testFilePath = globals.config.FUSEMountPointPath + "/" + testFileName - - testFile, err = os.Create(testFilePath) - if nil != err { - t.Fatalf("os.Create(\"%s\") failed: %v", testFilePath, err) - } - err = testFile.Close() - if nil != err { - t.Fatalf("testFile.Close() failed: %v", err) - } - testFile, err = os.OpenFile(testFilePath, os.O_RDWR, 0) - if nil != err { - t.Fatalf("os.OpenFile(\"%s\",,) failed: %v", testFilePath, err) - } - - testFileContents = []byte{} - - testExhaustiveOverwrites(t, "Phase1", false, false) - testExhaustiveOverwrites(t, "Phase2", false, true) - testExhaustiveOverwrites(t, "Phase2", true, false) - testExhaustiveOverwrites(t, "Phase2", true, true) - - err = testFile.Close() - if nil != err { - t.Fatalf("testFile.Close() failed: %v", err) - } - - err = os.Remove(testFilePath) - if nil != err { - t.Fatalf("os.Remove(testFilePath) failed: %v", err) - } - - testTeardown(t) -} diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go deleted file mode 100644 index ffd8666f..00000000 --- a/pfsagentd/globals.go +++ /dev/null @@ -1,799 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "container/list" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "os/exec" - "sync" - "time" - - "bazil.org/fuse" - - "github.com/NVIDIA/fission" - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/jrpcfs" - "github.com/NVIDIA/proxyfs/retryrpc" - "github.com/NVIDIA/proxyfs/utils" -) - -type configStruct struct { - FUSEVolumeName string - FUSEMountPointPath string // Unless starting with '/', relative to $CWD - FUSEUnMountRetryDelay time.Duration - FUSEUnMountRetryCap uint64 - PlugInPath string - PlugInEnvName string - PlugInEnvValue string // If "", assume it's already set as desired - SwiftTimeout time.Duration - SwiftRetryLimit uint64 - SwiftRetryDelay time.Duration - SwiftRetryDelayVariance uint8 - SwiftRetryExpBackoff float64 - SwiftConnectionPoolSize uint64 - FetchExtentsFromFileOffset uint64 - FetchExtentsBeforeFileOffset uint64 - ReadCacheLineSize uint64 // Aligned chunk of a LogSegment - ReadCacheLineCount uint64 - LeaseRetryLimit uint64 - LeaseRetryDelay time.Duration - LeaseRetryDelayVariance uint8 - LeaseRetryExpBackoff float64 - SharedLeaseLimit uint64 - ExclusiveLeaseLimit uint64 - ExtentMapEntryLimit uint64 - DirtyLogSegmentLimit uint64 - DirtyFileLimit uint64 // TODO - obsolete this - MaxFlushSize uint64 - MaxFlushTime time.Duration - LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled - LogToConsole bool - TraceEnabled bool - HTTPServerIPAddr string - HTTPServerTCPPort uint16 - ReadDirPlusEnabled bool - XAttrEnabled bool - EntryDuration time.Duration - AttrDuration time.Duration - ReaddirMaxEntries uint64 - FUSEMaxBackground uint16 - FUSECongestionThreshhold uint16 - FUSEMaxWrite uint32 - FUSEAllowOther bool - RetryRPCPublicIPAddr string - RetryRPCPort uint16 - RetryRPCDeadlineIO time.Duration - RetryRPCKeepAlivePeriod time.Duration - RetryRPCCACertFilePath string -} - -type retryDelayElementStruct struct { - nominal time.Duration - variance time.Duration -} - -type fileInodeLeaseStateType uint32 - -const ( - fileInodeLeaseStateNone fileInodeLeaseStateType = iota - fileInodeLeaseStateSharedRequested - fileInodeLeaseStateSharedGranted - fileInodeLeaseStateSharedPromoting - fileInodeLeaseStateSharedReleasing - fileInodeLeaseStateExclusiveRequested - fileInodeLeaseStateExclusiveGranted - fileInodeLeaseStateExclusiveDemoting - fileInodeLeaseStateExclusiveReleasing -) - -type fileInodeLeaseRequestType uint32 - -const ( - fileInodeLeaseRequestShutdown fileInodeLeaseRequestType = iota - fileInodeLeaseRequestShared - fileInodeLeaseRequestExclusive - fileInodeLeaseRequestDemote - fileInodeLeaseRequestRelease -) - -// singleObjectExtentStruct is used for chunkedPutContextStruct.extentMap. -// -type singleObjectExtentStruct struct { - fileOffset uint64 // Key in chunkedPutContextStruct.extentMap - objectOffset uint64 - length uint64 -} - -// singleObjectExtentWithLinkStruct is used to represent a ReadPlanStep. A chunkedPutContext == nil -// indicates a zero-filled extent rather that a read from a LogSegment not yet persisted by Swift. -// -type singleObjectExtentWithLinkStruct struct { - fileOffset uint64 - objectOffset uint64 - length uint64 - chunkedPutContext *chunkedPutContextStruct // If == nil, implies a zero-filled extent/ReadPlanStep -} - -// multiObjectExtentStruct is used for both the fileInodeStruct.extentMap as well -// as for representing a ReadPlanStep. In this latter case, an objectName == "" -// indicates a zero-filled extent rather than a read from a LogSegment already -// persisted by Swift. -// -type multiObjectExtentStruct struct { - fileOffset uint64 // Key in fileInodeStruct.extentMap - containerName string - objectName string // If == "", implies a zero-filled extent/ReadPlanStep - objectOffset uint64 - length uint64 -} - -const ( - chunkedPutContextStateOpen uint8 = iota // Initial state indicating Chunked PUT is available to send a chunk - chunkedPutContextStateClosing // After a zero-length chunk is sent to close the Chunked PUT... awaiting http.StatusCreated - chunkedPutContextStateClosed // Chunked PUT received an http.StatusCreated... - // but we cannot yet merge it's ExtentMap updates because - // an as-yet un-closed Chunked PUT needs to do so first - chunkedPutContextExitReadPollingRate = time.Millisecond -) - -type chunkedPutContextStruct struct { - sync.WaitGroup // Used to await completion of performChunkedPut goroutine - containerName string // - objectName string // - extentMap sortedmap.LLRBTree // Key == singleObjectExtentStruct.fileOffset; Value == *singleObjectExtentStruct - buf []byte // - chunkedPutListElement *list.Element // FIFO Element of fileInodeStruct.chunkedPutList - fileInode *fileInodeStruct // - state uint8 // One of chunkedPutContextState{Open|Closing|Closed} - pos int // ObjectOffset just after last sent chunk - sendChan chan struct{} // Single element buffered chan to wake up *chunkedPutContextStruct.sendDaemon() - // will be closed to indicate a flush is requested - wakeChan chan struct{} // Single element buffered chan to wake up *chunkedPutContextStruct.Read() - // will be closed to indicate a flush is requested - inRead bool // Set when in Read() as a hint to Close() to help Read() cleanly exit - flushRequested bool // Set to remember that a flush has been requested of *chunkedPutContextStruct.Read() -} - -type fileInodeStruct struct { - inode.InodeNumber - cachedStat *jrpcfs.StatStruct // Maintained valid/coherent with ProxyFS (possibly dirty until flushed) - lockWaiters *list.List // List of chan struct{} lock waiters: - // If == nil, no lock requests - // If .Len() == 0, lock grant in progress or granted but no blocked lock requests - // If .Len() != 0, lock grant in progress or granted and other lock requests are waiting - leaseState fileInodeLeaseStateType // One of fileInodeLeaseState* - pendingLeaseInterrupt *jrpcfs.RPCInterruptType // If non-nil, either jrpcfs.RPCInterruptTypeDemote or jrpcfs.RPCInterruptTypeRelease - leaseListElement *list.Element // Element on one of {unleased|shared|exclusive}FileInodeCacheLRU - // On globals.unleasedFileInodeCacheLRU if leaseState one of: - // fileInodeLeaseStateNone - // fileInodeLeaseStateSharedReleasing - // fileInodeLeaseStateExclusiveReleasing - // On globals.sharedLeaseFileInodeCacheLRU if leaseState one of: - // fileInodeLeaseStateSharedRequested - // fileInodeLeaseStateSharedGranted - // fileInodeLeaseStateExclusiveDemoting - // On globals.exclusiveLeaseFileInodeCacheLRU if leaseState one of: - // fileInodeLeaseStateSharedPromoting - // fileInodeLeaseStateExclusiveRequested - // fileInodeLeaseStateExclusiveGranted - extentMap sortedmap.LLRBTree // Key == multiObjectExtentStruct.fileOffset; Value == *multiObjectExtentStruct - chunkedPutList *list.List // FIFO List of chunkedPutContextStruct's - flushInProgress bool // Serializes (& singularizes) explicit Flush requests - chunkedPutFlushWaiterList *list.List // List of *sync.WaitGroup's for those awaiting an explicit Flush - // Note: These waiters cannot be holding fileInodeStruct.Lock - dirtyListElement *list.Element // Element on globals.fileInodeDirtyList (or nil) -} - -type fhSetType map[uint64]struct{} - -type logSegmentCacheElementStateType uint8 - -const ( - logSegmentCacheElementStateGetIssued logSegmentCacheElementStateType = iota - logSegmentCacheElementStateGetSuccessful - logSegmentCacheElementStateGetFailed // In which case it must not be in LLRBTree nor LRU -) - -type logSegmentCacheElementKeyStruct struct { - logSegmentNumber uint64 // Converted from logSegmentCacheElementStruct.objectName - cacheLineTag uint64 // == logSegmentCacheElementStruct.offset / globals.config.ReadCacheLineSize -} - -type logSegmentCacheElementStruct struct { - sync.WaitGroup // Used by those awaiting GET result - state logSegmentCacheElementStateType // if logSegmentCacheElementStateGetIssued - containerName string - objectName string - startingOffset uint64 - cacheLRUElement *list.Element // Element on globals.logSegmentCacheLRU - buf []byte -} - -type authPlugInControlStruct struct { - cmd *exec.Cmd - stdinPipe io.WriteCloser - stdoutPipe io.ReadCloser - stderrPipe io.ReadCloser - stdoutChan chan []byte - stderrChan chan []byte - wg sync.WaitGroup -} - -// metricsStruct contains Prometheus-styled field names to be output by serveGetOfMetrics(). -// -// In order to utilize Go Reflection, these field names must be capitalized (i.e. Global). - -type metricsStruct struct { - FUSE_DoLookup_calls uint64 - FUSE_DoForget_calls uint64 - FUSE_DoGetAttr_calls uint64 - FUSE_DoSetAttr_calls uint64 - FUSE_DoReadLink_calls uint64 - FUSE_DoSymLink_calls uint64 - FUSE_DoMkNod_calls uint64 - FUSE_DoMkDir_calls uint64 - FUSE_DoUnlink_calls uint64 - FUSE_DoRmDir_calls uint64 - FUSE_DoRename_calls uint64 - FUSE_DoLink_calls uint64 - FUSE_DoOpen_calls uint64 - FUSE_DoRead_calls uint64 - FUSE_DoWrite_calls uint64 - FUSE_DoStatFS_calls uint64 - FUSE_DoRelease_calls uint64 - FUSE_DoFSync_calls uint64 - FUSE_DoSetXAttr_calls uint64 - FUSE_DoGetXAttr_calls uint64 - FUSE_DoListXAttr_calls uint64 - FUSE_DoRemoveXAttr_calls uint64 - FUSE_DoFlush_calls uint64 - FUSE_DoInit_calls uint64 - FUSE_DoOpenDir_calls uint64 - FUSE_DoReadDir_calls uint64 - FUSE_DoReleaseDir_calls uint64 - FUSE_DoFSyncDir_calls uint64 - FUSE_DoGetLK_calls uint64 - FUSE_DoSetLK_calls uint64 - FUSE_DoSetLKW_calls uint64 - FUSE_DoAccess_calls uint64 - FUSE_DoCreate_calls uint64 - FUSE_DoInterrupt_calls uint64 - FUSE_DoBMap_calls uint64 - FUSE_DoDestroy_calls uint64 - FUSE_DoPoll_calls uint64 - FUSE_DoBatchForget_calls uint64 - FUSE_DoFAllocate_calls uint64 - FUSE_DoReadDirPlus_calls uint64 - FUSE_DoRename2_calls uint64 - FUSE_DoLSeek_calls uint64 - - FUSE_DoRead_bytes uint64 - FUSE_DoWrite_bytes uint64 - - FUSE_DoReadDir_entries uint64 - FUSE_DoReadDirPlus_entries uint64 - - FUSE_DoSetXAttr_bytes uint64 - FUSE_DoGetXAttr_bytes uint64 - FUSE_DoListXAttr_names uint64 - - FUSE_DoBatchForget_nodes uint64 - - ReadCacheHits uint64 - ReadCacheMisses uint64 - - LogSegmentPUTs uint64 - LogSegmentPUTReadHits uint64 - - LeaseRequests_Shared uint64 - LeaseRequests_Promote uint64 - LeaseRequests_Exclusive uint64 - LeaseRequests_Demote uint64 - LeaseRequests_Release uint64 - - LeaseInterrupts_Unmount uint64 - LeaseInterrupts_Demote uint64 - LeaseInterrupts_Release uint64 - - HTTPRequests uint64 - HTTPRequestSubmissionFailures uint64 - HTTPRequestResponseBodyCorruptions uint64 - HTTPRequestRetryLimitExceededCount uint64 - HTTPRequestsRequiringReauthorization uint64 - HTTPRequestRetries uint64 - HTTPRequestsInFlight uint64 -} - -type statsStruct struct { - FUSEDoReadBytes bucketstats.BucketLog2Round - FUSEDoWriteBytes bucketstats.BucketLog2Round - - LogSegmentGetUsec bucketstats.BucketLog2Round - - LogSegmentPutBytes bucketstats.BucketLog2Round - - LeaseRequests_Shared_Usec bucketstats.BucketLog2Round - LeaseRequests_Promote_Usec bucketstats.BucketLog2Round - LeaseRequests_Exclusive_Usec bucketstats.BucketLog2Round - LeaseRequests_Demote_Usec bucketstats.BucketLog2Round - LeaseRequests_Release_Usec bucketstats.BucketLog2Round -} - -type globalsStruct struct { - sync.Mutex - config configStruct - logFile *os.File // == nil if configStruct.LogFilePath == "" - retryRPCCACertPEM []byte - retryRPCClient *retryrpc.Client - entryValidSec uint64 - entryValidNSec uint32 - attrValidSec uint64 - attrValidNSec uint32 - httpServer *http.Server - httpServerWG sync.WaitGroup - httpClient *http.Client - retryDelay []retryDelayElementStruct - authPlugInControl *authPlugInControlStruct - swiftAuthWaitGroup *sync.WaitGroup // Protected by sync.Mutex of globalsStruct - swiftAuthToken string // Protected by swiftAuthWaitGroup - swiftStorageURL string // Protected by swiftAuthWaitGroup - mountID jrpcfs.MountIDAsString - fissionErrChan chan error - fissionVolume fission.Volume - fuseConn *fuse.Conn - jrpcLastID uint64 - fileInodeMap map[inode.InodeNumber]*fileInodeStruct - fileInodeDirtyList *list.List // LRU of fileInode's with non-empty chunkedPutList - fileInodeDirtyLogSegmentChan chan struct{} // Limits # of in-flight LogSegment Chunked PUTs - unleasedFileInodeCacheLRU *list.List // Front() is oldest fileInodeStruct.leaseListElement - sharedLeaseFileInodeCacheLRU *list.List // Front() is oldest fileInodeStruct.leaseListElement - exclusiveLeaseFileInodeCacheLRU *list.List // Front() is oldest fileInodeStruct.leaseListElement - fhToOpenRequestMap map[uint64]uint32 // Key == FH; Value == {Create|Open}In.Flags - fhToInodeNumberMap map[uint64]uint64 // Key == FH; Value == InodeNumber - inodeNumberToFHMap map[uint64]fhSetType // Key == InodeNumber; Value == set of FH's - lastFH uint64 // Valid FH's start at 1 - logSegmentCacheMap map[logSegmentCacheElementKeyStruct]*logSegmentCacheElementStruct - logSegmentCacheLRU *list.List // Front() is oldest logSegmentCacheElementStruct.cacheLRUElement - metrics *metricsStruct - stats *statsStruct -} - -var globals globalsStruct - -func initializeGlobals(confMap conf.ConfMap) { - var ( - configJSONified string - customTransport *http.Transport - defaultTransport *http.Transport - err error - fileInodeDirtyLogSegmentChanIndex uint64 - nextRetryDelay time.Duration - ok bool - plugInEnvValueSlice []string - retryIndex uint64 - ) - - // Default logging related globals - - globals.config.LogFilePath = "" - globals.config.LogToConsole = true - globals.logFile = nil - - // Process resultant confMap - - globals.config.FUSEVolumeName, err = confMap.FetchOptionValueString("Agent", "FUSEVolumeName") - if nil != err { - logFatal(err) - } - - globals.config.FUSEMountPointPath, err = confMap.FetchOptionValueString("Agent", "FUSEMountPointPath") - if nil != err { - logFatal(err) - } - - globals.config.FUSEUnMountRetryDelay, err = confMap.FetchOptionValueDuration("Agent", "FUSEUnMountRetryDelay") - if nil != err { - logFatal(err) - } - - globals.config.FUSEUnMountRetryCap, err = confMap.FetchOptionValueUint64("Agent", "FUSEUnMountRetryCap") - if nil != err { - logFatal(err) - } - - globals.config.PlugInPath, err = confMap.FetchOptionValueString("Agent", "PlugInPath") - if nil != err { - logFatal(err) - } - - globals.config.PlugInEnvName, err = confMap.FetchOptionValueString("Agent", "PlugInEnvName") - if nil != err { - logFatal(err) - } - - err = confMap.VerifyOptionIsMissing("Agent", "PlugInEnvValue") - if nil == err { - globals.config.PlugInEnvValue = "" - } else { - plugInEnvValueSlice, err = confMap.FetchOptionValueStringSlice("Agent", "PlugInEnvValue") - if nil != err { - logFatal(err) - } else { - switch len(plugInEnvValueSlice) { - case 0: - globals.config.PlugInEnvValue = "" - case 1: - globals.config.PlugInEnvValue = plugInEnvValueSlice[0] - default: - log.Fatalf("[Agent]PlugInEnvValue must be missing, empty, or single-valued: %#v", plugInEnvValueSlice) - } - } - } - - globals.config.SwiftTimeout, err = confMap.FetchOptionValueDuration("Agent", "SwiftTimeout") - if nil != err { - logFatal(err) - } - - globals.config.SwiftRetryLimit, err = confMap.FetchOptionValueUint64("Agent", "SwiftRetryLimit") - if nil != err { - logFatal(err) - } - - globals.config.SwiftRetryDelay, err = confMap.FetchOptionValueDuration("Agent", "SwiftRetryDelay") - if nil != err { - logFatal(err) - } - - globals.config.SwiftRetryDelayVariance, err = confMap.FetchOptionValueUint8("Agent", "SwiftRetryDelayVariance") - if nil != err { - logFatal(err) - } - if 0 == globals.config.SwiftRetryDelayVariance { - err = fmt.Errorf("[Agent]SwiftRetryDelayVariance must be > 0") - logFatal(err) - } - if 100 < globals.config.SwiftRetryDelayVariance { - err = fmt.Errorf("[Agent]SwiftRetryDelayVariance (%v) must be <= 100", globals.config.SwiftRetryDelayVariance) - logFatal(err) - } - - globals.config.SwiftRetryExpBackoff, err = confMap.FetchOptionValueFloat64("Agent", "SwiftRetryExpBackoff") - if nil != err { - logFatal(err) - } - - globals.config.SwiftConnectionPoolSize, err = confMap.FetchOptionValueUint64("Agent", "SwiftConnectionPoolSize") - if nil != err { - logFatal(err) - } - - globals.config.FetchExtentsFromFileOffset, err = confMap.FetchOptionValueUint64("Agent", "FetchExtentsFromFileOffset") - if nil != err { - logFatal(err) - } - - globals.config.FetchExtentsBeforeFileOffset, err = confMap.FetchOptionValueUint64("Agent", "FetchExtentsBeforeFileOffset") - if nil != err { - logFatal(err) - } - - globals.config.ReadCacheLineSize, err = confMap.FetchOptionValueUint64("Agent", "ReadCacheLineSize") - if nil != err { - logFatal(err) - } - - globals.config.ReadCacheLineCount, err = confMap.FetchOptionValueUint64("Agent", "ReadCacheLineCount") - if nil != err { - logFatal(err) - } - - globals.config.LeaseRetryLimit, err = confMap.FetchOptionValueUint64("Agent", "LeaseRetryLimit") - if nil != err { - logFatal(err) - } - - globals.config.LeaseRetryDelay, err = confMap.FetchOptionValueDuration("Agent", "LeaseRetryDelay") - if nil != err { - logFatal(err) - } - - globals.config.LeaseRetryDelayVariance, err = confMap.FetchOptionValueUint8("Agent", "LeaseRetryDelayVariance") - if nil != err { - logFatal(err) - } - if 0 == globals.config.LeaseRetryDelayVariance { - err = fmt.Errorf("[Agent]LeaseRetryDelayVariance must be > 0") - logFatal(err) - } - if 100 < globals.config.LeaseRetryDelayVariance { - err = fmt.Errorf("[Agent]LeaseRetryDelayVariance (%v) must be <= 100", globals.config.LeaseRetryDelayVariance) - logFatal(err) - } - - globals.config.LeaseRetryExpBackoff, err = confMap.FetchOptionValueFloat64("Agent", "LeaseRetryExpBackoff") - if nil != err { - logFatal(err) - } - - globals.config.SharedLeaseLimit, err = confMap.FetchOptionValueUint64("Agent", "SharedLeaseLimit") - if nil != err { - logFatal(err) - } - - globals.config.ExclusiveLeaseLimit, err = confMap.FetchOptionValueUint64("Agent", "ExclusiveLeaseLimit") - if nil != err { - logFatal(err) - } - - globals.config.ExtentMapEntryLimit, err = confMap.FetchOptionValueUint64("Agent", "ExtentMapEntryLimit") - if nil != err { - logFatal(err) - } - - globals.config.DirtyLogSegmentLimit, err = confMap.FetchOptionValueUint64("Agent", "DirtyLogSegmentLimit") - if nil != err { - logFatal(err) - } - - globals.config.DirtyFileLimit, err = confMap.FetchOptionValueUint64("Agent", "DirtyFileLimit") - if nil != err { - logFatal(err) // TODO - obsolete this - } - - globals.config.MaxFlushSize, err = confMap.FetchOptionValueUint64("Agent", "MaxFlushSize") - if nil != err { - logFatal(err) - } - - globals.config.MaxFlushTime, err = confMap.FetchOptionValueDuration("Agent", "MaxFlushTime") - if nil != err { - logFatal(err) - } - - err = confMap.VerifyOptionValueIsEmpty("Agent", "LogFilePath") - if nil == err { - globals.config.LogFilePath = "" - } else { - globals.config.LogFilePath, err = confMap.FetchOptionValueString("Agent", "LogFilePath") - if nil != err { - logFatal(err) - } - } - - globals.config.LogToConsole, err = confMap.FetchOptionValueBool("Agent", "LogToConsole") - if nil != err { - logFatal(err) - } - - globals.config.TraceEnabled, err = confMap.FetchOptionValueBool("Agent", "TraceEnabled") - if nil != err { - logFatal(err) - } - - globals.config.HTTPServerIPAddr, err = confMap.FetchOptionValueString("Agent", "HTTPServerIPAddr") - if nil != err { - logFatal(err) - } - globals.config.HTTPServerTCPPort, err = confMap.FetchOptionValueUint16("Agent", "HTTPServerTCPPort") - if nil != err { - logFatal(err) - } - - globals.config.AttrDuration, err = confMap.FetchOptionValueDuration("Agent", "AttrDuration") - if nil != err { - logFatal(err) - } - - globals.config.ReadDirPlusEnabled, err = confMap.FetchOptionValueBool("Agent", "ReadDirPlusEnabled") - if nil != err { - logFatal(err) - } - - globals.config.XAttrEnabled, err = confMap.FetchOptionValueBool("Agent", "XAttrEnabled") - if nil != err { - logFatal(err) - } - - globals.config.EntryDuration, err = confMap.FetchOptionValueDuration("Agent", "EntryDuration") - if nil != err { - logFatal(err) - } - - globals.config.AttrDuration, err = confMap.FetchOptionValueDuration("Agent", "AttrDuration") - if nil != err { - logFatal(err) - } - - globals.config.ReaddirMaxEntries, err = confMap.FetchOptionValueUint64("Agent", "ReaddirMaxEntries") - if nil != err { - logFatal(err) - } - - globals.config.FUSEMaxBackground, err = confMap.FetchOptionValueUint16("Agent", "FUSEMaxBackground") - if nil != err { - logFatal(err) - } - - globals.config.FUSECongestionThreshhold, err = confMap.FetchOptionValueUint16("Agent", "FUSECongestionThreshhold") - if nil != err { - logFatal(err) - } - - globals.config.FUSEMaxWrite, err = confMap.FetchOptionValueUint32("Agent", "FUSEMaxWrite") - if nil != err { - logFatal(err) - } - - globals.config.FUSEAllowOther, err = confMap.FetchOptionValueBool("Agent", "FUSEAllowOther") - if nil != err { - logFatal(err) - } - - globals.config.RetryRPCPublicIPAddr, err = confMap.FetchOptionValueString("Agent", "RetryRPCPublicIPAddr") - if nil != err { - logFatal(err) - } - - globals.config.RetryRPCPort, err = confMap.FetchOptionValueUint16("Agent", "RetryRPCPort") - if nil != err { - logFatal(err) - } - - globals.config.RetryRPCDeadlineIO, err = confMap.FetchOptionValueDuration("Agent", "RetryRPCDeadlineIO") - if nil != err { - logFatal(err) - } - - globals.config.RetryRPCKeepAlivePeriod, err = confMap.FetchOptionValueDuration("Agent", "RetryRPCKeepAlivePeriod") - if nil != err { - logFatal(err) - } - - globals.config.RetryRPCCACertFilePath, err = confMap.FetchOptionValueString("Agent", "RetryRPCCACertFilePath") - if nil != err { - globals.config.RetryRPCCACertFilePath = "" - } - - configJSONified = utils.JSONify(globals.config, true) - - logInfof("\n%s", configJSONified) - - if "" == globals.config.RetryRPCCACertFilePath { - globals.retryRPCCACertPEM = nil - } else { - globals.retryRPCCACertPEM, err = ioutil.ReadFile(globals.config.RetryRPCCACertFilePath) - if nil != err { - logFatal(err) - } - } - - globals.entryValidSec, globals.entryValidNSec = nsToUnixTime(uint64(globals.config.EntryDuration)) - globals.attrValidSec, globals.attrValidNSec = nsToUnixTime(uint64(globals.config.AttrDuration)) - - defaultTransport, ok = http.DefaultTransport.(*http.Transport) - if !ok { - log.Fatalf("http.DefaultTransport not a *http.Transport") - } - - customTransport = &http.Transport{ // Up-to-date as of Golang 1.11 - Proxy: defaultTransport.Proxy, - DialContext: defaultTransport.DialContext, - Dial: defaultTransport.Dial, - DialTLS: defaultTransport.DialTLS, - TLSClientConfig: defaultTransport.TLSClientConfig, - TLSHandshakeTimeout: globals.config.SwiftTimeout, - DisableKeepAlives: false, - DisableCompression: defaultTransport.DisableCompression, - MaxIdleConns: int(globals.config.SwiftConnectionPoolSize), - MaxIdleConnsPerHost: int(globals.config.SwiftConnectionPoolSize), - MaxConnsPerHost: int(globals.config.SwiftConnectionPoolSize), - IdleConnTimeout: globals.config.SwiftTimeout, - ResponseHeaderTimeout: globals.config.SwiftTimeout, - ExpectContinueTimeout: globals.config.SwiftTimeout, - TLSNextProto: defaultTransport.TLSNextProto, - ProxyConnectHeader: defaultTransport.ProxyConnectHeader, - MaxResponseHeaderBytes: defaultTransport.MaxResponseHeaderBytes, - } - - globals.httpClient = &http.Client{ - Transport: customTransport, - Timeout: globals.config.SwiftTimeout, - } - - globals.retryDelay = make([]retryDelayElementStruct, globals.config.SwiftRetryLimit) - - nextRetryDelay = globals.config.SwiftRetryDelay - - for retryIndex = 0; retryIndex < globals.config.SwiftRetryLimit; retryIndex++ { - globals.retryDelay[retryIndex].nominal = nextRetryDelay - globals.retryDelay[retryIndex].variance = nextRetryDelay * time.Duration(globals.config.SwiftRetryDelayVariance) / time.Duration(100) - nextRetryDelay = time.Duration(float64(nextRetryDelay) * globals.config.SwiftRetryExpBackoff) - } - - globals.authPlugInControl = nil - - globals.swiftAuthWaitGroup = nil - globals.swiftAuthToken = "" - globals.swiftStorageURL = "" - - globals.fissionErrChan = make(chan error) - - globals.jrpcLastID = 1 - - globals.fileInodeMap = make(map[inode.InodeNumber]*fileInodeStruct) - - globals.fileInodeDirtyList = list.New() - - globals.fileInodeDirtyLogSegmentChan = make(chan struct{}, globals.config.DirtyLogSegmentLimit) - - for fileInodeDirtyLogSegmentChanIndex = 0; fileInodeDirtyLogSegmentChanIndex < globals.config.DirtyLogSegmentLimit; fileInodeDirtyLogSegmentChanIndex++ { - globals.fileInodeDirtyLogSegmentChan <- struct{}{} - } - - globals.unleasedFileInodeCacheLRU = list.New() - globals.sharedLeaseFileInodeCacheLRU = list.New() - globals.exclusiveLeaseFileInodeCacheLRU = list.New() - - globals.fhToOpenRequestMap = make(map[uint64]uint32) - globals.fhToInodeNumberMap = make(map[uint64]uint64) - globals.inodeNumberToFHMap = make(map[uint64]fhSetType) - - globals.lastFH = 0 - - globals.logSegmentCacheMap = make(map[logSegmentCacheElementKeyStruct]*logSegmentCacheElementStruct) - globals.logSegmentCacheLRU = list.New() - - globals.metrics = &metricsStruct{} - globals.stats = &statsStruct{} - - bucketstats.Register("PFSAgent", "", globals.stats) -} - -func uninitializeGlobals() { - bucketstats.UnRegister("PFSAgent", "") - - globals.logFile = nil - globals.retryRPCCACertPEM = nil - globals.retryRPCClient = nil - globals.entryValidSec = 0 - globals.entryValidNSec = 0 - globals.attrValidSec = 0 - globals.attrValidNSec = 0 - globals.httpServer = nil - globals.httpClient = nil - globals.retryDelay = nil - globals.authPlugInControl = nil - globals.swiftAuthWaitGroup = nil - globals.swiftAuthToken = "" - globals.swiftStorageURL = "" - globals.fissionErrChan = nil - globals.fissionVolume = nil - globals.fuseConn = nil - globals.jrpcLastID = 0 - globals.fileInodeMap = nil - globals.fileInodeDirtyList = nil - globals.fileInodeDirtyLogSegmentChan = nil - globals.unleasedFileInodeCacheLRU = nil // TODO: Obsolete this - globals.sharedLeaseFileInodeCacheLRU = nil // TODO: Obsolete this - globals.exclusiveLeaseFileInodeCacheLRU = nil // TODO: Obsolete this - globals.fhToOpenRequestMap = nil - globals.fhToInodeNumberMap = nil - globals.inodeNumberToFHMap = nil - globals.lastFH = 0 - globals.logSegmentCacheMap = nil - globals.logSegmentCacheLRU = nil - globals.metrics = nil - globals.stats = nil -} diff --git a/pfsagentd/html_templates.go b/pfsagentd/html_templates.go deleted file mode 100644 index f0641221..00000000 --- a/pfsagentd/html_templates.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -// To use: fmt.Sprintf(indexDotHTMLTemplate, globals.ipAddrTCPPort) -const indexDotHTMLTemplate string = ` - - - - PFSAgent - %[1]v - - - config -
- leases -
- metrics -
- stats -
- version - - -` diff --git a/pfsagentd/http_server.go b/pfsagentd/http_server.go deleted file mode 100644 index faa8833d..00000000 --- a/pfsagentd/http_server.go +++ /dev/null @@ -1,431 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "container/list" - "context" - "encoding/json" - "fmt" - "log" - "net" - "net/http" - "net/http/pprof" - "reflect" - "runtime" - "strconv" - "strings" - "sync/atomic" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/version" -) - -type leaseReportStruct struct { - MountID string - None []string // inode.InodeNumber in 16-digit Hex (no leading "0x") - SharedRequested []string - SharedGranted []string - SharedPromoting []string - SharedReleasing []string - ExclusiveRequested []string - ExclusiveGranted []string - ExclusiveDemoting []string - ExclusiveReleasing []string -} - -func serveHTTP() { - var ( - ipAddrTCPPort string - ) - - ipAddrTCPPort = net.JoinHostPort(globals.config.HTTPServerIPAddr, strconv.Itoa(int(globals.config.HTTPServerTCPPort))) - - globals.httpServer = &http.Server{ - Addr: ipAddrTCPPort, - Handler: &globals, - } - - globals.httpServerWG.Add(1) - - go func() { - var ( - err error - ) - - err = globals.httpServer.ListenAndServe() - if http.ErrServerClosed != err { - log.Fatalf("httpServer.ListenAndServe() exited unexpectedly: %v", err) - } - - globals.httpServerWG.Done() - }() -} - -func unserveHTTP() { - var ( - err error - ) - - err = globals.httpServer.Shutdown(context.TODO()) - if nil != err { - log.Fatalf("httpServer.Shutdown() returned with an error: %v", err) - } - - globals.httpServerWG.Wait() -} - -func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - switch request.Method { - case http.MethodGet: - serveGet(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func serveGet(responseWriter http.ResponseWriter, request *http.Request) { - var ( - path string - ) - - path = strings.TrimRight(request.URL.Path, "/") - - switch { - case "" == path: - serveGetOfIndexDotHTML(responseWriter, request) - case "/config" == path: - serveGetOfConfig(responseWriter, request) - case "/debug/pprof/cmdline" == path: - pprof.Cmdline(responseWriter, request) - case "/debug/pprof/profile" == path: - pprof.Profile(responseWriter, request) - case "/debug/pprof/symbol" == path: - pprof.Symbol(responseWriter, request) - case "/debug/pprof/trace" == path: - pprof.Trace(responseWriter, request) - case strings.HasPrefix(path, "/debug/pprof"): - pprof.Index(responseWriter, request) - case "index.html" == path: - serveGetOfIndexDotHTML(responseWriter, request) - case "/leases" == path: - serveGetOfLeases(responseWriter, request) - case "/metrics" == path: - serveGetOfMetrics(responseWriter, request) - case "/stats" == path: - serveGetOfStats(responseWriter, request) - case "/version" == path: - serveGetOfVersion(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func serveGetOfConfig(responseWriter http.ResponseWriter, request *http.Request) { - var ( - confMapJSON bytes.Buffer - confMapJSONPacked []byte - ok bool - paramList []string - sendPackedConfig bool - ) - - paramList, ok = request.URL.Query()["compact"] - if ok { - if 0 == len(paramList) { - sendPackedConfig = false - } else { - sendPackedConfig = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - sendPackedConfig = false - } - - confMapJSONPacked, _ = json.Marshal(globals.config) - - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if sendPackedConfig { - _, _ = responseWriter.Write(confMapJSONPacked) - } else { - json.Indent(&confMapJSON, confMapJSONPacked, "", "\t") - _, _ = responseWriter.Write(confMapJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } -} - -func serveGetOfIndexDotHTML(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, net.JoinHostPort(globals.config.HTTPServerIPAddr, strconv.Itoa(int(globals.config.HTTPServerTCPPort)))))) -} - -func serveGetOfLeases(responseWriter http.ResponseWriter, request *http.Request) { - var ( - fileInode *fileInodeStruct - leaseListElement *list.Element - leaseReport *leaseReportStruct - leaseReportJSON bytes.Buffer - leaseReportJSONPacked []byte - ok bool - paramList []string - sendPackedLeaseReport bool - ) - - leaseReport = &leaseReportStruct{ - MountID: fmt.Sprintf("%s", globals.mountID), - None: make([]string, 0), - SharedRequested: make([]string, 0), - SharedGranted: make([]string, 0), - SharedPromoting: make([]string, 0), - SharedReleasing: make([]string, 0), - ExclusiveRequested: make([]string, 0), - ExclusiveGranted: make([]string, 0), - ExclusiveDemoting: make([]string, 0), - ExclusiveReleasing: make([]string, 0), - } - - globals.Lock() - - for leaseListElement = globals.unleasedFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { - fileInode = leaseListElement.Value.(*fileInodeStruct) - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - leaseReport.None = append(leaseReport.None, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateSharedReleasing: - leaseReport.SharedReleasing = append(leaseReport.SharedReleasing, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateExclusiveReleasing: - leaseReport.ExclusiveReleasing = append(leaseReport.ExclusiveReleasing, fmt.Sprintf("%016X", fileInode.InodeNumber)) - default: - logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.unleasedFileInodeCacheLRU", fileInode.leaseState) - } - } - - for leaseListElement = globals.sharedLeaseFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { - fileInode = leaseListElement.Value.(*fileInodeStruct) - switch fileInode.leaseState { - case fileInodeLeaseStateSharedRequested: - leaseReport.SharedRequested = append(leaseReport.SharedRequested, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateSharedGranted: - leaseReport.SharedGranted = append(leaseReport.SharedGranted, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateExclusiveDemoting: - leaseReport.ExclusiveDemoting = append(leaseReport.ExclusiveDemoting, fmt.Sprintf("%016X", fileInode.InodeNumber)) - default: - logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.sharedLeaseFileInodeCacheLRU", fileInode.leaseState) - } - } - - for leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { - fileInode = leaseListElement.Value.(*fileInodeStruct) - switch fileInode.leaseState { - case fileInodeLeaseStateSharedPromoting: - leaseReport.SharedPromoting = append(leaseReport.SharedPromoting, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateExclusiveRequested: - leaseReport.ExclusiveRequested = append(leaseReport.ExclusiveRequested, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateExclusiveGranted: - leaseReport.ExclusiveGranted = append(leaseReport.ExclusiveGranted, fmt.Sprintf("%016X", fileInode.InodeNumber)) - default: - logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.exclusiveLeaseFileInodeCacheLRU", fileInode.leaseState) - } - } - - globals.Unlock() - - paramList, ok = request.URL.Query()["compact"] - if ok { - if 0 == len(paramList) { - sendPackedLeaseReport = false - } else { - sendPackedLeaseReport = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - sendPackedLeaseReport = false - } - - leaseReportJSONPacked, _ = json.Marshal(leaseReport) - - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if sendPackedLeaseReport { - _, _ = responseWriter.Write(leaseReportJSONPacked) - } else { - json.Indent(&leaseReportJSON, leaseReportJSONPacked, "", "\t") - _, _ = responseWriter.Write(leaseReportJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } -} - -func serveGetOfMetrics(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - format string - i int - keyAsKey sortedmap.Key - keyAsString string - line string - longestKeyAsString int - longestValueAsString int - memStats runtime.MemStats - metricsFieldName string - metricsFieldValuePtr *uint64 - metricsLLRB sortedmap.LLRBTree - metricsLLRBLen int - metricsStructValue reflect.Value - metricsValue reflect.Value - ok bool - pauseNsAccumulator uint64 - valueAsString string - valueAsValue sortedmap.Value - ) - - runtime.ReadMemStats(&memStats) - - metricsLLRB = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - // General statistics. - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Alloc", memStats.Alloc) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_TotalAlloc", memStats.TotalAlloc) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Sys", memStats.Sys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Lookups", memStats.Lookups) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Mallocs", memStats.Mallocs) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Frees", memStats.Frees) - - // Main allocation heap statistics. - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapAlloc", memStats.HeapAlloc) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapSys", memStats.HeapSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapIdle", memStats.HeapIdle) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapInuse", memStats.HeapInuse) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapReleased", memStats.HeapReleased) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapObjects", memStats.HeapObjects) - - // Low-level fixed-size structure allocator statistics. - // Inuse is bytes used now. - // Sys is bytes obtained from system. - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_StackInuse", memStats.StackInuse) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_StackSys", memStats.StackSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MSpanInuse", memStats.MSpanInuse) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MSpanSys", memStats.MSpanSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MCacheInuse", memStats.MCacheInuse) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MCacheSys", memStats.MCacheSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_BuckHashSys", memStats.BuckHashSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_GCSys", memStats.GCSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_OtherSys", memStats.OtherSys) - - // Garbage collector statistics (fixed portion). - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_LastGC", memStats.LastGC) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseTotalNs", memStats.PauseTotalNs) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_NumGC", uint64(memStats.NumGC)) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_GCCPUPercentage", uint64(100.0*memStats.GCCPUFraction)) - - // Garbage collector statistics (go_runtime_MemStats_PauseAverageNs). - if 0 == memStats.NumGC { - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", 0) - } else { - pauseNsAccumulator = 0 - if memStats.NumGC < 255 { - for i = 0; i < int(memStats.NumGC); i++ { - pauseNsAccumulator += memStats.PauseNs[i] - } - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", pauseNsAccumulator/uint64(memStats.NumGC)) - } else { - for i = 0; i < 256; i++ { - pauseNsAccumulator += memStats.PauseNs[i] - } - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", pauseNsAccumulator/256) - } - } - - // Add in locally generated metrics - - metricsStructValue = reflect.Indirect(reflect.ValueOf(globals.metrics)) - metricsValue = reflect.ValueOf(globals.metrics).Elem() - - for i = 0; i < metricsStructValue.NumField(); i++ { - metricsFieldName = metricsStructValue.Type().Field(i).Name - metricsFieldValuePtr = metricsValue.Field(i).Addr().Interface().(*uint64) - insertInMetricsLLRB(metricsLLRB, metricsFieldName, atomic.LoadUint64(metricsFieldValuePtr)) - } - - // Produce sorted and column-aligned response - - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - metricsLLRBLen, err = metricsLLRB.Len() - if nil != err { - logFatalf("metricsLLRB.Len() failed: %v", err) - } - - longestKeyAsString = 0 - longestValueAsString = 0 - - for i = 0; i < metricsLLRBLen; i++ { - keyAsKey, valueAsValue, ok, err = metricsLLRB.GetByIndex(i) - if nil != err { - logFatalf("llrb.GetByIndex(%v) failed: %v", i, err) - } - if !ok { - logFatalf("llrb.GetByIndex(%v) returned ok == false", i) - } - keyAsString = keyAsKey.(string) - valueAsString = valueAsValue.(string) - if len(keyAsString) > longestKeyAsString { - longestKeyAsString = len(keyAsString) - } - if len(valueAsString) > longestValueAsString { - longestValueAsString = len(valueAsString) - } - } - - format = fmt.Sprintf("%%-%vs %%%vs\n", longestKeyAsString, longestValueAsString) - - for i = 0; i < metricsLLRBLen; i++ { - keyAsKey, valueAsValue, ok, err = metricsLLRB.GetByIndex(i) - if nil != err { - logFatalf("llrb.GetByIndex(%v) failed: %v", i, err) - } - if !ok { - logFatalf("llrb.GetByIndex(%v) returned ok == false", i) - } - keyAsString = keyAsKey.(string) - valueAsString = valueAsValue.(string) - line = fmt.Sprintf(format, keyAsString, valueAsString) - _, _ = responseWriter.Write([]byte(line)) - } -} - -func serveGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*"))) -} - -func insertInMetricsLLRB(metricsLLRB sortedmap.LLRBTree, metricKey string, metricValueAsUint64 uint64) { - var ( - err error - metricValueAsString string - ok bool - ) - - metricValueAsString = fmt.Sprintf("%v", metricValueAsUint64) - - ok, err = metricsLLRB.Put(metricKey, metricValueAsString) - if nil != err { - logFatalf("metricsLLRB.Put(%v, %v) failed: %v", metricKey, metricValueAsString, err) - } - if !ok { - logFatalf("metricsLLRB.Put(%v, %v) returned ok == false", metricKey, metricValueAsString) - } -} - -func serveGetOfVersion(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(version.ProxyFSVersion)) -} diff --git a/pfsagentd/jrpc.go b/pfsagentd/jrpc.go deleted file mode 100644 index 07c57856..00000000 --- a/pfsagentd/jrpc.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "encoding/json" - "fmt" -) - -type jrpcRequestMethodAndIDStruct struct { - Method string `json:"method"` - ID uint64 `json:"id"` -} - -type jrpcRequestStruct struct { - JSONrpc string `json:"jsonrpc"` - Method string `json:"method"` - ID uint64 `json:"id"` - Params [1]interface{} `json:"params"` -} - -type jrpcResponseIDStruct struct { - ID uint64 `json:"id"` -} - -type jrpcResponseIDAndErrorStruct struct { - ID uint64 `json:"id"` - Error string `json:"error"` -} - -type jrpcResponseNoErrorStruct struct { - ID uint64 `json:"id"` - Result interface{} `json:"result"` -} - -type jrpcResponseWithErrorStruct struct { - ID uint64 `json:"id"` - Error string `json:"error"` - Result interface{} `json:"result"` -} - -func jrpcMarshalRequest(requestMethod string, request interface{}) (requestID uint64, requestBuf []byte, marshalErr error) { - var ( - jrpcRequest *jrpcRequestStruct - ) - - globals.Lock() - requestID = globals.jrpcLastID + 1 - globals.jrpcLastID = requestID - globals.Unlock() - - jrpcRequest = &jrpcRequestStruct{ - JSONrpc: "2.0", - Method: requestMethod, - ID: requestID, - Params: [1]interface{}{request}, - } - - requestBuf, marshalErr = json.Marshal(jrpcRequest) - - return -} - -func jrpcMarshalResponse(requestID uint64, responseError error, response interface{}) (responseBuf []byte, marshalErr error) { - var ( - jrpcResponse interface{} - ) - - if nil == responseError { - if nil == response { - jrpcResponse = &jrpcResponseIDStruct{ - ID: requestID, - } - } else { - jrpcResponse = &jrpcResponseNoErrorStruct{ - ID: requestID, - Result: response, - } - } - } else { - if nil == response { - jrpcResponse = &jrpcResponseIDAndErrorStruct{ - ID: requestID, - Error: responseError.Error(), - } - } else { - jrpcResponse = &jrpcResponseWithErrorStruct{ - ID: requestID, - Error: responseError.Error(), - Result: response, - } - } - } - - responseBuf, marshalErr = json.Marshal(jrpcResponse) - - return -} - -func jrpcUnmarshalRequestForMethodAndID(requestBuf []byte) (requestMethod string, requestID uint64, unmarshalErr error) { - var ( - jrpcRequest *jrpcRequestMethodAndIDStruct - ) - - jrpcRequest = &jrpcRequestMethodAndIDStruct{} - - unmarshalErr = json.Unmarshal(requestBuf, jrpcRequest) - - if nil == unmarshalErr { - requestMethod = jrpcRequest.Method - requestID = jrpcRequest.ID - } - - return -} - -func jrpcUnmarshalRequest(requestID uint64, requestBuf []byte, request interface{}) (unmarshalErr error) { - var ( - jrpcRequest *jrpcRequestStruct - ) - - jrpcRequest = &jrpcRequestStruct{ - Params: [1]interface{}{request}, - } - - unmarshalErr = json.Unmarshal(requestBuf, jrpcRequest) - - if (nil == unmarshalErr) && (requestID != jrpcRequest.ID) { - unmarshalErr = fmt.Errorf("requestID mismatch") - } - - return -} - -func jrpcUnmarshalResponseForIDAndError(responseBuf []byte) (requestID uint64, responseErr error, unmarshalErr error) { - var ( - jrpcResponse *jrpcResponseIDAndErrorStruct - ) - - jrpcResponse = &jrpcResponseIDAndErrorStruct{} - - unmarshalErr = json.Unmarshal(responseBuf, jrpcResponse) - - if nil == unmarshalErr { - requestID = jrpcResponse.ID - if "" == jrpcResponse.Error { - responseErr = nil - } else { - responseErr = fmt.Errorf("%s", jrpcResponse.Error) - } - } - - return -} - -func jrpcUnmarshalResponse(requestID uint64, responseBuf []byte, response interface{}) (unmarshalErr error) { - var ( - jrpcResponse *jrpcResponseWithErrorStruct - ) - - jrpcResponse = &jrpcResponseWithErrorStruct{ - Result: response, - } - - unmarshalErr = json.Unmarshal(responseBuf, jrpcResponse) - - if (nil == unmarshalErr) && (requestID != jrpcResponse.ID) { - unmarshalErr = fmt.Errorf("requestID mismatch") - } - - return -} diff --git a/pfsagentd/lease.go b/pfsagentd/lease.go deleted file mode 100644 index fe6212d6..00000000 --- a/pfsagentd/lease.go +++ /dev/null @@ -1,982 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "container/list" - "encoding/json" - "time" - - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/jrpcfs" -) - -func (dummy *globalsStruct) Interrupt(rpcInterruptBuf []byte) { - var ( - err error - fileInode *fileInodeStruct - ok bool - rpcInterrupt *jrpcfs.RPCInterrupt - ) - - rpcInterrupt = &jrpcfs.RPCInterrupt{} - - err = json.Unmarshal(rpcInterruptBuf, rpcInterrupt) - if nil != err { - logFatalf("(*globalsStruct).Interrupt() call to json.Unmarshal() failed: %v", err) - } - - switch rpcInterrupt.RPCInterruptType { - case jrpcfs.RPCInterruptTypeUnmount: - logFatalf("UNSUPPORTED: (*globalsStruct).Interrupt() received jrpcfs.RPCInterruptTypeUnmount") - case jrpcfs.RPCInterruptTypeDemote: - logFatalf("UNSUPPORTED: (*globalsStruct).Interrupt() received jrpcfs.RPCInterruptTypeDemote") - case jrpcfs.RPCInterruptTypeRelease: - logFatalf("UNSUPPORTED: (*globalsStruct).Interrupt() received jrpcfs.RPCInterruptTypeRelease") - default: - logFatalf("(*globalsStruct).Interrupt() received unknown rpcInterrupt.RPCInterruptType: %v", rpcInterrupt.RPCInterruptType) - } - - globals.Lock() - - fileInode, ok = globals.fileInodeMap[inode.InodeNumber(rpcInterrupt.InodeNumber)] - if !ok { - globals.Unlock() - return - } - - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - case fileInodeLeaseStateSharedRequested: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType - case fileInodeLeaseStateSharedGranted: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - if nil == fileInode.lockWaiters { - // Shared Lease... with nobody currently referencing it - fileInode.lockWaiters = list.New() - if jrpcfs.RPCInterruptTypeDemote == rpcInterrupt.RPCInterruptType { - // We can safely ignore it - } else { // jrpcfs.RPCInterruptTypeRelease == rpcInterrupt.RPCInterruptType - fileInode.leaseState = fileInodeLeaseStateSharedReleasing - _ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode) - globals.Unlock() - // TODO - invalidate cached state (cachedStat and extentMap) - // TODO - inform ProxyFS that we'd like to Release our Shared Lease - // TODO - finally service any waiters that have arrived (or delete lockWaiters) - } - return - } - fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType - case fileInodeLeaseStateSharedPromoting: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType - case fileInodeLeaseStateSharedReleasing: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType - case fileInodeLeaseStateExclusiveRequested: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType - case fileInodeLeaseStateExclusiveGranted: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - if nil == fileInode.lockWaiters { - // Exclusive Lease... with nobody currently referencing it - if jrpcfs.RPCInterruptTypeDemote == rpcInterrupt.RPCInterruptType { - fileInode.leaseState = fileInodeLeaseStateExclusiveDemoting - _ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.sharedLeaseFileInodeCacheLRU.PushBack(fileInode) - globals.Unlock() - // TODO - flush dirty state (chunkedPutList) - // TODO - inform ProxyFS that we'd like to Demote our Exclusive Lease - // TODO - finally service any waiters that have arrived (or delete lockWaiters) - } else { // jrpcfs.RPCInterruptTypeRelease == rpcInterrupt.RPCInterruptType - fileInode.leaseState = fileInodeLeaseStateExclusiveReleasing - _ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode) - globals.Unlock() - // TODO - flush dirty state (chunkedPutList) - // TODO - invalidate cached state (cachedStat and extentMap) - // TODO - inform ProxyFS that we'd like to Release our Exclusive Lease - // TODO - finally service any waiters that have arrived (or delete lockWaiters) - } - return - } - fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType - case fileInodeLeaseStateExclusiveDemoting: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType - case fileInodeLeaseStateExclusiveReleasing: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - fileInode.pendingLeaseInterrupt = &rpcInterrupt.RPCInterruptType - default: - logFatalf("(*globalsStruct).Interrupt() found unknown fileInode.leaseState: %v", fileInode.leaseState) - } - - // If fileInode.pendingLeaseInterrupt was set, it'll be serviced in (*fileInodeStruct).unlock() - - globals.Unlock() -} - -func lockInodeWithSharedLease(inodeNumber inode.InodeNumber) (fileInode *fileInodeStruct) { - var ( - err error - leaseReply *jrpcfs.LeaseReply - leaseRequest *jrpcfs.LeaseRequest - leaseRequestEndTime time.Time - leaseRequestStartTime time.Time - ok bool - waitChan chan struct{} - ) - - globals.Lock() - - fileInode, ok = globals.fileInodeMap[inodeNumber] - if !ok { - fileInode = &fileInodeStruct{ - InodeNumber: inodeNumber, - cachedStat: nil, - lockWaiters: nil, - leaseState: fileInodeLeaseStateNone, - pendingLeaseInterrupt: nil, - leaseListElement: nil, - extentMap: nil, - chunkedPutList: list.New(), - flushInProgress: false, - chunkedPutFlushWaiterList: list.New(), - dirtyListElement: nil, - } - - fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode) - globals.fileInodeMap[inodeNumber] = fileInode - } - - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - if nil == fileInode.lockWaiters { - // No Lease currently held... so ask for a Shared Lease - - fileInode.lockWaiters = list.New() - fileInode.leaseState = fileInodeLeaseStateSharedRequested - _ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.sharedLeaseFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeShared, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("lockInodeWithSharedLease() unable to obtain Shared Lease [case 1] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - // Record leaseReply.LeaseReplyType - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeShared: - fileInode.leaseState = fileInodeLeaseStateSharedGranted - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - case jrpcfs.LeaseReplyTypeExclusive: - fileInode.leaseState = fileInodeLeaseStateExclusiveGranted - _ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode) - default: - logFatalf("lockInodeWithSharedLease() unable to obtain Shared Lease [case 2] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - - // Now that we have either a Shared or Exclusive Lease, we are also the first in line to use it - so just return - - globals.Unlock() - - return - } - - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateSharedRequested: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateSharedGranted: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - if nil == fileInode.lockWaiters { - // We already have a Shared Lease and we are also the first in line to use it - - fileInode.lockWaiters = list.New() - - globals.Unlock() - - return - } - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateSharedPromoting: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateSharedReleasing: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateExclusiveRequested: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateExclusiveGranted: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - if nil == fileInode.lockWaiters { - // We already have an Exclusive Lease and we are also the first in line to use it - - fileInode.lockWaiters = list.New() - - globals.Unlock() - - return - } - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateExclusiveDemoting: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateExclusiveReleasing: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - default: - logFatalf("lockInodeWithSharedLease() found unknown fileInode.leaseState [case 1]: %v", fileInode.leaseState) - } - - // Time to block - - waitChan = make(chan struct{}) - _ = fileInode.lockWaiters.PushBack(waitChan) - globals.Unlock() - _ = <-waitChan - - // Time to check fileInode.leaseState again [(*fileInodeStruct).unlock() may have changed fileInode.leaseState) - - globals.Lock() - - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - // No Lease currently held... so ask for a Shared Lease - - fileInode.leaseState = fileInodeLeaseStateSharedRequested - _ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.sharedLeaseFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeShared, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("lockInodeWithSharedLease() unable to obtain Shared Lease [case 3] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - // Record leaseReply.LeaseReplyType - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeShared: - fileInode.leaseState = fileInodeLeaseStateSharedGranted - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - case jrpcfs.LeaseReplyTypeExclusive: - fileInode.leaseState = fileInodeLeaseStateExclusiveGranted - _ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode) - default: - logFatalf("lockInodeWithSharedLease() unable to obtain Shared Lease [case 4] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - - // Now that we have a Shared or Exclusive Lease, we are also the first in line to use it - so we can grant the Lock - case fileInodeLeaseStateSharedRequested: - logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedRequested") - case fileInodeLeaseStateSharedGranted: - // Now that we have a Shared Lease, we are also the first in line to use it - so we can grant the Lock - case fileInodeLeaseStateSharedPromoting: - logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedPromoting") - case fileInodeLeaseStateSharedReleasing: - logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedReleasing") - case fileInodeLeaseStateExclusiveRequested: - logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveRequested") - case fileInodeLeaseStateExclusiveGranted: - // Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock - case fileInodeLeaseStateExclusiveDemoting: - logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveDemoting") - case fileInodeLeaseStateExclusiveReleasing: - logFatalf("lockInodeWithSharedLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveReleasing") - default: - logFatalf("lockInodeWithSharedLease() found unknown fileInode.leaseState [case 2]: %v", fileInode.leaseState) - } - - globals.Unlock() - - return -} - -// lockInodeIfExclusiveLeaseGranted is similar to lockInodeWithExclusiveLease() except that -// if an ExclusiveLease is not Granted, it will return nil. Note that callers may also be -// assured any inode state is not cached and dirty in the non-Granted cases. As such, if -// an ExclusiveLease is either being Demoted or Released, lockInodeIfExclusiveLeaseGranted() -// will block until the the Demote or Release has been completed. -// -func lockInodeIfExclusiveLeaseGranted(inodeNumber inode.InodeNumber) (fileInode *fileInodeStruct) { - var ( - keepLock bool - ok bool - waitChan chan struct{} - ) - - globals.Lock() - - fileInode, ok = globals.fileInodeMap[inodeNumber] - if !ok { - globals.Unlock() - fileInode = nil - return - } - - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - globals.Unlock() - fileInode = nil - return - case fileInodeLeaseStateSharedRequested: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - globals.Unlock() - fileInode = nil - return - case fileInodeLeaseStateSharedGranted: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - globals.Unlock() - fileInode = nil - return - case fileInodeLeaseStateSharedPromoting: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - globals.Unlock() - fileInode = nil - return - case fileInodeLeaseStateSharedReleasing: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - globals.Unlock() - fileInode = nil - return - case fileInodeLeaseStateExclusiveRequested: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - globals.Unlock() - fileInode = nil - return - case fileInodeLeaseStateExclusiveGranted: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - keepLock = true - case fileInodeLeaseStateExclusiveDemoting: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - keepLock = false - case fileInodeLeaseStateExclusiveReleasing: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - keepLock = false - default: - logFatalf("lockInodeIfExclusiveLeaseGranted() found unknown fileInode.leaseState [case 1]: %v", fileInode.leaseState) - } - - if nil == fileInode.lockWaiters { - fileInode.lockWaiters = list.New() - } else { - waitChan = make(chan struct{}) - _ = fileInode.lockWaiters.PushBack(waitChan) - globals.Unlock() - _ = <-waitChan - globals.Lock() - } - - if keepLock { - // Time to check fileInode.leaseState again [(*fileInodeStruct).unlock() may have changed fileInode.leaseState) - - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - globals.Unlock() - fileInode.unlock(false) - fileInode = nil - case fileInodeLeaseStateSharedRequested: - logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedRequested") - case fileInodeLeaseStateSharedGranted: - globals.Unlock() - fileInode.unlock(false) - fileInode = nil - case fileInodeLeaseStateSharedPromoting: - logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedPromoting") - case fileInodeLeaseStateSharedReleasing: - logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedReleasing") - case fileInodeLeaseStateExclusiveRequested: - logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveRequested") - case fileInodeLeaseStateExclusiveGranted: - // Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock - globals.Unlock() - case fileInodeLeaseStateExclusiveDemoting: - logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveDemoting") - case fileInodeLeaseStateExclusiveReleasing: - logFatalf("lockInodeIfExclusiveLeaseGranted() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveReleasing") - default: - logFatalf("lockInodeIfExclusiveLeaseGranted() found unknown fileInode.leaseState [case 2]: %v", fileInode.leaseState) - } - } else { - globals.Unlock() - fileInode.unlock(false) - fileInode = nil - } - - return -} - -func lockInodeWithExclusiveLease(inodeNumber inode.InodeNumber) (fileInode *fileInodeStruct) { - var ( - err error - leaseReply *jrpcfs.LeaseReply - leaseRequest *jrpcfs.LeaseRequest - leaseRequestEndTime time.Time - leaseRequestStartTime time.Time - ok bool - waitChan chan struct{} - ) - - globals.Lock() - - fileInode, ok = globals.fileInodeMap[inodeNumber] - if !ok { - fileInode = &fileInodeStruct{ - InodeNumber: inodeNumber, - cachedStat: nil, - lockWaiters: nil, - leaseState: fileInodeLeaseStateNone, - pendingLeaseInterrupt: nil, - leaseListElement: nil, - extentMap: nil, - chunkedPutList: list.New(), - flushInProgress: false, - chunkedPutFlushWaiterList: list.New(), - dirtyListElement: nil, - } - - fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode) - globals.fileInodeMap[inodeNumber] = fileInode - } - - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - if nil == fileInode.lockWaiters { - // No Lease currently held... so ask for an Exclusive Lease - - fileInode.lockWaiters = list.New() - fileInode.leaseState = fileInodeLeaseStateExclusiveRequested - _ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeExclusive, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 1] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - // Record leaseReply.LeaseReplyType - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeExclusive: - fileInode.leaseState = fileInodeLeaseStateExclusiveGranted - default: - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 2] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - - // Now that we have an Exclusive Lease, we are also the first in line to use it - so just return - - globals.Unlock() - - return - } - - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateSharedRequested: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateSharedGranted: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - if nil == fileInode.lockWaiters { - // We already have a Shared Lease and we are also the first in line to use it - // TODO: We need to promote it first... which could fail !!! - - fileInode.lockWaiters = list.New() - - globals.Unlock() - - return - } - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateSharedPromoting: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateSharedReleasing: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateExclusiveRequested: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateExclusiveGranted: - globals.exclusiveLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - if nil == fileInode.lockWaiters { - // We already have an Exclusive Lease and we are also the first in line to use it - - fileInode.lockWaiters = list.New() - - globals.Unlock() - - return - } - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateExclusiveDemoting: - globals.sharedLeaseFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - case fileInodeLeaseStateExclusiveReleasing: - globals.unleasedFileInodeCacheLRU.MoveToBack(fileInode.leaseListElement) - - // Somebody else is ahead of us... so fall through - - default: - logFatalf("lockInodeWithExclusiveLease() found unknown fileInode.leaseState [case 1]: %v", fileInode.leaseState) - } - - // Time to block - - waitChan = make(chan struct{}) - _ = fileInode.lockWaiters.PushBack(waitChan) - globals.Unlock() - _ = <-waitChan - - // Time to check fileInode.leaseState again [(*fileInodeStruct).unlock() may have changed fileInode.leaseState) - - globals.Lock() - - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - // No Lease currently held... so ask for an Exclusive Lease - - fileInode.leaseState = fileInodeLeaseStateExclusiveRequested - _ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeExclusive, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 3] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - // Record leaseReply.LeaseReplyType - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeExclusive: - fileInode.leaseState = fileInodeLeaseStateExclusiveGranted - default: - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 4] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - - // Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock - case fileInodeLeaseStateSharedRequested: - logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedRequested") - case fileInodeLeaseStateSharedGranted: - // Now that we have a Shared Lease, we are also the first in line to use it - so we need to Promote the Lease - - fileInode.leaseState = fileInodeLeaseStateSharedPromoting - _ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypePromote, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 5] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - // Record leaseReply.LeaseReplyType - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeExclusive: - fileInode.leaseState = fileInodeLeaseStateExclusiveGranted - - // Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock - case jrpcfs.LeaseReplyTypeDenied: - // We need to Release first... and may have been interrupted to do so already - - // TODO - invalidate cached state (cachedStat and extentMap) - - fileInode.pendingLeaseInterrupt = nil - - fileInode.leaseState = fileInodeLeaseStateSharedReleasing - _ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeRelease, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 6] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeReleased: - // Now we can obtain Exclusive Lease - - fileInode.leaseState = fileInodeLeaseStateExclusiveRequested - _ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(inodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeRelease, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 7] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeExclusive: - fileInode.leaseState = fileInodeLeaseStateExclusiveGranted - default: - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 8] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - - // Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock - default: - logFatalf("lockInodeWithExclusiveLease() unable to release Shared Lease - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - default: - logFatalf("lockInodeWithExclusiveLease() unable to obtain Exclusive Lease [case 8] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - case fileInodeLeaseStateSharedPromoting: - logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedPromoting") - case fileInodeLeaseStateSharedReleasing: - logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateSharedReleasing") - case fileInodeLeaseStateExclusiveRequested: - logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveRequested") - case fileInodeLeaseStateExclusiveGranted: - // Now that we have an Exclusive Lease, we are also the first in line to use it - so we can grant the Lock - case fileInodeLeaseStateExclusiveDemoting: - logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveDemoting") - case fileInodeLeaseStateExclusiveReleasing: - logFatalf("lockInodeWithExclusiveLease() found unexpected fileInode.leaseState: fileInodeLeaseStateExclusiveReleasing") - default: - logFatalf("lockInodeWithExclusiveLease() found unknown fileInode.leaseState [case 2]: %v", fileInode.leaseState) - } - - globals.Unlock() - - return -} - -func (fileInode *fileInodeStruct) unlock(forceRelease bool) { - var ( - err error - forcedRPCInterruptTypeRelease jrpcfs.RPCInterruptType - leaseReply *jrpcfs.LeaseReply - leaseRequest *jrpcfs.LeaseRequest - leaseRequestEndTime time.Time - leaseRequestStartTime time.Time - lockWaitersListElement *list.Element - waitChan chan struct{} - ) - - globals.Lock() - - if forceRelease { - // The unlock() call was likely made for a non-existing Inode - // in which case we might as well clear out both the current - // Lease as well as the fileInodeStruct (if not being referenced) - - forcedRPCInterruptTypeRelease = jrpcfs.RPCInterruptTypeRelease - fileInode.pendingLeaseInterrupt = &forcedRPCInterruptTypeRelease - } - - for nil != fileInode.pendingLeaseInterrupt { - switch *fileInode.pendingLeaseInterrupt { - case jrpcfs.RPCInterruptTypeUnmount: - logFatalf("UNSUPPORTED: (*fileInodeStruct).unlock() received jrpcfs.RPCInterruptTypeUnmount") - case jrpcfs.RPCInterruptTypeDemote: - fileInode.pendingLeaseInterrupt = nil - - if fileInodeLeaseStateExclusiveGranted == fileInode.leaseState { - fileInode.leaseState = fileInodeLeaseStateExclusiveDemoting - _ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.sharedLeaseFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - // TODO - flush dirty state (chunkedPutList) - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(fileInode.InodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeDemote, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 1] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - // Record leaseReply.LeaseReplyType - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeDemoted: - fileInode.leaseState = fileInodeLeaseStateSharedGranted - default: - logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 2] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - } - case jrpcfs.RPCInterruptTypeRelease: - fileInode.pendingLeaseInterrupt = nil - - if fileInodeLeaseStateSharedGranted == fileInode.leaseState { - fileInode.leaseState = fileInodeLeaseStateSharedReleasing - _ = globals.sharedLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - // TODO - flush dirty state (chunkedPutList) - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(fileInode.InodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeRelease, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 3] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - // Record leaseReply.LeaseReplyType - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeReleased: - fileInode.leaseState = fileInodeLeaseStateNone - default: - logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 4] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - } else if fileInodeLeaseStateExclusiveGranted == fileInode.leaseState { - fileInode.leaseState = fileInodeLeaseStateExclusiveReleasing - _ = globals.exclusiveLeaseFileInodeCacheLRU.Remove(fileInode.leaseListElement) - fileInode.leaseListElement = globals.unleasedFileInodeCacheLRU.PushBack(fileInode) - - globals.Unlock() - - // TODO - invalidate cached state (cachedStat and extentMap) - // TODO - flush dirty state (chunkedPutList) - - leaseRequest = &jrpcfs.LeaseRequest{ - InodeHandle: jrpcfs.InodeHandle{ - MountID: globals.mountID, - InodeNumber: int64(fileInode.InodeNumber), - }, - LeaseRequestType: jrpcfs.LeaseRequestTypeRelease, - } - leaseReply = &jrpcfs.LeaseReply{} - - leaseRequestStartTime = time.Now() - err = globals.retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) - if nil != err { - logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 5] - retryRPCClient.Send() failed: %v", err) - } - leaseRequestEndTime = time.Now() - globals.stats.LeaseRequests_Shared_Usec.Add(uint64(leaseRequestEndTime.Sub(leaseRequestStartTime) / time.Microsecond)) - - // Record leaseReply.LeaseReplyType - - globals.Lock() - - switch leaseReply.LeaseReplyType { - case jrpcfs.LeaseReplyTypeReleased: - fileInode.leaseState = fileInodeLeaseStateNone - default: - logFatalf("(*fileInodeStruct).unlock() unable to Demote Exclusive Lease [case 6] - LeaseReplyType == %v", leaseReply.LeaseReplyType) - } - } - default: - logFatalf("(*fileInodeStruct).unlock() received unknown rpcInterrupt.RPCInterruptType: %v", *fileInode.pendingLeaseInterrupt) - } - - // We need to loop here in case another Interrupt arrived while - // doing any of the above Demotions/Releases while globals unlocked - } - - if 0 == fileInode.lockWaiters.Len() { - // Nobody was waiting for the fileInode lock - - if fileInodeLeaseStateNone == fileInode.leaseState { - // And there is no held Lease... so just destroy it - - delete(globals.fileInodeMap, fileInode.InodeNumber) - _ = globals.unleasedFileInodeCacheLRU.Remove(fileInode.leaseListElement) - } else { - // We should keep holding the Lease we have... but indicate the lock is available - - fileInode.lockWaiters = nil - } - - globals.Unlock() - return - } - - // At least one caller to lockInodeWith{Shared|Exclusive}Lease() is waiting... so wake them up before exiting - - lockWaitersListElement = fileInode.lockWaiters.Front() - fileInode.lockWaiters.Remove(lockWaitersListElement) - - globals.Unlock() - - waitChan = lockWaitersListElement.Value.(chan struct{}) - - waitChan <- struct{}{} -} diff --git a/pfsagentd/log.go b/pfsagentd/log.go deleted file mode 100644 index 7fd73ccd..00000000 --- a/pfsagentd/log.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "log" - "os" - "time" -) - -func logFatal(err error) { - logf("FATAL", "%v", err) - os.Exit(1) -} - -func logFatalf(format string, args ...interface{}) { - logf("FATAL", format, args...) - os.Exit(1) -} - -func logErrorf(format string, args ...interface{}) { - logf("ERROR", format, args...) -} - -func logWarnf(format string, args ...interface{}) { - logf("WARN", format, args...) -} - -func logInfof(format string, args ...interface{}) { - logf("INFO", format, args...) -} - -func logTracef(format string, args ...interface{}) { - if globals.config.TraceEnabled { - logf("TRACE", format, args...) - } -} - -func logf(level string, format string, args ...interface{}) { - var ( - enhancedArgs []interface{} - enhancedFormat string - err error - logMsg string - ) - - enhancedFormat = "[%s][%s][%d][%s] " + format - enhancedArgs = append([]interface{}{time.Now().Format(time.RFC3339Nano), globals.config.FUSEVolumeName, os.Getpid(), level}, args...) - - logMsg = fmt.Sprintf(enhancedFormat, enhancedArgs[:]...) - - if nil == globals.logFile { - if "" != globals.config.LogFilePath { - globals.logFile, err = os.OpenFile(globals.config.LogFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666) - if nil == err { - globals.logFile.WriteString(logMsg + "\n") - } else { - globals.logFile = nil - } - } - } else { - globals.logFile.WriteString(logMsg + "\n") - } - if globals.config.LogToConsole { - fmt.Fprintln(os.Stderr, logMsg) - } -} - -func newLogger() *log.Logger { - return log.New(&globals, "", 0) -} - -func (dummy *globalsStruct) Write(p []byte) (n int, err error) { - logf("FISSION", "%s", string(p[:])) - return 0, nil -} diff --git a/pfsagentd/main.go b/pfsagentd/main.go deleted file mode 100644 index e0f9eb75..00000000 --- a/pfsagentd/main.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "log" - "os" - "os/signal" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" -) - -func main() { - var ( - confMap conf.ConfMap - err error - signalChan chan os.Signal - ) - - if len(os.Args) < 2 { - log.Fatalf("no .conf file specified") - } - - // Parse arguments (at this point, logging goes only to the console) - - globals.logFile = nil - globals.config.LogFilePath = "" - globals.config.LogToConsole = true - - confMap, err = conf.MakeConfMapFromFile(os.Args[1]) - if nil != err { - log.Fatalf("failed to load config: %v", err) - } - - err = confMap.UpdateFromStrings(os.Args[2:]) - if nil != err { - log.Fatalf("failed to apply config overrides: %v", err) - } - - // Arm signal handler used to indicate termination and wait on it - // - // Note: signalled chan must be buffered to avoid race with window between - // arming handler and blocking on the chan read - - signalChan = make(chan os.Signal, 1) - - signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) - - // Initialize globals - - initializeGlobals(confMap) - - // Trigger initial auth plug-in invocation - - updateAuthTokenAndStorageURL() - - // Perform mount via ProxyFS - - doMountProxyFS() - - // Start serving FUSE mount point - - performMountFUSE() - - // Start serving HTTP - - serveHTTP() - - // Await any of specified signals or fission exit - - select { - case _ = <-signalChan: - // Normal termination due to one of the above registered signals - case err = <-globals.fissionErrChan: - // Unexpected exit of fission.Volume - logFatalf("unexpected error from package fission: %v", err) - } - - // Stop serving HTTP - - unserveHTTP() - - // Stop serving FUSE mount point - - performUnmountFUSE() - - // Flush all dirty fileInode's - - emptyFileInodeDirtyListAndLogSegmentChan() - - // Perform unmount via ProxyFS - - doUnmountProxyFS() - - // Terminate authPlugIn - - stopAuthPlugIn() - - // Uninitialize globals - - uninitializeGlobals() -} diff --git a/pfsagentd/pfsagent.conf b/pfsagentd/pfsagent.conf deleted file mode 100644 index b34ee52d..00000000 --- a/pfsagentd/pfsagent.conf +++ /dev/null @@ -1,48 +0,0 @@ -[Agent] -FUSEVolumeName: CommonVolume -FUSEMountPointPath: AgentMountPoint -FUSEUnMountRetryDelay: 100ms -FUSEUnMountRetryCap: 100 -PlugInPath: pfsagentd-swift-auth-plugin -PlugInEnvName: SwiftAuthBlob -PlugInEnvValue: {"AuthURL":"http://localhost:8080/auth/v1.0"\u002C"AuthUser":"test:tester"\u002C"AuthKey":"testing"\u002C"Account":"AUTH_test"} -SwiftTimeout: 10m -SwiftRetryLimit: 10 -SwiftRetryDelay: 1s -SwiftRetryDelayVariance: 25 -SwiftRetryExpBackoff: 1.4 -SwiftConnectionPoolSize: 200 -FetchExtentsFromFileOffset: 32 -FetchExtentsBeforeFileOffset: 0 -ReadCacheLineSize: 1048576 -ReadCacheLineCount: 1000 -LeaseRetryLimit: 10 -LeaseRetryDelay: 1s -LeaseRetryDelayVariance: 25 -LeaseRetryExpBackoff: 1.4 -SharedLeaseLimit: 1000 -ExclusiveLeaseLimit: 100 -ExtentMapEntryLimit: 1048576 -DirtyLogSegmentLimit: 50 -DirtyFileLimit: 50 # TODO - obsolete this -MaxFlushSize: 10485760 -MaxFlushTime: 200ms -LogFilePath: /var/log/pfsagentd.log -LogToConsole: true -TraceEnabled: false -HTTPServerIPAddr: 0.0.0.0 -HTTPServerTCPPort: 9090 -ReadDirPlusEnabled: false -XAttrEnabled: false -EntryDuration: 0s -AttrDuration: 0s -ReaddirMaxEntries: 1024 -FUSEMaxBackground: 100 -FUSECongestionThreshhold: 0 -FUSEMaxWrite: 131072 -FUSEAllowOther: true -RetryRPCPublicIPAddr: 127.0.0.1 -RetryRPCPort: 32356 -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCACertFilePath: diff --git a/pfsagentd/pfsagentd-init/Makefile b/pfsagentd/pfsagentd-init/Makefile deleted file mode 100644 index 1251963f..00000000 --- a/pfsagentd/pfsagentd-init/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfsagentd/pfsagentd-init - -include ../../GoMakefile diff --git a/pfsagentd/pfsagentd-init/README.md b/pfsagentd/pfsagentd-init/README.md deleted file mode 100644 index 7f1be971..00000000 --- a/pfsagentd/pfsagentd-init/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# PFSAgent Init Daemon For Use In Containers - -Docker-style containers are designed to run a single program. If that program -has a dependency on running services, such as `pfsagentd`, there is no opportunity -to ensure these are running with a subsystem like `systemd`. While there are a -number of `systemd` substitutes in wide use in containers that have this service -requirement (e.g. `supervisord` and `s6`), these tools generally don't have a -mechanism to order such services nor ensure that each is up and running sufficient -for the subsequent service or the application to immediately consume. - -The program in this directory, `pfsagentd-init`, provides the capability to launch -a number of instances of `pfsagentd`, ensures that they are able to receive traffic -via their `FUSEMountPointPath`, and finally launching the desired application. - -As may be common, the application to be dependent upon one or more `pfsagentd` -instances has already been containerized in some existing container image. -A `Dockerfile` in `../container/build` is provided to build the components that -need to be added to a derived container image based off the application's original -container image. Then, a tool in `../container/insert` will create a `Dockerfile` -derived from the application's original container image that imports these -to-be-inserted components. - -As part of the created `Dockerfile` is a replacement for any ENTRYPOINT and/or -CMD that will launch `pfsagentd-init` that accomplishes this conversion. diff --git a/pfsagentd/pfsagentd-init/dummy_test.go b/pfsagentd/pfsagentd-init/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfsagentd/pfsagentd-init/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfsagentd/pfsagentd-init/main.go b/pfsagentd/pfsagentd-init/main.go deleted file mode 100644 index c251317a..00000000 --- a/pfsagentd/pfsagentd-init/main.go +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "encoding/json" - "io/ioutil" - "log" - "net" - "net/http" - "os" - "os/exec" - "os/signal" - "runtime" - "strconv" - "strings" - "sync" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "golang.org/x/sys/unix" -) - -const ( - // PFSAgentReadyAttemptLimit defines the number of times pfsagentd-init - // will poll for the successful launching of each child pfsagentd instance - // - PFSAgentReadyAttemptLimit = uint32(100) - - // PFSAgentReadyRetryDelay is the delay between polling attempts by - // pfsagentd-init of each child pfsagentd instance - // - PFSAgentReadyRetryDelay = "100ms" -) - -type pfsagentdInstanceStruct struct { - confFileName string // - confMap conf.ConfMap // - httpServerHostPort string // - stopChan chan struct{} // Closed by main() to tell (*pfsagentdInstanceStruct).daemon() to exit -} - -type globalsStruct struct { - parentEntrypoint []string // A non-existent or empty PFSAGENT_PARENT_ENTRYPOINT will result in []string{} - parentCmd []string // A non-existent or empty PFSAGENT_PARENT_CMD will result in []string{} - parentWorkingDir string // A non-existent or empty PFSAGENT_PARENT_WORKING_DIR will result in "/" - pfsagentdReadyRetryDelay time.Duration // - pfsagentdInstance []*pfsagentdInstanceStruct // - pfsagentdInstanceUpChan chan struct{} // Signaled by each pfsagentdInstance once they are up - childErrorChan chan struct{} // Signaled by an unexpectedly exiting child to - // indicate that the entire container should exit - applicationCleanExitChan chan struct{} // Signaled by applicationDaemon() on clean exit - applicationDaemonStopChan chan struct{} // Closed by main() to tell applicationDaemon() to exit - signalChan chan os.Signal // Signaled by OS or Container Framework to trigger exit - pfsagentdInstanceWG sync.WaitGroup // - applicationDaemonWG sync.WaitGroup // -} - -var globals globalsStruct - -func main() { - var ( - err error - fileInfo os.FileInfo - fileInfoName string - fileInfoSlice []os.FileInfo - parentCmdBuf string - parentEntrypointBuf string - pfsagentdInstance *pfsagentdInstanceStruct - pfsagentdInstanceFUSEMountPointPath string - pfsagentdInstanceHTTPServerIPAddr string - pfsagentdInstanceHTTPServerTCPPort uint16 - pfsagentdInstanceUpCount int - ) - - log.Printf("Launch of %s\n", os.Args[0]) - - parentEntrypointBuf = os.Getenv("PFSAGENT_PARENT_ENTRYPOINT") - if (parentEntrypointBuf == "") || (parentEntrypointBuf == "null") { - globals.parentEntrypoint = make([]string, 0) - } else { - globals.parentEntrypoint = make([]string, 0) - err = json.Unmarshal([]byte(parentEntrypointBuf), &globals.parentEntrypoint) - if err != nil { - log.Fatalf("Couldn't parse PFSAGENT_PARENT_ENTRYPOINT: \"%s\"\n", parentEntrypointBuf) - } - } - - parentCmdBuf = os.Getenv("PFSAGENT_PARENT_CMD") - if (parentCmdBuf == "") || (parentCmdBuf == "null") { - globals.parentCmd = make([]string, 0) - } else { - globals.parentCmd = make([]string, 0) - err = json.Unmarshal([]byte(parentCmdBuf), &globals.parentCmd) - if err != nil { - log.Fatalf("Couldn't parse PFSAGENT_PARENT_CMD: \"%s\"\n", parentCmdBuf) - } - } - - globals.parentWorkingDir = os.Getenv("PFSAGENT_PARENT_WORKING_DIR") - if globals.parentWorkingDir == "" { - globals.parentWorkingDir = "/" - } - - fileInfoSlice, err = ioutil.ReadDir(".") - if err != nil { - log.Fatalf("Couldn't read directory: %v\n", err) - } - - globals.pfsagentdReadyRetryDelay, err = time.ParseDuration(PFSAgentReadyRetryDelay) - if err != nil { - log.Fatalf("Couldn't parse PFSAgentReadyRetryDelay\n") - } - - globals.pfsagentdInstance = make([]*pfsagentdInstanceStruct, 0) - - for _, fileInfo = range fileInfoSlice { - fileInfoName = fileInfo.Name() - if strings.HasPrefix(fileInfoName, "pfsagent.conf_") { - if fileInfoName != "pfsagent.conf_TEMPLATE" { - pfsagentdInstance = &pfsagentdInstanceStruct{ - confFileName: fileInfoName, - stopChan: make(chan struct{}), - } - globals.pfsagentdInstance = append(globals.pfsagentdInstance, pfsagentdInstance) - } - } - } - - globals.pfsagentdInstanceUpChan = make(chan struct{}, len(globals.pfsagentdInstance)) - globals.childErrorChan = make(chan struct{}, len(globals.pfsagentdInstance)+1) - - globals.signalChan = make(chan os.Signal, 1) - signal.Notify(globals.signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) - - globals.pfsagentdInstanceWG.Add(len(globals.pfsagentdInstance)) - - for _, pfsagentdInstance = range globals.pfsagentdInstance { - pfsagentdInstance.confMap, err = conf.MakeConfMapFromFile(pfsagentdInstance.confFileName) - if err != nil { - log.Fatalf("Unable to parse %s: %v\n", pfsagentdInstance.confFileName, err) - } - - pfsagentdInstanceFUSEMountPointPath, err = pfsagentdInstance.confMap.FetchOptionValueString("Agent", "FUSEMountPointPath") - if err != nil { - log.Fatalf("In %s, parsing [Agent]FUSEMountPointPath failed: %v\n", pfsagentdInstance.confFileName, err) - } - - err = os.MkdirAll(pfsagentdInstanceFUSEMountPointPath, 0777) - if err != nil { - log.Fatalf("For %s, could not create directory path %s: %v\n", pfsagentdInstance.confFileName, pfsagentdInstanceFUSEMountPointPath, err) - } - - pfsagentdInstanceHTTPServerIPAddr, err = pfsagentdInstance.confMap.FetchOptionValueString("Agent", "HTTPServerIPAddr") - if err != nil { - log.Fatalf("In %s, parsing [Agent]HTTPServerIPAddr failed: %v\n", pfsagentdInstance.confFileName, err) - } - if pfsagentdInstanceHTTPServerIPAddr == "0.0.0.0" { - pfsagentdInstanceHTTPServerIPAddr = "127.0.0.1" - } else if pfsagentdInstanceHTTPServerIPAddr == "::" { - pfsagentdInstanceHTTPServerIPAddr = "::1" - } - - pfsagentdInstanceHTTPServerTCPPort, err = pfsagentdInstance.confMap.FetchOptionValueUint16("Agent", "HTTPServerTCPPort") - if err != nil { - log.Fatalf("In %s, parsing [Agent]HTTPServerTCPPort failed: %v\n", pfsagentdInstance.confFileName, err) - } - - pfsagentdInstance.httpServerHostPort = net.JoinHostPort(pfsagentdInstanceHTTPServerIPAddr, strconv.Itoa(int(pfsagentdInstanceHTTPServerTCPPort))) - - go pfsagentdInstance.daemon() - } - - if len(globals.pfsagentdInstance) > 0 { - pfsagentdInstanceUpCount = 0 - - for { - select { - case _ = <-globals.pfsagentdInstanceUpChan: - pfsagentdInstanceUpCount++ - if pfsagentdInstanceUpCount == len(globals.pfsagentdInstance) { - goto pfsagentdInstanceUpComplete - } - case _ = <-globals.childErrorChan: - for _, pfsagentdInstance = range globals.pfsagentdInstance { - close(pfsagentdInstance.stopChan) - } - - globals.pfsagentdInstanceWG.Wait() - - log.Fatalf("Abnormal exit during PFSAgent Instances creation\n") - case _ = <-globals.signalChan: - for _, pfsagentdInstance = range globals.pfsagentdInstance { - close(pfsagentdInstance.stopChan) - } - - globals.pfsagentdInstanceWG.Wait() - - log.Printf("Signaled exit during PFSAgent Instances creation\n") - os.Exit(0) - } - } - - pfsagentdInstanceUpComplete: - } - - globals.applicationCleanExitChan = make(chan struct{}, 1) - globals.applicationDaemonStopChan = make(chan struct{}) - - globals.applicationDaemonWG.Add(1) - - go applicationDaemon() - - for { - select { - case _ = <-globals.childErrorChan: - close(globals.applicationDaemonStopChan) - - globals.applicationDaemonWG.Wait() - - for _, pfsagentdInstance = range globals.pfsagentdInstance { - close(pfsagentdInstance.stopChan) - } - - globals.pfsagentdInstanceWG.Wait() - - log.Fatalf("Abnormal exit after PFSAgent Instances creation\n") - case _ = <-globals.applicationCleanExitChan: - globals.applicationDaemonWG.Wait() - - for _, pfsagentdInstance = range globals.pfsagentdInstance { - close(pfsagentdInstance.stopChan) - } - - globals.pfsagentdInstanceWG.Wait() - - log.Printf("Normal exit do to application clean exit\n") - os.Exit(0) - case _ = <-globals.signalChan: - close(globals.applicationDaemonStopChan) - - globals.applicationDaemonWG.Wait() - - for _, pfsagentdInstance = range globals.pfsagentdInstance { - close(pfsagentdInstance.stopChan) - } - - globals.pfsagentdInstanceWG.Wait() - - log.Printf("Signaled exit after PFSAgent Instances creation\n") - os.Exit(0) - } - } -} - -func (pfsagentdInstance *pfsagentdInstanceStruct) daemon() { - var ( - cmd *exec.Cmd - cmdExitChan chan struct{} - err error - getResponse *http.Response - readyAttempt uint32 - ) - - cmd = exec.Command("pfsagentd", pfsagentdInstance.confFileName) - - err = cmd.Start() - if err != nil { - globals.childErrorChan <- struct{}{} - globals.pfsagentdInstanceWG.Done() - runtime.Goexit() - } - - readyAttempt = 1 - - for { - getResponse, err = http.Get("http://" + pfsagentdInstance.httpServerHostPort + "/version") - if err == nil { - _, err = ioutil.ReadAll(getResponse.Body) - if err != nil { - log.Fatalf("Failure to read GET Response Body: %v\n", err) - } - err = getResponse.Body.Close() - if err != nil { - log.Fatalf("Failure to close GET Response Body: %v\n", err) - } - - if getResponse.StatusCode == http.StatusOK { - break - } - } - - readyAttempt++ - - if readyAttempt > PFSAgentReadyAttemptLimit { - log.Fatalf("Ready attempt limit exceeded for pfsagentd instance %s\n", pfsagentdInstance.confFileName) - } - - time.Sleep(globals.pfsagentdReadyRetryDelay) - } - - globals.pfsagentdInstanceUpChan <- struct{}{} - - cmdExitChan = make(chan struct{}, 1) - go watchCmdDaemon(cmd, cmdExitChan) - - for { - select { - case _ = <-cmdExitChan: - globals.childErrorChan <- struct{}{} - globals.pfsagentdInstanceWG.Done() - runtime.Goexit() - case _, _ = <-pfsagentdInstance.stopChan: - err = cmd.Process.Signal(unix.SIGTERM) - if err == nil { - _ = cmd.Wait() - } - globals.pfsagentdInstanceWG.Done() - runtime.Goexit() - } - } -} - -func applicationDaemon() { - var ( - cmd *exec.Cmd - cmdArgs []string - cmdExitChan chan struct{} - cmdPath string - err error - ) - - cmdArgs = make([]string, 0) - - if len(globals.parentEntrypoint) > 0 { - cmdPath = globals.parentEntrypoint[0] - cmdArgs = globals.parentEntrypoint[1:] - - if len(os.Args) > 1 { - cmdArgs = append(cmdArgs, os.Args[1:]...) - } else { - cmdArgs = append(cmdArgs, globals.parentCmd...) - } - } else { // len(globals.parentEntrypoint) == 0 - if len(os.Args) > 1 { - cmdPath = os.Args[1] - cmdArgs = os.Args[2:] - } else { - if len(globals.parentCmd) > 0 { - cmdPath = globals.parentCmd[0] - cmdArgs = globals.parentCmd[1:] - } else { - cmdPath = "/bin/sh" - cmdArgs = make([]string, 0) - } - } - } - - cmd = exec.Command(cmdPath, cmdArgs...) - - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - err = cmd.Start() - if err != nil { - globals.childErrorChan <- struct{}{} - globals.applicationDaemonWG.Done() - runtime.Goexit() - } - - cmdExitChan = make(chan struct{}, 1) - go watchCmdDaemon(cmd, cmdExitChan) - - for { - select { - case _ = <-cmdExitChan: - if cmd.ProcessState.ExitCode() == 0 { - globals.applicationCleanExitChan <- struct{}{} - } else { - globals.childErrorChan <- struct{}{} - } - globals.applicationDaemonWG.Done() - runtime.Goexit() - case _, _ = <-globals.applicationDaemonStopChan: - globals.applicationDaemonWG.Done() - runtime.Goexit() - } - } -} - -func watchCmdDaemon(cmd *exec.Cmd, cmdExitChan chan struct{}) { - _ = cmd.Wait() - - cmdExitChan <- struct{}{} -} diff --git a/pfsagentd/pfsagentd-swift-auth-plugin/Makefile b/pfsagentd/pfsagentd-swift-auth-plugin/Makefile deleted file mode 100644 index 2b2feefe..00000000 --- a/pfsagentd/pfsagentd-swift-auth-plugin/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfsagentd/pfsagentd-swift-auth-plugin - -include ../../GoMakefile diff --git a/pfsagentd/pfsagentd-swift-auth-plugin/README.md b/pfsagentd/pfsagentd-swift-auth-plugin/README.md deleted file mode 100644 index e64c6f53..00000000 --- a/pfsagentd/pfsagentd-swift-auth-plugin/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# PFSAgent OpenStack Swift Authorization PlugIn - -To provide a workable solution for those following standard Swift Authentication, -this plug-in instance may be employed. Otherwise, consider this implementation a -template for the desired instantiation of whatever Authentication mechanism is -employed. - -For the standard Swift Authentication instantiation, the value of the ENV -varialble specified by [Agent]PlugInEnvName (optionally set to the value of -[Agent]PlugInEnvValue by PFSAgent) is required to be a UTF-8 encoded JSON Document: - -``` -{ - "AuthURL" : "/auth/v1.0>", - "AuthUser" : "", - "AuthKey" : "", - "Account" : "" -} -``` - -There are three modifications to the Storage URL normally returned by a -standard Swift Authentication operation: - -* The `scheme` used to authenticate may be either `http` or `https`. In the - case of `https`, it is likely that some form of TLS termination prior - to reaching the Swift Proxy has rewritten the `scheme` to be `http`. In such - a case, the Storage URL returned will specify `http` as its scheme. Since - the client must continue to use `https` to reach the Swift Proxy for each - authenticated subsequent request, the plug-in will rewrite the scheme to - be `https`. Note that this is an incomplete solution in cases where standard - port numbers (i.e. `80` for `http` and `443` for `https`) are not assumed - (i.e. port numbers are specified in the URL). - -* The path portion of the Storage URL returned by the Swift Proxy will start - with `v1` to indicate the version of the OpenStack Swift protocol being used. - No `v2` has ever been defined, so this is a constant in normal requests. - To indicate the client, in the case of GETs and PUTs, has specified the - `physical` path (i.e. to LogSegments of a FileInode) rather than the - `virtual` path (i.e. the full path of the file in the file system), this - protocol version will be changed to `proxyfs`. This updated protocol verion - will also be used with the `PROXYFS` method used to pass a `mount` request, - via JSON RPC, over to the ProxyFS process serving the volume to be mounted. - -* The final element of the path portion of the Storage URL returned by the - Swift Proxy will typically be the Account associated with the specified - AuthUser (e.g. AuthUser `test` typically has a corresponding Account named - `AUTH_test`). The volume being accessed may, however be stored in a different - Account than this. As such, the account element of the path will be replaced - with the `Account` as requested. diff --git a/pfsagentd/pfsagentd-swift-auth-plugin/dummy_test.go b/pfsagentd/pfsagentd-swift-auth-plugin/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfsagentd/pfsagentd-swift-auth-plugin/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfsagentd/pfsagentd-swift-auth-plugin/main.go b/pfsagentd/pfsagentd-swift-auth-plugin/main.go deleted file mode 100644 index 7e9037b5..00000000 --- a/pfsagentd/pfsagentd-swift-auth-plugin/main.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "strings" - - "github.com/NVIDIA/proxyfs/version" -) - -type authInStruct struct { - AuthURL string - AuthUser string - AuthKey string - Account string -} - -type authOutStruct struct { - AuthToken string - StorageURL string -} - -const ( - stdinReadSize = 1 -) - -func main() { - var ( - authIn authInStruct - err error - plugInEnvName string - plugInEnvValue string - stdinReadBuf []byte - ) - - switch len(os.Args) { - case 0: - fmt.Fprintf(os.Stderr, "Logic error... len(os.Args) cannot be zero\n") - os.Exit(1) - case 1: - fmt.Fprintf(os.Stderr, "Missing PlugInEnvName\n") - os.Exit(1) - case 2: - plugInEnvName = os.Args[1] - default: - fmt.Fprintf(os.Stderr, "Superfluous arguments (beyond PlugInEnvName) supplied: %v\n", os.Args[2:]) - os.Exit(1) - } - - plugInEnvValue = os.Getenv(plugInEnvName) - - err = json.Unmarshal([]byte(plugInEnvValue), &authIn) - if nil != err { - fmt.Fprintf(os.Stderr, "json.Unmarshal(\"%s\",) failed: %v\n", plugInEnvValue, err) - os.Exit(1) - } - - performAuth(&authIn) - - stdinReadBuf = make([]byte, stdinReadSize) - - for { - stdinReadBuf = stdinReadBuf[:stdinReadSize] - _, err = os.Stdin.Read(stdinReadBuf) - if nil == err { - performAuth(&authIn) - } else { - if io.EOF == err { - os.Exit(0) - } - fmt.Fprintf(os.Stderr, "os.Stdin.Read(stdinReadBuf) failed: %v\n", err) - os.Exit(1) - } - } -} - -func performAuth(authIn *authInStruct) { - var ( - authOut *authOutStruct - authOutJSON []byte - authRequest *http.Request - authResponse *http.Response - err error - storageURLSplit []string - ) - - authRequest, err = http.NewRequest("GET", authIn.AuthURL, nil) - if nil != err { - fmt.Fprintf(os.Stderr, "http.NewRequest(\"GET\", \"%s\", nil) failed: %v\n", authIn.AuthURL, err) - os.Exit(1) - } - - authRequest.Header.Add("X-Auth-User", authIn.AuthUser) - authRequest.Header.Add("X-Auth-Key", authIn.AuthKey) - - authRequest.Header.Add("User-Agent", "PFSAgent-Auth "+version.ProxyFSVersion) - - authResponse, err = http.DefaultClient.Do(authRequest) - if nil != err { - fmt.Fprintf(os.Stderr, "http.DefaultClient.Do(authRequest) failed: %v\n", err) - os.Exit(1) - } - - if http.StatusOK != authResponse.StatusCode { - fmt.Fprintf(os.Stderr, "authResponse.Status unexpecte: %v\n", authResponse.Status) - os.Exit(1) - } - - authOut = &authOutStruct{ - AuthToken: authResponse.Header.Get("X-Auth-Token"), - StorageURL: authResponse.Header.Get("X-Storage-Url"), - } - - if strings.HasPrefix(authIn.AuthURL, "https://") && strings.HasPrefix(authOut.StorageURL, "http://") { - // We need to correct for the case where AuthURL starts with "https://"" - // but the Swift Proxy is behind a TLS terminating proxy. In this case, - // Swift Proxy auth will actually see an AuthURL starting with "http://"" - // and respond with a StorageURL starting with "http://"". - - authOut.StorageURL = strings.Replace(authOut.StorageURL, "http://", "https://", 1) - } - - storageURLSplit = strings.Split(authOut.StorageURL, "/") - - storageURLSplit[3] = "proxyfs" - storageURLSplit[4] = authIn.Account - - authOut.StorageURL = strings.Join(storageURLSplit, "/") - - authOutJSON, err = json.Marshal(authOut) - if nil != err { - fmt.Fprintf(os.Stderr, "json.Marshal(authOut) failed: %v\n", err) - os.Exit(1) - } - - _, err = os.Stdout.Write(authOutJSON) - if nil != err { - fmt.Fprintf(os.Stderr, "os.Stdout.Write(authOutJSON) failed: %v\n", err) - os.Exit(1) - } -} diff --git a/pfsagentd/request.go b/pfsagentd/request.go deleted file mode 100644 index 58eff0e5..00000000 --- a/pfsagentd/request.go +++ /dev/null @@ -1,566 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "encoding/json" - "io" - "io/ioutil" - "math/rand" - "net/http" - "os" - "os/exec" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/NVIDIA/proxyfs/jrpcfs" - "github.com/NVIDIA/proxyfs/retryrpc" - "github.com/NVIDIA/proxyfs/version" -) - -const ( - authPipeReadBufSize = 1024 -) - -type authOutStruct struct { - AuthToken string - StorageURL string -} - -func doMountProxyFS() { - var ( - accountName string - err error - mountReply *jrpcfs.MountByAccountNameReply - mountRequest *jrpcfs.MountByAccountNameRequest - swiftStorageURL string - swiftStorageURLSplit []string - ) - - retryrpcConfig := &retryrpc.ClientConfig{ - DNSOrIPAddr: globals.config.RetryRPCPublicIPAddr, - Port: int(globals.config.RetryRPCPort), - RootCAx509CertificatePEM: globals.retryRPCCACertPEM, - Callbacks: &globals, - DeadlineIO: globals.config.RetryRPCDeadlineIO, - KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, - } - - globals.retryRPCClient, err = retryrpc.NewClient(retryrpcConfig) - if nil != err { - logFatalf("unable to retryRPCClient.NewClient(%v,%v): Volume: %s err: %v", globals.config.RetryRPCPublicIPAddr, globals.config.RetryRPCPort, globals.config.FUSEVolumeName, accountName, err) - } - - swiftStorageURL = fetchStorageURL() - if "" == swiftStorageURL { - logFatalf("unable to fetchStorageURL()") - } - - swiftStorageURLSplit = strings.Split(swiftStorageURL, "/") - - accountName = swiftStorageURLSplit[4] - - mountRequest = &jrpcfs.MountByAccountNameRequest{ - AccountName: accountName, - AuthToken: fetchAuthToken(), - } - - mountReply = &jrpcfs.MountByAccountNameReply{} - - err = globals.retryRPCClient.Send("RpcMountByAccountName", mountRequest, mountReply) - if nil != err { - logFatalf("unable to mount Volume %s (Account: %s): %v", globals.config.FUSEVolumeName, accountName, err) - } - - globals.mountID = mountReply.MountID - -} - -func doUnmountProxyFS() { - var ( - err error - unmountReply *jrpcfs.Reply - unmountRequest *jrpcfs.UnmountRequest - ) - - // TODO: Flush outstanding FileInode's - // TODO: Tell ProxyFS we are releasing all leases - // TODO: Tell ProxyFS we are unmounting - - unmountRequest = &jrpcfs.UnmountRequest{ - MountID: globals.mountID, - } - - unmountReply = &jrpcfs.Reply{} - - err = globals.retryRPCClient.Send("RpcUnmount", unmountRequest, unmountReply) - if nil != err { - logFatalf("unable to unmount Volume %s: %v", globals.config.FUSEVolumeName, err) - } - - globals.retryRPCClient.Close() -} - -func doJRPCRequest(jrpcMethod string, jrpcParam interface{}, jrpcResult interface{}) (err error) { - var ( - httpErr error - httpRequest *http.Request - jrpcRequest []byte - jrpcRequestID uint64 - jrpcResponse []byte - marshalErr error - ok bool - swiftStorageURL string - unmarshalErr error - ) - - jrpcRequestID, jrpcRequest, marshalErr = jrpcMarshalRequest(jrpcMethod, jrpcParam) - if nil != marshalErr { - logFatalf("unable to marshal request (jrpcMethod=%s jrpcParam=%v): %#v", jrpcMethod, jrpcParam, marshalErr) - } - - swiftStorageURL = fetchStorageURL() - if "" == swiftStorageURL { - logFatalf("unable to fetchStorageURL()") - } - - httpRequest, httpErr = http.NewRequest("PROXYFS", swiftStorageURL, bytes.NewReader(jrpcRequest)) - if nil != httpErr { - logFatalf("unable to create PROXYFS http.Request (jrpcMethod=%s jrpcParam=%#v): %v", jrpcMethod, jrpcParam, httpErr) - } - - httpRequest.Header["Content-Type"] = []string{"application/json"} - - _, jrpcResponse, ok, _ = doHTTPRequest(httpRequest, http.StatusOK, http.StatusUnprocessableEntity) - if !ok { - logFatalf("unable to contact ProxyFS") - } - - _, err, unmarshalErr = jrpcUnmarshalResponseForIDAndError(jrpcResponse) - if nil != unmarshalErr { - logFatalf("unable to unmarshal response [case 1] (jrpcMethod=%s jrpcParam=%#v): %v", jrpcMethod, jrpcParam, unmarshalErr) - } - - if nil != err { - return - } - - unmarshalErr = jrpcUnmarshalResponse(jrpcRequestID, jrpcResponse, jrpcResult) - if nil != unmarshalErr { - logFatalf("unable to unmarshal response [case 2] (jrpcMethod=%s jrpcParam=%#v): %v", jrpcMethod, jrpcParam, unmarshalErr) - } - - return -} - -func doHTTPRequest(request *http.Request, okStatusCodes ...int) (response *http.Response, responseBody []byte, ok bool, statusCode int) { - var ( - err error - okStatusCode int - okStatusCodesSet map[int]struct{} - retryDelay time.Duration - retryIndex uint64 - swiftAuthToken string - ) - - _ = atomic.AddUint64(&globals.metrics.HTTPRequests, 1) - - request.Header["User-Agent"] = []string{"PFSAgent " + version.ProxyFSVersion} - - okStatusCodesSet = make(map[int]struct{}) - for _, okStatusCode = range okStatusCodes { - okStatusCodesSet[okStatusCode] = struct{}{} - } - - retryIndex = 0 - - for { - swiftAuthToken = fetchAuthToken() - - request.Header["X-Auth-Token"] = []string{swiftAuthToken} - - _ = atomic.AddUint64(&globals.metrics.HTTPRequestsInFlight, 1) - response, err = globals.httpClient.Do(request) - _ = atomic.AddUint64(&globals.metrics.HTTPRequestsInFlight, ^uint64(0)) - if nil != err { - _ = atomic.AddUint64(&globals.metrics.HTTPRequestSubmissionFailures, 1) - logErrorf("doHTTPRequest(%s %s) failed to submit request: %v", request.Method, request.URL.String(), err) - ok = false - return - } - - responseBody, err = ioutil.ReadAll(response.Body) - _ = response.Body.Close() - if nil != err { - _ = atomic.AddUint64(&globals.metrics.HTTPRequestResponseBodyCorruptions, 1) - logErrorf("doHTTPRequest(%s %s) failed to read responseBody: %v", request.Method, request.URL.String(), err) - ok = false - return - } - - _, ok = okStatusCodesSet[response.StatusCode] - if ok { - statusCode = response.StatusCode - return - } - - if retryIndex >= globals.config.SwiftRetryLimit { - _ = atomic.AddUint64(&globals.metrics.HTTPRequestRetryLimitExceededCount, 1) - logWarnf("doHTTPRequest(%s %s) reached SwiftRetryLimit", request.Method, request.URL.String()) - ok = false - return - } - - if http.StatusUnauthorized == response.StatusCode { - _ = atomic.AddUint64(&globals.metrics.HTTPRequestsRequiringReauthorization, 1) - logInfof("doHTTPRequest(%s %s) needs to call updateAuthTokenAndStorageURL()", request.Method, request.URL.String()) - updateAuthTokenAndStorageURL() - } else { - logWarnf("doHTTPRequest(%s %s) needs to retry due to unexpected http.Status: %s", request.Method, request.URL.String(), response.Status) - - // Close request.Body (if any) at this time just in case... - // - // It appears that net/http.Do() will actually return - // even if it has an outstanding Read() call to - // request.Body.Read() and calling request.Body.Close() - // will give it a chance to force request.Body.Read() - // to exit cleanly. - - if nil != request.Body { - _ = request.Body.Close() - } - } - - retryDelay = globals.retryDelay[retryIndex].nominal - time.Duration(rand.Int63n(int64(globals.retryDelay[retryIndex].variance))) - time.Sleep(retryDelay) - retryIndex++ - - _ = atomic.AddUint64(&globals.metrics.HTTPRequestRetries, 1) - } -} - -func fetchAuthToken() (swiftAuthToken string) { - var ( - swiftAuthWaitGroup *sync.WaitGroup - ) - - for { - globals.Lock() - - // Make a copy of globals.swiftAuthWaitGroup (if any) thus - // avoiding a race where, after the active instance of - // updateAuthTokenAndStorageURL() signals completion, it - // will erase it from globals (indicating no auth is in - // progress anymore) - - swiftAuthWaitGroup = globals.swiftAuthWaitGroup - - if nil == swiftAuthWaitGroup { - swiftAuthToken = globals.swiftAuthToken - globals.Unlock() - return - } - - globals.Unlock() - - swiftAuthWaitGroup.Wait() - } -} - -func fetchStorageURL() (swiftStorageURL string) { - var ( - swiftAuthWaitGroup *sync.WaitGroup - ) - - for { - globals.Lock() - - // Make a copy of globals.swiftAuthWaitGroup (if any) thus - // avoiding a race where, after the active instance of - // updateAuthTokenAndStorageURL() signals completion, it - // will erase it from globals (indicating no auth is in - // progress anymore) - - swiftAuthWaitGroup = globals.swiftAuthWaitGroup - - if nil == swiftAuthWaitGroup { - swiftStorageURL = globals.swiftStorageURL - globals.Unlock() - return - } - - globals.Unlock() - - swiftAuthWaitGroup.Wait() - } -} - -func updateAuthTokenAndStorageURL() { - var ( - authOut authOutStruct - err error - stderrChanBuf []byte - stderrChanBufChunk []byte - stdoutChanBuf []byte - stdoutChanBufChunk []byte - swiftAuthWaitGroup *sync.WaitGroup - ) - - // First check and see if another instance is already in-flight - - globals.Lock() - - // Make a copy of globals.swiftAuthWaitGroup (if any) thus - // avoiding a race where, after the active instance signals - // completion, it will erase it from globals (indicating no - // auth is in progress anymore) - - swiftAuthWaitGroup = globals.swiftAuthWaitGroup - - if nil != swiftAuthWaitGroup { - // Another instance is already in flight... just await its completion - - globals.Unlock() - - swiftAuthWaitGroup.Wait() - - return - } - - // We will be doing active instance performing the auth, - // so create a sync.WaitGroup for other instances and fetches to await - - globals.swiftAuthWaitGroup = &sync.WaitGroup{} - globals.swiftAuthWaitGroup.Add(1) - - globals.Unlock() - - if nil != globals.authPlugInControl { - // There seems to be an active authPlugIn... drain any bytes sent to stdoutChan first - - for { - select { - case _ = <-globals.authPlugInControl.stdoutChan: - default: - goto EscapeStdoutChanDrain - } - } - - EscapeStdoutChanDrain: - - // See if there is anything in stderrChan - - stderrChanBuf = make([]byte, 0, authPipeReadBufSize) - - for { - select { - case stderrChanBufChunk = <-globals.authPlugInControl.stderrChan: - stderrChanBuf = append(stderrChanBuf, stderrChanBufChunk...) - default: - goto EscapeStderrChanRead1 - } - } - - EscapeStderrChanRead1: - - if 0 < len(stderrChanBuf) { - logWarnf("got unexpected authPlugInStderr data: %s", string(stderrChanBuf[:])) - - stopAuthPlugIn() - } else { - // No errors... so lets try sending a byte to authPlugIn to request a fresh authorization - - _, err = globals.authPlugInControl.stdinPipe.Write([]byte{0}) - if nil != err { - logWarnf("got unexpected error sending SIGHUP to authPlugIn: %v", err) - - stopAuthPlugIn() - } - } - } - - if nil == globals.authPlugInControl { - // Either authPlugIn wasn't (thought to be) running, or it failed above... so (re)start it - - startAuthPlugIn() - } - - // Now read authPlugInStdout for a valid JSON-marshalled authOutStruct - - stdoutChanBuf = make([]byte, 0, authPipeReadBufSize) - - for { - select { - case stdoutChanBufChunk = <-globals.authPlugInControl.stdoutChan: - stdoutChanBuf = append(stdoutChanBuf, stdoutChanBufChunk...) - - // Perhaps we've received the entire authOutStruct - - err = json.Unmarshal(stdoutChanBuf, &authOut) - if nil == err { - // Got a clean JSON-formatted authOutStruct - - goto EscapeFetchAuthOut - } - case stderrChanBuf = <-globals.authPlugInControl.stderrChan: - // Uh oh... started receiving an error... drain it and "error out" - - for { - select { - case stderrChanBufChunk = <-globals.authPlugInControl.stderrChan: - stderrChanBuf = append(stderrChanBuf, stderrChanBufChunk...) - default: - goto EscapeStderrChanRead2 - } - } - - EscapeStderrChanRead2: - - logWarnf("got unexpected authPlugInStderr data: %s", string(stderrChanBuf[:])) - - authOut.AuthToken = "" - authOut.StorageURL = "" - - goto EscapeFetchAuthOut - } - } - -EscapeFetchAuthOut: - - globals.Lock() - - globals.swiftAuthToken = authOut.AuthToken - globals.swiftStorageURL = authOut.StorageURL - - // Finally, indicate to waiters we are done and also enable - // the next call to updateAuthTokenAndStorageURL() to perform - // the auth again - - globals.swiftAuthWaitGroup.Done() - globals.swiftAuthWaitGroup = nil - - globals.Unlock() -} - -func authPlugInPipeReader(pipeToRead io.ReadCloser, chanToWrite chan []byte, wg *sync.WaitGroup) { - var ( - buf []byte - eof bool - err error - n int - ) - - for { - buf = make([]byte, authPipeReadBufSize) - - n, err = pipeToRead.Read(buf) - switch err { - case nil: - eof = false - case io.EOF: - eof = true - default: - logFatalf("got unexpected error reading authPlugInPipe: %v", err) - } - - if 0 < n { - chanToWrite <- buf[:n] - } - - if eof { - wg.Done() - return // Exits this goroutine - } - } -} - -func startAuthPlugIn() { - var ( - err error - ) - - globals.authPlugInControl = &authPlugInControlStruct{ - cmd: exec.Command(globals.config.PlugInPath, globals.config.PlugInEnvName), - stdoutChan: make(chan []byte), - stderrChan: make(chan []byte), - } - - globals.authPlugInControl.stdinPipe, err = globals.authPlugInControl.cmd.StdinPipe() - if nil != err { - logFatalf("got unexpected error creating authPlugIn stdinPipe: %v", err) - } - globals.authPlugInControl.stdoutPipe, err = globals.authPlugInControl.cmd.StdoutPipe() - if nil != err { - logFatalf("got unexpected error creating authPlugIn stdoutPipe: %v", err) - } - globals.authPlugInControl.stderrPipe, err = globals.authPlugInControl.cmd.StderrPipe() - if nil != err { - logFatalf("got unexpected error creating authPlugIn stderrPipe: %v", err) - } - - globals.authPlugInControl.wg.Add(2) - - go authPlugInPipeReader(globals.authPlugInControl.stdoutPipe, globals.authPlugInControl.stdoutChan, &globals.authPlugInControl.wg) - go authPlugInPipeReader(globals.authPlugInControl.stderrPipe, globals.authPlugInControl.stderrChan, &globals.authPlugInControl.wg) - - if "" != globals.config.PlugInEnvValue { - globals.authPlugInControl.cmd.Env = append(os.Environ(), globals.config.PlugInEnvName+"="+globals.config.PlugInEnvValue) - } - - err = globals.authPlugInControl.cmd.Start() - if nil != err { - logFatalf("got unexpected error starting authPlugIn: %v", err) - } -} - -func stopAuthPlugIn() { - if nil == globals.authPlugInControl { - // No authPlugIn running - return - } - - // Stop authPlugIn (ignore errors since they just indicate authPlugIn failed) - - _ = globals.authPlugInControl.stdinPipe.Close() - - _ = globals.authPlugInControl.cmd.Wait() - - // Drain stdoutChan & stderrChan - - for { - select { - case _ = <-globals.authPlugInControl.stdoutChan: - default: - goto EscapeStdoutChanDrain - } - } - -EscapeStdoutChanDrain: - - for { - select { - case _ = <-globals.authPlugInControl.stderrChan: - default: - goto EscapeStderrChanDrain - } - } - -EscapeStderrChanDrain: - - // Tell stdoutPipe and stderrPipe readers to go away (ignore errors) - - _ = globals.authPlugInControl.stdoutPipe.Close() - _ = globals.authPlugInControl.stderrPipe.Close() - - // Wait for stdoutPipe and stderrPipe readers to exit - - globals.authPlugInControl.wg.Wait() - - // Finally, clean out authPlugInControl - - globals.authPlugInControl = nil -} diff --git a/pfsagentd/setup_teardown_test.go b/pfsagentd/setup_teardown_test.go deleted file mode 100644 index d55990be..00000000 --- a/pfsagentd/setup_teardown_test.go +++ /dev/null @@ -1,375 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "io/ioutil" - "net/http" - "os" - "strconv" - "sync" - "testing" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/proxyfsd" - "github.com/NVIDIA/proxyfs/ramswift" -) - -const ( - testAccountName = "AUTH_test" - testAuthKey = "testing" - testAuthUser = "test:tester" - testDaemonStartPollInterval = 1 * time.Second - testProxyFSDaemonHTTPPort = "15347" - testProxyFSDaemonIPAddr = "127.0.0.1" - testSwiftNoAuthIPAddr = "127.0.0.1" - testSwiftNoAuthPort = "38090" - testSwiftProxyAddr = "localhost:38080" - testRetryRPCPort = "54328" -) - -type testDaemonGlobalsStruct struct { - proxyfsdErrChan chan error - proxyfsdWG sync.WaitGroup - ramswiftDoneChan chan bool -} - -var testDaemonGlobals testDaemonGlobalsStruct - -func testSetup(t *testing.T) { - var ( - err error - infoResponse *http.Response - ramswiftSignalHandlerIsArmedWG sync.WaitGroup - testConfMap conf.ConfMap - testConfStrings []string - testDir string - testPlugInEnvValue string - versionResponse *http.Response - ) - - testDir, err = ioutil.TempDir(os.TempDir(), "pfsagentd_test_") - if nil != err { - t.Fatalf("ioutil.TempDir() failed: %v", err) - } - - err = os.Chdir(testDir) - if nil != err { - t.Fatalf("os.Chdir() failed: %v", err) - } - - err = os.Mkdir("ProxyFSMountPointPath", 0777) // Agent.FUSEMountPointPath - if nil != err { - t.Fatalf("os.Mkdir() failed: %v", err) - } - - err = os.Mkdir("PfsAgentMountPointPath", 0777) // Volume:CommonVolume.FUSEMountPointName - if nil != err { - t.Fatalf("os.Mkdir() failed: %v", err) - } - - testPlugInEnvValue = "{\"AuthURL\":\"http://" - testPlugInEnvValue += testSwiftProxyAddr - testPlugInEnvValue += "/auth/v1.0\"\\u002C\"AuthUser\":\"" - testPlugInEnvValue += testAuthUser - testPlugInEnvValue += "\"\\u002C\"AuthKey\":\"" - testPlugInEnvValue += testAuthKey - testPlugInEnvValue += "\"\\u002C\"Account\":\"" - testPlugInEnvValue += testAccountName - testPlugInEnvValue += "\"}" - - testConfStrings = []string{ - "Agent.FUSEVolumeName=CommonVolume", - "Agent.FUSEMountPointPath=PfsAgentMountPointPath", - "Agent.FUSEUnMountRetryDelay=100ms", - "Agent.FUSEUnMountRetryCap=100", - "Agent.PlugInPath=/dev/null", // Using hard-coded AUTH - "Agent.PlugInEnvName=SwiftAuthBlob", - "Agent.PlugInEnvValue=" + testPlugInEnvValue, - "Agent.SwiftTimeout=20s", - "Agent.SwiftRetryLimit=10", - "Agent.SwiftRetryDelay=10ms", - "Agent.SwiftRetryDelayVariance=25", - "Agent.SwiftRetryExpBackoff=1.4", - "Agent.SwiftConnectionPoolSize=200", - "Agent.FetchExtentsFromFileOffset=32", - "Agent.FetchExtentsBeforeFileOffset=0", - "Agent.ReadCacheLineSize=1048576", - "Agent.ReadCacheLineCount=1000", - "Agent.LeaseRetryLimit=10", - "Agent.LeaseRetryDelay=10ms", - "Agent.LeaseRetryDelayVariance=25", - "Agent.LeaseRetryExpBackoff=1.4", - "Agent.SharedLeaseLimit=1000", - "Agent.ExclusiveLeaseLimit=100", - "Agent.ExtentMapEntryLimit=1048576", - "Agent.DirtyLogSegmentLimit=50", - "Agent.DirtyFileLimit=50", // TODO - obsolete this - "Agent.MaxFlushSize=10485760", - "Agent.MaxFlushTime=10s", - "Agent.LogFilePath=", - "Agent.LogToConsole=false", - "Agent.TraceEnabled=false", - "Agent.HTTPServerIPAddr=127.0.0.1", - "Agent.HTTPServerTCPPort=54323", - "Agent.ReadDirPlusEnabled=false", - "Agent.XAttrEnabled=false", - "Agent.EntryDuration=10s", - "Agent.AttrDuration=10s", - "Agent.ReaddirMaxEntries=1024", - "Agent.FUSEMaxBackground=100", - "Agent.FUSECongestionThreshhold=0", - "Agent.FUSEMaxWrite=131072", // Linux max... 128KiB is good enough for testing - "Agent.FUSEAllowOther=false", - "Agent.RetryRPCPublicIPAddr=" + testProxyFSDaemonIPAddr, - "Agent.RetryRPCPort=" + testRetryRPCPort, - "Agent.RetryRPCDeadlineIO=60s", - "Agent.RetryRPCKeepAlivePeriod=60s", - "Agent.RetryRPCCACertFilePath=", - - "Stats.IPAddr=localhost", - "Stats.UDPPort=54324", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - - "StatsLogger.Period=0m", - "StatsLogger.Verbose=false", - - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - - "Peer:Peer0.PublicIPAddr=" + testProxyFSDaemonIPAddr, - "Peer:Peer0.PrivateIPAddr=" + testProxyFSDaemonIPAddr, - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - - "Cluster.WhoAmI=Peer0", - "Cluster.Peers=Peer0", - "Cluster.ServerGuid=a66488e9-a051-4ff7-865d-87bfb84cc2ae", - "Cluster.PrivateClusterUDPPort=54325", - "Cluster.UDPPacketSendSize=1400", - "Cluster.UDPPacketRecvSize=1500", - "Cluster.UDPPacketCapPerMessage=5", - "Cluster.HeartBeatDuration=1s", - "Cluster.HeartBeatMissLimit=3", - "Cluster.MessageQueueDepthPerPeer=4", - "Cluster.MaxRequestDuration=1s", - "Cluster.LivenessCheckRedundancy=2", - - "HTTPServer.TCPPort=" + testProxyFSDaemonHTTPPort, - - "SwiftClient.NoAuthIPAddr=" + testSwiftNoAuthIPAddr, - "SwiftClient.NoAuthTCPPort=" + testSwiftNoAuthPort, - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=1", - "SwiftClient.RetryLimitObject=1", - "SwiftClient.RetryDelay=10ms", - "SwiftClient.RetryDelayObject=10ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=64", - "SwiftClient.NonChunkedConnectionPoolSize=32", - "SwiftClient.SwiftReconChecksPerConfCheck=0", - - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000", - - "Volume:CommonVolume.FSID=1", - "Volume:CommonVolume.FUSEMountPointName=ProxyFSMountPointPath", - "Volume:CommonVolume.NFSExportClientMapList=CommonVolumeNFSClient0", - "Volume:CommonVolume.SMBShareName=CommonShare", - "Volume:CommonVolume.PrimaryPeer=Peer0", - "Volume:CommonVolume.AccountName=AUTH_test", - "Volume:CommonVolume.CheckpointContainerName=.__checkpoint__", - "Volume:CommonVolume.CheckpointContainerStoragePolicy=gold", - "Volume:CommonVolume.CheckpointInterval=10s", - "Volume:CommonVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way", - "Volume:CommonVolume.MaxFlushSize=10485760", - "Volume:CommonVolume.MaxFlushTime=10s", - "Volume:CommonVolume.FileDefragmentChunkSize=10485760", - "Volume:CommonVolume.FileDefragmentChunkDelay=10ms", - "Volume:CommonVolume.NonceValuesToReserve=100", - "Volume:CommonVolume.MaxEntriesPerDirNode=32", - "Volume:CommonVolume.MaxExtentsPerFileNode=32", - "Volume:CommonVolume.MaxInodesPerMetadataNode=32", - "Volume:CommonVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:CommonVolume.MaxDirFileNodesPerMetadataNode=16", - "Volume:CommonVolume.MaxBytesInodeCache=100000", - "Volume:CommonVolume.InodeCacheEvictInterval=1s", - "Volume:CommonVolume.AutoFormat=true", - "Volume:CommonVolume.ActiveLeaseEvictLowLimit=5000", - "Volume:CommonVolume.ActiveLeaseEvictHighLimit=5010", - - "NFSClientMap:CommonVolumeNFSClient0.ClientPattern=*", - "NFSClientMap:CommonVolumeNFSClient0.AccessMode=rw", - "NFSClientMap:CommonVolumeNFSClient0.RootSquash=no_root_squash", - "NFSClientMap:CommonVolumeNFSClient0.Secure=insecure", - - "VolumeGroup:CommonVolumeGroup.VolumeList=CommonVolume", - "VolumeGroup:CommonVolumeGroup.VirtualIPAddr=", - "VolumeGroup:CommonVolumeGroup.PrimaryPeer=Peer0", - "VolumeGroup:CommonVolumeGroup.ReadCacheLineSize=1000000", - "VolumeGroup:CommonVolumeGroup.ReadCacheWeight=100", - - "FSGlobals.VolumeGroupList=CommonVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.DirEntryCacheEvictLowLimit=10000", - "FSGlobals.DirEntryCacheEvictHighLimit=10010", - "FSGlobals.FileExtentMapEvictLowLimit=10000", - "FSGlobals.FileExtentMapEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - - "JSONRPCServer.TCPPort=54326", - "JSONRPCServer.FastTCPPort=54327", - "JSONRPCServer.RetryRPCPort=" + testRetryRPCPort, - "JSONRPCServer.RetryRPCTTLCompleted=10s", - "JSONRPCServer.RetryRPCAckTrim=10ms", - "JSONRPCServer.RetryRPCCertFilePath=", - "JSONRPCServer.RetryRPCKeyFilePath=", - "JSONRPCServer.DataPathLogging=false", - "JSONRPCServer.MinLeaseDuration=250ms", - "JSONRPCServer.LeaseInterruptInterval=250ms", - "JSONRPCServer.LeaseInterruptLimit=20", - } - - testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxAccountNameLength="+strconv.FormatUint(testMaxAccountNameLength, 10)) - testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxContainerNameLength="+strconv.FormatUint(testMaxContainerNameLength, 10)) - testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxObjectNameLength="+strconv.FormatUint(testMaxObjectNameLength, 10)) - testConfStrings = append(testConfStrings, "RamSwiftInfo.AccountListingLimit="+strconv.FormatUint(testAccountListingLimit, 10)) - testConfStrings = append(testConfStrings, "RamSwiftInfo.ContainerListingLimit="+strconv.FormatUint(testContainerListingLimit, 10)) - - ramswiftSignalHandlerIsArmedWG.Add(1) - testDaemonGlobals.ramswiftDoneChan = make(chan bool, 1) - - go ramswift.Daemon("/dev/null", testConfStrings, &ramswiftSignalHandlerIsArmedWG, testDaemonGlobals.ramswiftDoneChan, unix.SIGUSR1) - - ramswiftSignalHandlerIsArmedWG.Wait() - - for { - infoResponse, err = http.Get("http://" + testSwiftNoAuthIPAddr + ":" + testSwiftNoAuthPort + "/info") - if (nil == err) && (http.StatusOK == infoResponse.StatusCode) { - break - } - - time.Sleep(testDaemonStartPollInterval) - } - - testDaemonGlobals.proxyfsdErrChan = make(chan error, 1) // Must be buffered to avoid race - - go proxyfsd.Daemon("/dev/null", testConfStrings, testDaemonGlobals.proxyfsdErrChan, &testDaemonGlobals.proxyfsdWG, []string{}, unix.SIGUSR2) - - err = <-testDaemonGlobals.proxyfsdErrChan - if nil != err { - t.Fatalf("proxyfsd.Daemon() startup failed: %v", err) - } - - for { - versionResponse, err = http.Get("http://" + testProxyFSDaemonIPAddr + ":" + testProxyFSDaemonHTTPPort + "/version") - if (nil == err) && (http.StatusOK == versionResponse.StatusCode) { - break - } - - time.Sleep(testDaemonStartPollInterval) - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - globals.logFile = nil - globals.config.LogFilePath = "" - globals.config.LogToConsole = true - - startSwiftProxyEmulator(t, testConfMap) - - initializeGlobals(testConfMap) - - // Fake out that plug-in auth has already obtained AuthToken & StorageURL - - globals.swiftAuthToken = testAuthToken - globals.swiftStorageURL = "http://" + testSwiftProxyAddr + "/proxyfs/" + testAccountName - - go testSwallowFissionErrChan(t, globals.fissionErrChan) - - doMountProxyFS() - - performMountFUSE() -} - -func testSwallowFissionErrChan(t *testing.T, fissionErrChan chan error) { - var ( - err error - ) - - err = <-fissionErrChan - if nil != err { - t.Fatalf("fissionErrChan received err: %v", err) - } -} - -func testTeardown(t *testing.T) { - var ( - err error - testDir string - ) - - performUnmountFUSE() - - emptyFileInodeDirtyListAndLogSegmentChan() - - doUnmountProxyFS() - - uninitializeGlobals() - - _ = unix.Kill(unix.Getpid(), unix.SIGUSR2) - - err = <-testDaemonGlobals.proxyfsdErrChan - - testDaemonGlobals.proxyfsdWG.Wait() - - if nil != err { - t.Fatalf("proxyfsd.Daemon() exited with error: %v", err) - } - - unix.Kill(unix.Getpid(), unix.SIGUSR1) - - _ = <-testDaemonGlobals.ramswiftDoneChan - - testDir, err = os.Getwd() - if nil != err { - t.Fatalf("os.Getwd() failed: %v", err) - } - - err = os.Chdir("..") - if nil != err { - t.Fatalf("os.Chdir() failed: %v", err) - } - - err = os.RemoveAll(testDir) - if nil != err { - t.Fatalf("os.RemoveAll() failed: %v", err) - } - - stopSwiftProxyEmulator() -} diff --git a/pfsagentd/swift_proxy_emulator_test.go b/pfsagentd/swift_proxy_emulator_test.go deleted file mode 100644 index 48511fa4..00000000 --- a/pfsagentd/swift_proxy_emulator_test.go +++ /dev/null @@ -1,411 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "context" - "io/ioutil" - "net" - "net/http" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/conf" -) - -const ( - testAuthToken = "AUTH_tkTestToken" - testJrpcResponseBufSize = 1024 * 1024 -) - -type testSwiftProxyEmulatorGlobalsStruct struct { - sync.WaitGroup - t *testing.T - ramswiftNoAuthURL string - proxyfsdJrpcTCPAddr *net.TCPAddr - jrpcResponsePool *sync.Pool - httpClient *http.Client - httpServer *http.Server -} - -var testSwiftProxyEmulatorGlobals testSwiftProxyEmulatorGlobalsStruct - -func startSwiftProxyEmulator(t *testing.T, confMap conf.ConfMap) { - var ( - err error - infoResponse *http.Response - jrpcServerIPAddr string - jrpcServerTCPPort uint16 - swiftClientNoAuthIPAddr string - swiftClientNoAuthTCPPort uint16 - whoAmI string - ) - - testSwiftProxyEmulatorGlobals.t = t - - swiftClientNoAuthIPAddr, err = confMap.FetchOptionValueString("SwiftClient", "NoAuthIPAddr") - if nil != err { - t.Fatal(err) - } - - swiftClientNoAuthTCPPort, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") - if nil != err { - t.Fatal(err) - } - - testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL = "http://" + net.JoinHostPort(swiftClientNoAuthIPAddr, strconv.FormatUint(uint64(swiftClientNoAuthTCPPort), 10)) - - whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - t.Fatal(err) - } - - jrpcServerIPAddr, err = confMap.FetchOptionValueString("Peer:"+whoAmI, "PrivateIPAddr") - if nil != err { - t.Fatal(err) - } - - jrpcServerTCPPort, err = confMap.FetchOptionValueUint16("JSONRPCServer", "TCPPort") - if nil != err { - t.Fatal(err) - } - - testSwiftProxyEmulatorGlobals.proxyfsdJrpcTCPAddr, err = net.ResolveTCPAddr("tcp", net.JoinHostPort(jrpcServerIPAddr, strconv.FormatUint(uint64(jrpcServerTCPPort), 10))) - if nil != err { - t.Fatal(err) - } - - testSwiftProxyEmulatorGlobals.httpClient = &http.Client{} - - testSwiftProxyEmulatorGlobals.httpServer = &http.Server{ - Addr: testSwiftProxyAddr, - Handler: &testSwiftProxyEmulatorGlobals, - } - - testSwiftProxyEmulatorGlobals.jrpcResponsePool = &sync.Pool{ - New: func() (bufAsInterface interface{}) { - var ( - bufAsByteSlice []byte - ) - - bufAsByteSlice = make([]byte, testJrpcResponseBufSize) - - bufAsInterface = bufAsByteSlice - - return - }, - } - - testSwiftProxyEmulatorGlobals.Add(1) - - go func() { - _ = testSwiftProxyEmulatorGlobals.httpServer.ListenAndServe() - - testSwiftProxyEmulatorGlobals.Done() - }() - - for { - infoResponse, err = http.Get("http://" + testSwiftProxyAddr + "/info") - if nil == err { - break - } - - time.Sleep(testDaemonStartPollInterval) - } - - if http.StatusOK != infoResponse.StatusCode { - t.Fatalf("GET /info from ServeHTTP() got unexpected status %s", infoResponse.Status) - } -} - -func stopSwiftProxyEmulator() { - var ( - err error - ) - - err = testSwiftProxyEmulatorGlobals.httpServer.Shutdown(context.Background()) - if nil != err { - testSwiftProxyEmulatorGlobals.t.Fatalf("testSwiftProxyEmulatorGlobals.httpServer.Shutdown() failed: %v", err) - } - - testSwiftProxyEmulatorGlobals.Wait() - - testSwiftProxyEmulatorGlobals.jrpcResponsePool = nil -} - -func (dummy *testSwiftProxyEmulatorGlobalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - // Handle the GET of/on info & AuthURL cases - - if http.MethodGet == request.Method { - switch request.URL.Path { - case "/info": - doInfo(responseWriter) - return - case "/auth/v1.0": - doAuth(responseWriter, request) - return - default: - // Fall through to normal processing - } - } - - // Reject unauthorized requests - - if request.Header.Get("X-Auth-Token") != testAuthToken { - responseWriter.WriteHeader(http.StatusUnauthorized) - return - } - - // Branch off to individual request method handlers - - switch request.Method { - case http.MethodGet: - doGET(responseWriter, request) - case http.MethodPut: - doPUT(responseWriter, request) - case "PROXYFS": - doRPC(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func doInfo(authResponseWriter http.ResponseWriter) { - var ( - err error - getBuf []byte - noAuthResponse *http.Response - ) - - noAuthResponse, err = http.Get(testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL + "/info") - if nil != err { - testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info from ramswift failed: %v", err) - } - - if http.StatusOK != noAuthResponse.StatusCode { - testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info from ramswift returned bad status: %v", noAuthResponse.Status) - } - - getBuf, err = ioutil.ReadAll(noAuthResponse.Body) - if nil != err { - testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info returned unreadable Body: %v", err) - } - - err = noAuthResponse.Body.Close() - if nil != err { - testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info returned uncloseable Body: %v", err) - } - - authResponseWriter.WriteHeader(http.StatusOK) - _, _ = authResponseWriter.Write(getBuf) -} - -func doAuth(authResponseWriter http.ResponseWriter, authRequest *http.Request) { - if authRequest.Header.Get("X-Auth-User") != testAuthUser { - authResponseWriter.WriteHeader(http.StatusUnauthorized) - return - } - if authRequest.Header.Get("X-Auth-Key") != testAuthKey { - authResponseWriter.WriteHeader(http.StatusUnauthorized) - return - } - authResponseWriter.Header().Add("X-Auth-Token", testAuthToken) - authResponseWriter.Header().Add("X-Storage-Url", "http://"+testSwiftProxyAddr+"/v1/"+testAccountName) - authResponseWriter.WriteHeader(http.StatusOK) -} - -// doGET proxies the GET over to ramswift -// -func doGET(authResponseWriter http.ResponseWriter, authRequest *http.Request) { - var ( - contentRangeHeader string - contentTypeHeader string - err error - getBuf []byte - hostHeader string - noAuthPath string - noAuthRequest *http.Request - noAuthResponse *http.Response - noAuthStatusCode int - rangeHeader string - ) - - if !strings.HasPrefix(authRequest.URL.Path, "/proxyfs/"+testAccountName) { - authResponseWriter.WriteHeader(http.StatusNotFound) - return - } - - noAuthPath = strings.Replace(authRequest.URL.Path, "proxyfs", "v1", 1) - - noAuthRequest, err = http.NewRequest("GET", testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL+noAuthPath, nil) - if nil != err { - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - hostHeader = authRequest.Header.Get("Host") - if "" != hostHeader { - noAuthRequest.Header.Add("Host", hostHeader) - } - - rangeHeader = authRequest.Header.Get("Range") - if "" != rangeHeader { - noAuthRequest.Header.Add("Range", rangeHeader) - } - - noAuthResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(noAuthRequest) - if nil != err { - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - noAuthStatusCode = noAuthResponse.StatusCode - - if (http.StatusOK != noAuthStatusCode) && (http.StatusPartialContent != noAuthStatusCode) { - _ = noAuthResponse.Body.Close() - authResponseWriter.WriteHeader(noAuthStatusCode) - return - } - - getBuf, err = ioutil.ReadAll(noAuthResponse.Body) - if nil != err { - _ = noAuthResponse.Body.Close() - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - err = noAuthResponse.Body.Close() - if nil != err { - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - contentTypeHeader = noAuthResponse.Header.Get("Content-Type") - if "" != contentTypeHeader { - authResponseWriter.Header().Add("Content-Type", contentTypeHeader) - } - - contentRangeHeader = noAuthResponse.Header.Get("Content-Range") - if "" != contentRangeHeader { - authResponseWriter.Header().Add("Content-Range", contentRangeHeader) - } - - authResponseWriter.WriteHeader(noAuthStatusCode) - - _, _ = authResponseWriter.Write(getBuf) -} - -// doPUT proxies the GET over to ramswift -// -func doPUT(authResponseWriter http.ResponseWriter, authRequest *http.Request) { - var ( - err error - hostHeader string - noAuthPath string - noAuthRequest *http.Request - noAuthResponse *http.Response - ) - - if !strings.HasPrefix(authRequest.URL.Path, "/proxyfs/"+testAccountName) { - _ = authRequest.Body.Close() - authResponseWriter.WriteHeader(http.StatusNotFound) - return - } - - noAuthPath = strings.Replace(authRequest.URL.Path, "proxyfs", "v1", 1) - - noAuthRequest, err = http.NewRequest("PUT", testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL+noAuthPath, authRequest.Body) - if nil != err { - _ = authRequest.Body.Close() - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - hostHeader = authRequest.Header.Get("Host") - if "" != hostHeader { - noAuthRequest.Header.Add("Host", hostHeader) - } - - noAuthResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(noAuthRequest) - if nil != err { - _ = authRequest.Body.Close() - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - err = authRequest.Body.Close() - if nil != err { - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - authResponseWriter.WriteHeader(noAuthResponse.StatusCode) -} - -// doRPC proxies the payload as a JSON RPC request over to proxyfsd -// -func doRPC(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - jrpcResponseBuf []byte - jrpcResponseLen int - jrpcRequestBuf []byte - tcpConn *net.TCPConn - ) - - if !strings.HasPrefix(request.URL.Path, "/proxyfs/"+testAccountName) { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - jrpcRequestBuf, err = ioutil.ReadAll(request.Body) - _ = request.Body.Close() - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - if request.Header.Get("Content-Type") != "application/json" { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - tcpConn, err = net.DialTCP("tcp", nil, testSwiftProxyEmulatorGlobals.proxyfsdJrpcTCPAddr) - if nil != err { - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - _, err = tcpConn.Write(jrpcRequestBuf) - if nil != err { - _ = tcpConn.Close() - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - jrpcResponseBuf = testSwiftProxyEmulatorGlobals.jrpcResponsePool.Get().([]byte) - - jrpcResponseLen, err = tcpConn.Read(jrpcResponseBuf) - if nil != err { - _ = tcpConn.Close() - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - err = tcpConn.Close() - if nil != err { - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - responseWriter.Header().Add("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(jrpcResponseBuf[:jrpcResponseLen]) - - testSwiftProxyEmulatorGlobals.jrpcResponsePool.Put(jrpcResponseBuf) -} diff --git a/pfsagentd/unit_test.go b/pfsagentd/unit_test.go deleted file mode 100644 index 2a168bf6..00000000 --- a/pfsagentd/unit_test.go +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/jrpcfs" -) - -const ( - testMaxAccountNameLength = 256 - testMaxContainerNameLength = 512 - testMaxObjectNameLength = 1024 - testAccountListingLimit = 10000 - testContainerListingLimit = 20000 -) - -type testSwiftInfoInnerStruct struct { - MaxAccountNameLength uint64 `json:"max_account_name_length"` - MaxContainerNameLength uint64 `json:"max_container_name_length"` - MaxObjectNameLength uint64 `json:"max_object_name_length"` - AccountListingLimit uint64 `json:"account_listing_limit"` - ContainerListingLimit uint64 `json:"container_listing_limit"` -} - -type testSwiftInfoOuterStruct struct { - Swift testSwiftInfoInnerStruct `json:"swift"` -} - -func TestJRPCRequest(t *testing.T) { - var ( - err error - pingReply *jrpcfs.PingReply - pingReq *jrpcfs.PingReq - ) - - testSetup(t) - - pingReq = &jrpcfs.PingReq{ - Message: "TestMessage", - } - - pingReply = &jrpcfs.PingReply{} - - err = doJRPCRequest("Server.RpcPing", pingReq, pingReply) - - if nil != err { - t.Fatalf("doJRPCRequest(\"Server.RpcPing\",,) failed: %v", err) - } - if fmt.Sprintf("pong %d bytes", len("TestMessage")) != pingReply.Message { - t.Fatalf("doJRPCRequest(\"Server.RpcPing\",,) returned unexpected response: %s", pingReply.Message) - } - - testTeardown(t) -} - -func TestSwiftProxyEmulation(t *testing.T) { - var ( - authRequest *http.Request - authResponse *http.Response - authURL string - err error - getBuf []byte - getObjectRequest *http.Request - getObjectResponse *http.Response - infoBuf []byte - infoResponse *http.Response - infoURL string - pingReq *jrpcfs.PingReq - pingReqBuf []byte - pingReqMessageID uint64 - pingRequest *http.Request - pingReply *jrpcfs.PingReply - pingReplyBuf []byte - pingReplyMessageID uint64 - pingResponse *http.Response - putContainerRequest *http.Request - putContainerResponse *http.Response - putObjectRequest *http.Request - putObjectResponse *http.Response - responseErr error - swiftAccountURLSplit []string - swiftInfo *testSwiftInfoOuterStruct - unmarshalErr error - ) - - testSetup(t) - - for { - swiftAccountURLSplit = strings.Split(globals.swiftStorageURL, "/") - if len(swiftAccountURLSplit) >= 3 { - infoURL = strings.Join(append(strings.Split(globals.swiftStorageURL, "/")[:3], "info"), "/") - break - } - time.Sleep(100 * time.Millisecond) - } - - infoResponse, err = http.Get(infoURL) - if nil != err { - t.Fatalf("GET /info failed: %v", err) - } - if http.StatusOK != infoResponse.StatusCode { - t.Fatalf("GET /info returned bad status: %v", infoResponse.Status) - } - - infoBuf, err = ioutil.ReadAll(infoResponse.Body) - if nil != err { - t.Fatalf("GET /info returned unreadable Body: %v", err) - } - - err = infoResponse.Body.Close() - if nil != err { - t.Fatalf("GET /info returned uncloseable Body: %v", err) - } - - swiftInfo = &testSwiftInfoOuterStruct{} - - err = json.Unmarshal(infoBuf, swiftInfo) - if nil != err { - t.Fatalf("GET /info returned unparseable Body: %v", err) - } - - if (testMaxAccountNameLength != swiftInfo.Swift.MaxAccountNameLength) || - (testMaxContainerNameLength != swiftInfo.Swift.MaxContainerNameLength) || - (testMaxObjectNameLength != swiftInfo.Swift.MaxObjectNameLength) || - (testAccountListingLimit != swiftInfo.Swift.AccountListingLimit) || - (testContainerListingLimit != swiftInfo.Swift.ContainerListingLimit) { - t.Fatalf("GET /info returned unexpected swiftInfo") - } - - authURL = strings.Join(append(strings.Split(globals.swiftStorageURL, "/")[:3], "auth", "v1.0"), "/") - - authRequest, err = http.NewRequest("GET", authURL, nil) - if nil != err { - t.Fatalf("Creating GET /auth/v1.0 http.Request failed: %v", err) - } - - authRequest.Header.Add("X-Auth-User", testAuthUser) - authRequest.Header.Add("X-Auth-Key", testAuthKey) - - authResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(authRequest) - if nil != err { - t.Fatalf("GET /auth/v1.0 failed: %v", err) - } - if http.StatusOK != authResponse.StatusCode { - t.Fatalf("GET /auth/v1.0 returned bad status: %v", authResponse.Status) - } - - err = authResponse.Body.Close() - if nil != err { - t.Fatalf("GET /auth/v1.0 returned uncloseable Body: %v", err) - } - - if testAuthToken != authResponse.Header.Get("X-Auth-Token") { - t.Fatalf("GET /auth/v1.0 returned incorrect X-Auth-Token") - } - - if "http://"+testSwiftProxyAddr+"/v1/"+testAccountName != authResponse.Header.Get("X-Storage-Url") { - t.Fatalf("GET /auth/v1.0 returned incorrect X-Storage-Url") - } - - putContainerRequest, err = http.NewRequest("PUT", globals.swiftStorageURL+"/TestContainer", nil) - if nil != err { - t.Fatalf("Creating PUT .../TestContainer failed: %v", err) - } - - putContainerRequest.Header.Add("X-Auth-Token", testAuthToken) - - putContainerResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(putContainerRequest) - if nil != err { - t.Fatalf("PUT .../TestContainer failed: %v", err) - } - if http.StatusCreated != putContainerResponse.StatusCode { - t.Fatalf("PUT .../TestContainer returned bad status: %v", putContainerResponse.Status) - } - - putObjectRequest, err = http.NewRequest("PUT", globals.swiftStorageURL+"/TestContainer/TestObject", bytes.NewReader([]byte{0x00, 0x01, 0x02, 0x03, 0x04})) - if nil != err { - t.Fatalf("Creating PUT .../TestContainer/TestObject failed: %v", err) - } - - putObjectRequest.Header.Add("X-Auth-Token", testAuthToken) - - putObjectResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(putObjectRequest) - if nil != err { - t.Fatalf("PUT .../TestContainer/TestObject failed: %v", err) - } - if http.StatusCreated != putObjectResponse.StatusCode { - t.Fatalf("PUT .../TestContainer/TestObject returned bad status: %v", putObjectResponse.Status) - } - - getObjectRequest, err = http.NewRequest("GET", globals.swiftStorageURL+"/TestContainer/TestObject", nil) - if nil != err { - t.Fatalf("Creating GET .../TestContainer/TestObject failed: %v", err) - } - - getObjectRequest.Header.Add("X-Auth-Token", testAuthToken) - - getObjectRequest.Header.Add("Range", "bytes=1-3") - - getObjectResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(getObjectRequest) - if nil != err { - t.Fatalf("GET .../TestContainer/TestObject failed: %v", err) - } - if http.StatusPartialContent != getObjectResponse.StatusCode { - t.Fatalf("GET .../TestContainer/TestObject returned bad status: %v", getObjectResponse.Status) - } - - getBuf, err = ioutil.ReadAll(getObjectResponse.Body) - if nil != err { - t.Fatalf("GET .../TestContainer/TestObject returned unreadable Body: %v", err) - } - - err = getObjectResponse.Body.Close() - if nil != err { - t.Fatalf("GET .../TestContainer/TestObject returned uncloseable Body: %v", err) - } - - if (3 != len(getBuf)) || - (0x01 != getBuf[0]) || - (0x02 != getBuf[1]) || - (0x03 != getBuf[2]) { - t.Fatalf("GET .../TestContainer/TestObject returned unexpected contents") - } - - pingReq = &jrpcfs.PingReq{ - Message: "TestMessage", - } - - pingReqMessageID, pingReqBuf, err = jrpcMarshalRequest("Server.RpcPing", pingReq) - if nil != err { - t.Fatalf("Marshaling pingReq failed: %v", err) - } - - pingRequest, err = http.NewRequest("PROXYFS", globals.swiftStorageURL, bytes.NewReader(pingReqBuf)) - if nil != err { - t.Fatalf("Creating PROXYFS pingReq failed: %v", err) - } - - pingRequest.Header.Add("X-Auth-Token", testAuthToken) - pingRequest.Header.Add("Content-Type", "application/json") - - pingResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(pingRequest) - if nil != err { - t.Fatalf("PROXYFS pingReq failed: %v", err) - } - if http.StatusOK != pingResponse.StatusCode { - t.Fatalf("PROXYFS pingReq returned bad status: %v", pingResponse.Status) - } - - pingReplyBuf, err = ioutil.ReadAll(pingResponse.Body) - if nil != err { - t.Fatalf("PROXYFS pingReq returned unreadable Body: %v", err) - } - - err = pingResponse.Body.Close() - if nil != err { - t.Fatalf("PROXYFS pingReq returned uncloseable Body: %v", err) - } - - pingReplyMessageID, responseErr, unmarshalErr = jrpcUnmarshalResponseForIDAndError(pingReplyBuf) - if nil != unmarshalErr { - t.Fatalf("Unmarshaling ID & Error failed: %v", unmarshalErr) - } - if nil != responseErr { - t.Fatalf("Unmarshaling ID & Error returned unexpected Error") - } - if pingReqMessageID != pingReplyMessageID { - t.Fatal("Unmarshaling ID & Error returned unexpected ID") - } - - pingReply = &jrpcfs.PingReply{} - - err = jrpcUnmarshalResponse(pingReqMessageID, pingReplyBuf, pingReply) - if nil != err { - t.Fatalf("Unmarshaling pingReply failed: %v", err) - } - if fmt.Sprintf("pong %d bytes", len("TestMessage")) != pingReply.Message { - t.Fatalf("PROXYFS pingReply returned unexpected Message") - } - - testTeardown(t) -} - -func TestJRPCMarshaling(t *testing.T) { - var ( - marshalErr error - pingReplyBuf []byte - pingReplyInput *jrpcfs.PingReply - pingReplyInputRequestID uint64 - pingReplyOutput *jrpcfs.PingReply - pingReplyOutputRequestID uint64 - pingReqBuf []byte - pingReqInput *jrpcfs.PingReq - pingReqInputRequestID uint64 - pingReqOutput *jrpcfs.PingReq - pingReqOutputRequestID uint64 - requestMethod string - responseErr error - unmarshalErr error - ) - - pingReqInput = &jrpcfs.PingReq{ - Message: "TestMessage", - } - - pingReqInputRequestID, pingReqBuf, marshalErr = jrpcMarshalRequest("Server.RpcPing", pingReqInput) - if nil != marshalErr { - t.Fatalf("jrpcMarshalRequest() failed: %v", marshalErr) - } - - requestMethod, pingReqOutputRequestID, unmarshalErr = jrpcUnmarshalRequestForMethodAndID(pingReqBuf) - if nil != unmarshalErr { - t.Fatalf("jrpcUnmarshalRequestForMethod() failed: %v", unmarshalErr) - } - if "Server.RpcPing" != requestMethod { - t.Fatalf("jrpcUnmarshalRequestForMethod() returned unexpected requestMethod") - } - if pingReqInputRequestID != pingReqOutputRequestID { - t.Fatalf("jrpcUnmarshalRequest() returned unexpected requestID") - } - - pingReqOutput = &jrpcfs.PingReq{} - - unmarshalErr = jrpcUnmarshalRequest(pingReqOutputRequestID, pingReqBuf, pingReqOutput) - if nil != unmarshalErr { - t.Fatalf("jrpcUnmarshalRequest() failed: %v", unmarshalErr) - } - if "TestMessage" != pingReqOutput.Message { - t.Fatalf("jrpcUnmarshalRequest() returned unexpected jrpcfs.PingReq.Message") - } - - pingReplyInputRequestID = pingReqOutputRequestID - - pingReplyInput = &jrpcfs.PingReply{ - Message: "TestMessage", - } - - pingReplyBuf, marshalErr = jrpcMarshalResponse(pingReplyInputRequestID, nil, pingReplyInput) - if nil != marshalErr { - t.Fatalf("jrpcMarshalResponse(,nil-responseErr,non-nil-response) failed: %v", marshalErr) - } - - pingReplyOutputRequestID, responseErr, unmarshalErr = jrpcUnmarshalResponseForIDAndError(pingReplyBuf) - if nil != unmarshalErr { - t.Fatalf("jrpcUnmarshalResponseForIDAndError() failed: %v", unmarshalErr) - } - if nil != responseErr { - t.Fatalf("jrpcUnmarshalResponseForIDAndError() returned unexpected responseErr") - } - if pingReplyInputRequestID != pingReplyOutputRequestID { - t.Fatalf("jrpcUnmarshalResponseForIDAndError() returned unexpected requestID") - } - - pingReplyOutput = &jrpcfs.PingReply{} - - unmarshalErr = jrpcUnmarshalResponse(pingReplyOutputRequestID, pingReplyBuf, pingReplyOutput) - if nil != unmarshalErr { - t.Fatalf("jrpcUnmarshalResponse() failed: %v", unmarshalErr) - } - if "TestMessage" != pingReplyOutput.Message { - t.Fatalf("jrpcUnmarshalRequest() returned unexpected jrpcfs.PingReply.Message") - } - - pingReplyBuf, marshalErr = jrpcMarshalResponse(pingReplyInputRequestID, nil, nil) - if nil != marshalErr { - t.Fatalf("jrpcMarshalResponse(,nil-responseErr,nil-response) failed: %v", marshalErr) - } - - pingReplyBuf, marshalErr = jrpcMarshalResponse(pingReplyInputRequestID, fmt.Errorf("TestError"), nil) - if nil != marshalErr { - t.Fatalf("jrpcMarshalResponse(,non-nil-responseErr,nil-response) failed: %v", marshalErr) - } - - pingReplyOutputRequestID, responseErr, unmarshalErr = jrpcUnmarshalResponseForIDAndError(pingReplyBuf) - if nil != unmarshalErr { - t.Fatalf("jrpcUnmarshalResponseForIDAndError() failed: %v", unmarshalErr) - } - if (nil == responseErr) || ("TestError" != responseErr.Error()) { - t.Fatalf("jrpcUnmarshalResponseForIDAndError() returned unexpected responseErr") - } - if pingReplyInputRequestID != pingReplyOutputRequestID { - t.Fatalf("jrpcUnmarshalResponseForIDAndError() returned unexpected requestID") - } - - pingReplyBuf, marshalErr = jrpcMarshalResponse(pingReplyInputRequestID, fmt.Errorf("TestError"), pingReplyOutput) - if nil != marshalErr { - t.Fatalf("jrpcMarshalResponse(,non-nil-responseErr,non-nil-response) failed: %v", marshalErr) - } -} diff --git a/pfsconfjson/Makefile b/pfsconfjson/Makefile deleted file mode 100644 index 63e2cd57..00000000 --- a/pfsconfjson/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfsconfjson - -include ../GoMakefile diff --git a/pfsconfjson/dummy_test.go b/pfsconfjson/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfsconfjson/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfsconfjson/main.go b/pfsconfjson/main.go deleted file mode 100644 index 180db517..00000000 --- a/pfsconfjson/main.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "log" - "os" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/transitions" -) - -func main() { - var ( - args []string - confErr error - confMap conf.ConfMap - confMapJSON bytes.Buffer - confMapJSONPacked []byte - ) - - args = os.Args[1:] - - // Read in the program's os.Arg[1]-specified (and required) .conf file - if len(args) == 0 { - log.Fatalf("no .conf file specified") - } - - confMap, confErr = conf.MakeConfMapFromFile(args[0]) - if nil != confErr { - log.Fatalf("failed to load config: %v", confErr) - } - - // Update confMap with any extra os.Args supplied - confErr = confMap.UpdateFromStrings(args[1:]) - if nil != confErr { - log.Fatalf("failed to load config overrides: %v", confErr) - } - - confErr = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != confErr { - log.Fatalf("failed to upgrade config: %v", confErr) - } - - confMapJSONPacked, _ = json.Marshal(confMap) - json.Indent(&confMapJSON, confMapJSONPacked, "", "\t") - - fmt.Printf("%v\n", confMapJSON.String()) -} diff --git a/pfsconfjsonpacked/Makefile b/pfsconfjsonpacked/Makefile deleted file mode 100644 index 069317fb..00000000 --- a/pfsconfjsonpacked/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfsconfjsonpacked - -include ../GoMakefile diff --git a/pfsconfjsonpacked/dummy_test.go b/pfsconfjsonpacked/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfsconfjsonpacked/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfsconfjsonpacked/main.go b/pfsconfjsonpacked/main.go deleted file mode 100644 index fcd9576f..00000000 --- a/pfsconfjsonpacked/main.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "encoding/json" - "fmt" - "log" - "os" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/transitions" -) - -func main() { - args := os.Args[1:] - - // Read in the program's os.Arg[1]-specified (and required) .conf file - if len(args) == 0 { - log.Fatalf("no .conf file specified") - } - - confMap, confErr := conf.MakeConfMapFromFile(args[0]) - if nil != confErr { - log.Fatalf("failed to load config: %v", confErr) - } - - // Update confMap with any extra os.Args supplied - confErr = confMap.UpdateFromStrings(args[1:]) - if nil != confErr { - log.Fatalf("failed to load config overrides: %v", confErr) - } - - confErr = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != confErr { - log.Fatalf("failed to upgrade config: %v", confErr) - } - - confMapJSONPacked, _ := json.Marshal(confMap) - - fmt.Printf("%v", string(confMapJSONPacked[:])) -} diff --git a/pfsworkout/Makefile b/pfsworkout/Makefile deleted file mode 100644 index eab2cae9..00000000 --- a/pfsworkout/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfsworkout - -include ../GoMakefile diff --git a/pfsworkout/dummy_test.go b/pfsworkout/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/pfsworkout/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfsworkout/main.go b/pfsworkout/main.go deleted file mode 100644 index b575ccc5..00000000 --- a/pfsworkout/main.go +++ /dev/null @@ -1,1240 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "fmt" - "math/rand" - "os" - "strconv" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/headhunter" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" - "github.com/NVIDIA/proxyfs/utils" -) - -const ( - basenamePrefix = "__pfsworkout_" -) - -type rwTimesStruct struct { - writeDuration time.Duration - readDuration time.Duration -} - -type rwSizeEachStruct struct { - name string - KiB uint64 - dirTimes rwTimesStruct - fuseTimes rwTimesStruct - fsTimes rwTimesStruct - inodeTimes rwTimesStruct - swiftclientTimes rwTimesStruct - VolumeHandle fs.VolumeHandle // Only used if all threads use same file - FileInodeNumber inode.InodeNumber // Only used if all threads use same file - ObjectPath string // Only used if all threads use same object -} - -var ( - dirPath string - doNextStepChan chan bool - mountPointName string - mutex trackedlock.Mutex - rwSizeTotal uint64 - stepErrChan chan error - volumeList []string - volumeName string - headhunterVolumeHandle headhunter.VolumeHandle -) - -func usage(file *os.File) { - fmt.Fprintf(file, "Usage:\n") - fmt.Fprintf(file, " %v d threads rw-size-in-mb dir-path\n", os.Args[0]) - fmt.Fprintf(file, " %v [mfisru] threads rw-size-in-mb conf-file [section.option=value]*\n", os.Args[0]) - fmt.Fprintf(file, " where:\n") - fmt.Fprintf(file, " d run tests against specified target directory\n") - fmt.Fprintf(file, " m run tests against FUSE mount point\n") - fmt.Fprintf(file, " f run tests against package fs\n") - fmt.Fprintf(file, " i run tests against package inode\n") - fmt.Fprintf(file, " s run tests against package swiftclient\n") - fmt.Fprintf(file, " r run tests with random I/O instead of sequential\n") - fmt.Fprintf(file, " u run multiple readers/writers on same file against packages\n") - fmt.Fprintf(file, " threads number of threads (currently must be '1')\n") - fmt.Fprintf(file, " rw-size-in-mb number of MiB per thread per test case\n") - fmt.Fprintf(file, " dir-path target directory\n") - fmt.Fprintf(file, " conf-file input to conf.MakeConfMapFromFile()\n") - fmt.Fprintf(file, " [section.option=value]* optional input to conf.UpdateFromStrings()\n") - fmt.Fprintf(file, "\n") - fmt.Fprintf(file, "Note: At least one of f, i, or s must be specified\n") - fmt.Fprintf(file, "\n") - fmt.Fprintf(file, "The default is a sequential test on a different file per thread.\n") - fmt.Fprintf(file, " r specifies that the I/O is random for fs and inodee packages instead of sequential.\n") - fmt.Fprintf(file, " u specifies that all threads operate on the same file.\n") -} - -func main() { - var ( - confMap conf.ConfMap - - proxyfsRequired = false - - doDirWorkout = false - doFuseWorkout = false - doFsWorkout = false - doInodeWorkout = false - doSwiftclientWorkout = false - doSameFile = false - doRandomIO = false - - primaryPeer string - volumeGroupToCheck string - volumeGroupToUse string - volumeGroupList []string - volumeList []string - whoAmI string - - timeBeforeWrites time.Time - timeAfterWrites time.Time - timeBeforeReads time.Time - timeAfterReads time.Time - - bandwidthNumerator float64 - - rwSizeEachArray = [...]*rwSizeEachStruct{ - &rwSizeEachStruct{name: " 4 KiB", KiB: 4}, - &rwSizeEachStruct{name: " 8 KiB", KiB: 8}, - &rwSizeEachStruct{name: "16 KiB", KiB: 16}, - &rwSizeEachStruct{name: "32 KiB", KiB: 32}, - &rwSizeEachStruct{name: "64 KiB", KiB: 64}, - } - ) - - // Parse arguments - - if 5 > len(os.Args) { - usage(os.Stderr) - os.Exit(1) - } - - for _, workoutSelector := range os.Args[1] { - switch workoutSelector { - case 'd': - doDirWorkout = true - case 'm': - proxyfsRequired = true - doFuseWorkout = true - case 'f': - proxyfsRequired = true - doFsWorkout = true - case 'i': - proxyfsRequired = true - doInodeWorkout = true - case 's': - proxyfsRequired = true - doSwiftclientWorkout = true - case 'r': - proxyfsRequired = true - doRandomIO = true - case 'u': - proxyfsRequired = true - doSameFile = true - default: - fmt.Fprintf(os.Stderr, "workoutSelector ('%v') must be one of 'd', 'm', 'f', 'i', or 's'\n", string(workoutSelector)) - os.Exit(1) - } - } - - if doDirWorkout { - if doFuseWorkout || doFsWorkout || doInodeWorkout || doSwiftclientWorkout { - fmt.Fprintf(os.Stderr, "workoutSelectors cannot include both 'd' and any of 'm', 'f', 'i', or 's'\n") - os.Exit(1) - } - } else { - if !(doFuseWorkout || doFsWorkout || doInodeWorkout || doSwiftclientWorkout) { - fmt.Fprintf(os.Stderr, "workoutSelectors must include at least one of 'm', 'f', 'i', or 's' when 'd' is not selected") - os.Exit(1) - } - } - - threads, err := strconv.ParseUint(os.Args[2], 10, 64) - if nil != err { - fmt.Fprintf(os.Stderr, "strconv.ParseUint(\"%v\", 10, 64) failed: %v\n", os.Args[2], err) - os.Exit(1) - } - if 0 == threads { - fmt.Fprintf(os.Stderr, "threads must be a positive number\n") - os.Exit(1) - } - - rwSizeTotalMiB, err := strconv.ParseUint(os.Args[3], 10, 64) - if nil != err { - fmt.Fprintf(os.Stderr, "strconv.ParseUint(\"%v\", 10, 64) failed: %v\n", os.Args[3], err) - os.Exit(1) - } - - rwSizeTotal = rwSizeTotalMiB * 1024 * 1024 - - if doDirWorkout { - dirPath = os.Args[4] - } else { - confMap, err = conf.MakeConfMapFromFile(os.Args[4]) - if nil != err { - fmt.Fprintf(os.Stderr, "conf.MakeConfMapFromFile(\"%v\") failed: %v\n", os.Args[4], err) - os.Exit(1) - } - - if 5 < len(os.Args) { - err = confMap.UpdateFromStrings(os.Args[5:]) - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.UpdateFromStrings(%#v) failed: %v\n", os.Args[5:], err) - os.Exit(1) - } - } - - // Upgrade confMap if necessary - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "Failed to upgrade config: %v", err) - os.Exit(1) - } - - // Select first Volume of the first "active" VolumeGroup in [FSGlobals]VolumeGroupList - - whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueString(\"Cluster\", \"WhoAmI\") failed: %v\n", err) - os.Exit(1) - } - - volumeGroupList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"FSGlobals\", \"VolumeGroupList\") failed: %v\n", err) - os.Exit(1) - } - - volumeGroupToUse = "" - - for _, volumeGroupToCheck = range volumeGroupList { - primaryPeer, err = confMap.FetchOptionValueString("VolumeGroup:"+volumeGroupToCheck, "PrimaryPeer") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueString(\"VolumeGroup:%s\", \"PrimaryPeer\") failed: %v\n", volumeGroupToCheck, err) - os.Exit(1) - } - if whoAmI == primaryPeer { - volumeGroupToUse = volumeGroupToCheck - break - } - } - - if "" == volumeGroupToUse { - fmt.Fprintf(os.Stderr, "confMap didn't contain an \"active\" VolumeGroup") - os.Exit(1) - } - - volumeList, err = confMap.FetchOptionValueStringSlice("VolumeGroup:"+volumeGroupToUse, "VolumeList") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"VolumeGroup:%s\", \"PrimaryPeer\") failed: %v\n", volumeGroupToUse, err) - os.Exit(1) - } - if 1 > len(volumeList) { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueStringSlice(\"VolumeGroup:%s\", \"VolumeList\") returned empty volumeList", volumeGroupToUse) - os.Exit(1) - } - - volumeName = volumeList[0] - - mountPointName, err = confMap.FetchOptionValueString("Volume:"+volumeName, "FUSEMountPointName") - if nil != err { - fmt.Fprintf(os.Stderr, "confMap.FetchOptionValueString(\"Volume:%s\", \"FUSEMountPointName\") failed: %v\n", volumeName, err) - os.Exit(1) - } - } - - if proxyfsRequired { - // Start up needed ProxyFS components - - err = transitions.Up(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "transitions.Up() failed: %v\n", err) - os.Exit(1) - } - - headhunterVolumeHandle, err = headhunter.FetchVolumeHandle(volumeName) - if nil != err { - fmt.Fprintf(os.Stderr, "headhunter.FetchVolumeHandle(\"%s\") failed: %v\n", volumeName, err) - os.Exit(1) - } - } - - // Perform tests - - stepErrChan = make(chan error, threads) - doNextStepChan = make(chan bool, threads) - - if doDirWorkout { - for _, rwSizeEach := range rwSizeEachArray { - // Do initialization step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - go dirWorkout(rwSizeEach, threadIndex) - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "dirWorkout() initialization step returned: %v\n", err) - os.Exit(1) - } - } - // Do writes step - timeBeforeWrites = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "dirWorkout() write step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterWrites = time.Now() - // Do reads step - timeBeforeReads = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "dirWorkout() read step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterReads = time.Now() - // Do shutdown step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "dirWorkout() shutdown step returned: %v\n", err) - os.Exit(1) - } - } - - rwSizeEach.dirTimes.writeDuration = timeAfterWrites.Sub(timeBeforeWrites) - rwSizeEach.dirTimes.readDuration = timeAfterReads.Sub(timeBeforeReads) - } - } - - if doFuseWorkout { - for _, rwSizeEach := range rwSizeEachArray { - // Do initialization step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - go fuseWorkout(rwSizeEach, threadIndex) - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fuseWorkout() initialization step returned: %v\n", err) - os.Exit(1) - } - } - // Do writes step - timeBeforeWrites = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fuseWorkout() write step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterWrites = time.Now() - // Do reads step - timeBeforeReads = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fuseWorkout() read step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterReads = time.Now() - // Do shutdown step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fuseWorkout() shutdown step returned: %v\n", err) - os.Exit(1) - } - } - - rwSizeEach.fuseTimes.writeDuration = timeAfterWrites.Sub(timeBeforeWrites) - rwSizeEach.fuseTimes.readDuration = timeAfterReads.Sub(timeBeforeReads) - } - } - - if doFsWorkout { - for _, rwSizeEach := range rwSizeEachArray { - var fileName string - - // If we are doing the operations on the same file for all threads, create the file now. - if doSameFile { - // Save off MountHandle and FileInodeNumber in rwSizeEach since all threads need this - err, rwSizeEach.VolumeHandle, rwSizeEach.FileInodeNumber, fileName = createFsFile() - if nil != err { - // In an error, no point in continuing. Just break from this for loop. - break - } - } - - // Do initialization step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - go fsWorkout(rwSizeEach, threadIndex, doSameFile, doRandomIO) - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fsWorkout() initialization step returned: %v\n", err) - os.Exit(1) - } - } - // Do writes step - timeBeforeWrites = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fsWorkout() write step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterWrites = time.Now() - // Do reads step - timeBeforeReads = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fsWorkout() read step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterReads = time.Now() - // Do shutdown step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "fsWorkout() shutdown step returned: %v\n", err) - os.Exit(1) - } - } - - // Remove file if all threads used same file - if doSameFile { - _ = unlinkFsFile(rwSizeEach.VolumeHandle, fileName) - } - - rwSizeEach.fsTimes.writeDuration = timeAfterWrites.Sub(timeBeforeWrites) - rwSizeEach.fsTimes.readDuration = timeAfterReads.Sub(timeBeforeReads) - } - } - - if doInodeWorkout { - for _, rwSizeEach := range rwSizeEachArray { - // If we are doing the operations on the same object for all threads, create the object now. - if doSameFile { - err, rwSizeEach.FileInodeNumber = createInode() - if nil != err { - // In an error, no point in continuing. Just break from this for loop. - break - } - } - - // Do initialization step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - go inodeWorkout(rwSizeEach, threadIndex, doSameFile, doRandomIO) - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "inodeWorkout() initialization step returned: %v\n", err) - os.Exit(1) - } - } - // Do writes step - timeBeforeWrites = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "inodeWorkout() write step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterWrites = time.Now() - // Do reads step - timeBeforeReads = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "inodeWorkout() read step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterReads = time.Now() - // Do shutdown step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "inodeWorkout() shutdown step returned: %v\n", err) - os.Exit(1) - } - } - - // Remove inode if all threads use same inode - if doSameFile { - _ = destroyInode(rwSizeEach.FileInodeNumber) - } - - rwSizeEach.inodeTimes.writeDuration = timeAfterWrites.Sub(timeBeforeWrites) - rwSizeEach.inodeTimes.readDuration = timeAfterReads.Sub(timeBeforeReads) - } - } - - if doSwiftclientWorkout { - for _, rwSizeEach := range rwSizeEachArray { - - // Create object used by all threads - if doSameFile { - err, rwSizeEach.ObjectPath = createObject() - if nil != err { - // In an error, no point in continuing. Just break from this for loop. - break - } - } - - // Do initialization step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - go swiftclientWorkout(rwSizeEach, threadIndex, doSameFile) - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "swiftclientWorkout() initialization step returned: %v\n", err) - os.Exit(1) - } - } - // Do writes step - timeBeforeWrites = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "swiftclientWorkout() write step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterWrites = time.Now() - // Do reads step - timeBeforeReads = time.Now() - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "swiftclientWorkout() read step returned: %v\n", err) - os.Exit(1) - } - } - timeAfterReads = time.Now() - // Do shutdown step - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - doNextStepChan <- true - } - for threadIndex := uint64(0); threadIndex < threads; threadIndex++ { - err = <-stepErrChan - if nil != err { - fmt.Fprintf(os.Stderr, "swiftclientWorkout() shutdown step returned: %v\n", err) - os.Exit(1) - } - } - - // Remove object if all threads use same object - if doSameFile { - _ = deleteObject(rwSizeEach.ObjectPath) - } - - rwSizeEach.swiftclientTimes.writeDuration = timeAfterWrites.Sub(timeBeforeWrites) - rwSizeEach.swiftclientTimes.readDuration = timeAfterReads.Sub(timeBeforeReads) - } - } - - if proxyfsRequired { - // Stop ProxyFS components launched above - - err = transitions.Down(confMap) - if nil != err { - fmt.Fprintf(os.Stderr, "transitions.Down() failed: %v\n", err) - os.Exit(1) - } - } - - // Report results - - bandwidthNumerator = float64(threads*rwSizeTotal) / float64(1024*1024) - - var fileAccess = "Sequential" - var threadFile = "different files" - if doSameFile { - threadFile = "same file" - } - if doRandomIO { - fileAccess = "Random" - } - - fmt.Printf(" I/O type: %v and %v per thread\n", fileAccess, threadFile) - fmt.Printf(" (in MiB/sec) ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %s", rwSizeEach.name) - } - fmt.Println() - - if doDirWorkout { - fmt.Printf("dir read ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.dirTimes.readDuration.Seconds()) - } - fmt.Println() - fmt.Printf(" write ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.dirTimes.writeDuration.Seconds()) - } - fmt.Println() - } - - if doFuseWorkout { - fmt.Printf("fuse read ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.fuseTimes.readDuration.Seconds()) - } - fmt.Println() - fmt.Printf(" write ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.fuseTimes.writeDuration.Seconds()) - } - fmt.Println() - } - - if doFsWorkout { - fmt.Printf("fs read ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.fsTimes.readDuration.Seconds()) - } - fmt.Println() - fmt.Printf(" write ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.fsTimes.writeDuration.Seconds()) - } - fmt.Println() - } - - if doInodeWorkout { - fmt.Printf("inode read ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.inodeTimes.readDuration.Seconds()) - } - fmt.Println() - fmt.Printf(" write ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.inodeTimes.writeDuration.Seconds()) - } - fmt.Println() - } - - if doSwiftclientWorkout { - fmt.Printf("swiftclient read ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.swiftclientTimes.readDuration.Seconds()) - } - fmt.Println() - fmt.Printf(" write ") - for _, rwSizeEach := range rwSizeEachArray { - fmt.Printf(" %8.2f", bandwidthNumerator/rwSizeEach.swiftclientTimes.writeDuration.Seconds()) - } - fmt.Println() - } -} - -func dirWorkout(rwSizeEach *rwSizeEachStruct, threadIndex uint64) { - fileName := fmt.Sprintf("%s/%s%016X", dirPath, basenamePrefix, threadIndex) - - file, err := os.Create(fileName) - if nil != err { - stepErrChan <- fmt.Errorf("os.Create(\"%v\") failed: %v\n", fileName, err) - return - } - - rwSizeRequested := rwSizeEach.KiB * 1024 - - bufWritten := make([]byte, rwSizeRequested) - for i := uint64(0); i < rwSizeRequested; i++ { - bufWritten[i] = 0 - } - - bufRead := make([]byte, rwSizeRequested) - - stepErrChan <- nil - _ = <-doNextStepChan - - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - _, err = file.WriteAt(bufWritten, int64(rwOffset)) - if nil != err { - stepErrChan <- fmt.Errorf("file.WriteAt(bufWritten, int64(rwOffset)) failed: %v\n", err) - return - } - } - - err = file.Sync() - if nil != err { - stepErrChan <- fmt.Errorf("file.Sync() failed: %v\n", err) - return - } - - stepErrChan <- nil - _ = <-doNextStepChan - - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - _, err = file.ReadAt(bufRead, int64(rwOffset)) - if nil != err { - stepErrChan <- fmt.Errorf("file.ReadAt(bufRead, int64(rwOffset)) failed: %v\n", err) - return - } - } - - stepErrChan <- nil - _ = <-doNextStepChan - - err = file.Close() - if nil != err { - stepErrChan <- fmt.Errorf("file.Close() failed: %v\n", err) - return - } - err = os.Remove(fileName) - if nil != err { - stepErrChan <- fmt.Errorf("os.Remove(fileName) failed: %v\n", err) - return - } - - stepErrChan <- nil -} - -func fuseWorkout(rwSizeEach *rwSizeEachStruct, threadIndex uint64) { - nonce := headhunterVolumeHandle.FetchNonce() - - fileName := fmt.Sprintf("%s/%s%016X", mountPointName, basenamePrefix, nonce) - - file, err := os.Create(fileName) - if nil != err { - stepErrChan <- fmt.Errorf("os.Create(\"%v\") failed: %v\n", fileName, err) - return - } - - rwSizeRequested := rwSizeEach.KiB * 1024 - - bufWritten := make([]byte, rwSizeRequested) - for i := uint64(0); i < rwSizeRequested; i++ { - bufWritten[i] = 0 - } - - bufRead := make([]byte, rwSizeRequested) - - stepErrChan <- nil - _ = <-doNextStepChan - - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - _, err = file.WriteAt(bufWritten, int64(rwOffset)) - if nil != err { - stepErrChan <- fmt.Errorf("file.WriteAt(bufWritten, int64(rwOffset)) failed: %v\n", err) - return - } - } - - err = file.Sync() - if nil != err { - stepErrChan <- fmt.Errorf("file.Sync() failed: %v\n", err) - return - } - - stepErrChan <- nil - _ = <-doNextStepChan - - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - _, err = file.ReadAt(bufRead, int64(rwOffset)) - if nil != err { - stepErrChan <- fmt.Errorf("file.ReadAt(bufRead, int64(rwOffset)) failed: %v\n", err) - return - } - } - - stepErrChan <- nil - _ = <-doNextStepChan - - err = file.Close() - if nil != err { - stepErrChan <- fmt.Errorf("file.Close() failed: %v\n", err) - return - } - err = os.Remove(fileName) - if nil != err { - stepErrChan <- fmt.Errorf("os.Remove(fileName) failed: %v\n", err) - return - } - - stepErrChan <- nil -} - -func createFsFile() (err error, volumeHandle fs.VolumeHandle, fileInodeNumber inode.InodeNumber, fileName string) { - volumeHandle, err = fs.FetchVolumeHandleByVolumeName(volumeName) - if nil != err { - stepErrChan <- fmt.Errorf("fs.FetchVolumeHandleByVolumeName(\"%v\") failed: %v\n", volumeName, err) - return - } - - nonce := headhunterVolumeHandle.FetchNonce() - - fileName = fmt.Sprintf("%s%016X", basenamePrefix, nonce) - - fileInodeNumber, err = volumeHandle.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, fileName, inode.PosixModePerm) - if nil != err { - stepErrChan <- fmt.Errorf("fs.Create(,,,, fileName==\"%s\", inode.PosixModePerm) failed: %v\n", fileName, err) - return - } - return -} - -func unlinkFsFile(volumeHandle fs.VolumeHandle, fileName string) (err error) { - err = volumeHandle.Unlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, inode.RootDirInodeNumber, fileName) - if nil != err { - stepErrChan <- fmt.Errorf("fs.Unlink(,,,, rootInodeNumber, \"%v\") failed: %v\n", fileName, err) - return - } - return -} - -func fsWorkout(rwSizeEach *rwSizeEachStruct, threadIndex uint64, doSameFile bool, doRandomIO bool) { - var ( - err error - fileInodeNumber inode.InodeNumber - fileName string - volumeHandle fs.VolumeHandle - ) - - if !doSameFile { - // Create the file for this thread - err, volumeHandle, fileInodeNumber, fileName = createFsFile() - if nil != err { - return - } - } else { - // File was already created during main() - volumeHandle = rwSizeEach.VolumeHandle - fileInodeNumber = rwSizeEach.FileInodeNumber - } - rwSizeRequested := rwSizeEach.KiB * 1024 - - bufWritten := make([]byte, rwSizeRequested) - for i := uint64(0); i < rwSizeRequested; i++ { - bufWritten[i] = 0 - } - - stepErrChan <- nil - _ = <-doNextStepChan - - if doRandomIO { - var rwOffset int64 - - // Calculate number of I/Os to do since we cannot use size of file in the random case. - var numberIOsNeeded uint64 = rwSizeTotal / rwSizeRequested - for i := uint64(0); i < numberIOsNeeded; i++ { - // For the first I/O, we set it to (rwSizeTotal - rwSizeRequested). This guarantees that we write - // the full size of the buffer. - if i == 0 { - rwOffset = int64(rwSizeTotal - rwSizeRequested) - } else { - - // Pick a random offset within the buffer. We back off from end of buffer by rwSizeRequested - // to make sure we do not go past end of file. - rwOffset = rand.Int63n(int64(rwSizeTotal - rwSizeRequested)) - } - rwSizeDelivered, err := volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInodeNumber, uint64(rwOffset), bufWritten, nil) - if nil != err { - stepErrChan <- fmt.Errorf("fs.Write(,,,, fileInodeNumber, rwOffset, bufWritten) failed: %v\n", err) - return - } - if rwSizeRequested != rwSizeDelivered { - stepErrChan <- fmt.Errorf("fs.Write(,,,, fileInodeNumber, rwOffset, bufWritten) failed to transfer all requested bytes\n") - return - } - } - } else { - - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - rwSizeDelivered, err := volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInodeNumber, rwOffset, bufWritten, nil) - if nil != err { - stepErrChan <- fmt.Errorf("fs.Write(,,,, fileInodeNumber, rwOffset, bufWritten) failed: %v\n", err) - return - } - if rwSizeRequested != rwSizeDelivered { - stepErrChan <- fmt.Errorf("fs.Write(,,,, fileInodeNumber, rwOffset, bufWritten) failed to transfer all requested bytes\n") - return - } - } - } - - err = volumeHandle.Flush(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInodeNumber) - if nil != err { - stepErrChan <- fmt.Errorf("fs.Flush(,,,, fileInodeNumber) failed: %v\n", err) - return - } - - stepErrChan <- nil - _ = <-doNextStepChan - - if doRandomIO { - // Calculate number of I/Os to do since we cannot use size of file in the random case. - var numberIOsNeeded uint64 = rwSizeTotal / rwSizeRequested - for i := uint64(0); i < numberIOsNeeded; i++ { - - // Calculate random offset - rwOffset := uint64(rand.Int63n(int64(rwSizeTotal - rwSizeRequested))) - - bufRead, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInodeNumber, rwOffset, rwSizeRequested, nil) - if nil != err { - stepErrChan <- fmt.Errorf("fs.Read(,,,, fileInodeNumber, rwOffset, rwSizeRequested) failed: %v\n", err) - return - } - if rwSizeRequested != uint64(len(bufRead)) { - stepErrChan <- fmt.Errorf("fs.Read(,,,, fileInodeNumber, rwOffset, rwSizeRequested) failed to transfer all requested bytes\n") - return - } - } - } else { - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - bufRead, err := volumeHandle.Read(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInodeNumber, rwOffset, rwSizeRequested, nil) - if nil != err { - stepErrChan <- fmt.Errorf("fs.Read(,,,, fileInodeNumber, rwOffset, rwSizeRequested) failed: %v\n", err) - return - } - if rwSizeRequested != uint64(len(bufRead)) { - stepErrChan <- fmt.Errorf("fs.Read(,,,, fileInodeNumber, rwOffset, rwSizeRequested) failed to transfer all requested bytes\n") - return - } - } - } - - stepErrChan <- nil - _ = <-doNextStepChan - - if !doSameFile { - err = unlinkFsFile(volumeHandle, fileName) - if nil != err { - return - } - } - - stepErrChan <- nil -} - -func createInode() (err error, fileInodeNumber inode.InodeNumber) { - mutex.Lock() - volumeHandle, err := inode.FetchVolumeHandle(volumeName) - mutex.Unlock() - if nil != err { - stepErrChan <- fmt.Errorf("inode.FetchVolumeHandle(\"%v\") failed: %v\n", volumeName, err) - return - } - - fileInodeNumber, err = volumeHandle.CreateFile(inode.PosixModePerm, inode.InodeUserID(0), inode.InodeGroupID(0)) - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.CreateFile(inode.PosixModePerm, inode.InodeUserID(0), inode.InodeGroupID(0)) failed: %v\n", err) - return - } - return -} - -func destroyInode(fileInodeNumber inode.InodeNumber) (err error) { - mutex.Lock() - volumeHandle, err := inode.FetchVolumeHandle(volumeName) - mutex.Unlock() - if nil != err { - stepErrChan <- fmt.Errorf("inode.FetchVolumeHandle(\"%v\") failed: %v\n", volumeName, err) - return - } - err = volumeHandle.Purge(fileInodeNumber) - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.Purge(fileInodeNumber) failed: %v\n", err) - return - } - err = volumeHandle.Destroy(fileInodeNumber) - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.Destroy(fileInodeNumber) failed: %v\n", err) - return - } - return -} - -func inodeWorkout(rwSizeEach *rwSizeEachStruct, threadIndex uint64, doSameFile bool, doRandomIO bool) { - mutex.Lock() - volumeHandle, err := inode.FetchVolumeHandle(volumeName) - mutex.Unlock() - - if nil != err { - stepErrChan <- fmt.Errorf("inode.FetchVolumeHandle(\"%v\") failed: %v\n", volumeName, err) - return - } - - var fileInodeNumber inode.InodeNumber - if !doSameFile { - err, fileInodeNumber = createInode() - if nil != err { - return - } - } else { - fileInodeNumber = rwSizeEach.FileInodeNumber - } - - rwSizeRequested := rwSizeEach.KiB * 1024 - - bufWritten := make([]byte, rwSizeRequested) - for i := uint64(0); i < rwSizeRequested; i++ { - bufWritten[i] = 0 - } - - stepErrChan <- nil - _ = <-doNextStepChan - - if doRandomIO { - var rwOffset int64 - - // Calculate number of I/Os to do since we cannot use size of file in the random case. - var numberIOsNeeded uint64 = rwSizeTotal / rwSizeRequested - for i := uint64(0); i < numberIOsNeeded; i++ { - - // For the first I/O, we set it to (rwSizeTotal - rwSizeRequested). This guarantees that we write - // the full size of the buffer. - if i == 0 { - rwOffset = int64(rwSizeTotal - rwSizeRequested) - } else { - - // Pick a random offset within the buffer. We back off from end of buffer by rwSizeRequested - // to make sure we do not go past end of file. - rwOffset = rand.Int63n(int64(rwSizeTotal - rwSizeRequested)) - } - err = volumeHandle.Write(fileInodeNumber, uint64(rwOffset), bufWritten, nil) - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.Write(fileInodeNumber, rwOffset, bufWritten) failed: %v\n", err) - return - } - } - } else { - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - err = volumeHandle.Write(fileInodeNumber, rwOffset, bufWritten, nil) - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.Write(fileInodeNumber, rwOffset, bufWritten) failed: %v\n", err) - return - } - } - } - - err = volumeHandle.Flush(fileInodeNumber, false) - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.Flush(rwSizeEach.FileInodeNumber, false) failed: %v\n", err) - return - } - - stepErrChan <- nil - _ = <-doNextStepChan - - if doRandomIO { - // Calculate number of I/Os to do since we cannot use size of file in the random case. - var numberIOsNeeded uint64 = rwSizeTotal / rwSizeRequested - for i := uint64(0); i < numberIOsNeeded; i++ { - - // Calculate random offset - rwOffset := uint64(rand.Int63n(int64(rwSizeTotal - rwSizeRequested))) - bufRead, err := volumeHandle.Read(fileInodeNumber, rwOffset, rwSizeRequested, nil) - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.Read(rwSizeEach.FileInodeNumber, rwOffset, rwSizeRequested) failed: %v\n", err) - return - } - if rwSizeRequested != uint64(len(bufRead)) { - stepErrChan <- fmt.Errorf("volumeHandle.Read(rwSizeEach.FileInodeNumber, rwOffset, rwSizeRequested) failed to transfer all requested bytes\n") - return - } - } - } else { - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - bufRead, err := volumeHandle.Read(fileInodeNumber, rwOffset, rwSizeRequested, nil) - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.Read(rwSizeEach.FileInodeNumber, rwOffset, rwSizeRequested) failed: %v\n", err) - return - } - if rwSizeRequested != uint64(len(bufRead)) { - stepErrChan <- fmt.Errorf("volumeHandle.Read(rwSizeEach.FileInodeNumber, rwOffset, rwSizeRequested) failed to transfer all requested bytes\n") - return - } - } - } - - stepErrChan <- nil - _ = <-doNextStepChan - - if !doSameFile { - err = destroyInode(fileInodeNumber) - if nil != err { - return - } - } - - stepErrChan <- nil -} - -func createObject() (err error, objectPath string) { - mutex.Lock() - volumeHandle, err := inode.FetchVolumeHandle(volumeName) - mutex.Unlock() - if nil != err { - stepErrChan <- fmt.Errorf("inode.FetchVolumeHandle(\"%v\") failed: %v\n", volumeName, err) - return - } - - objectPath, err = volumeHandle.ProvisionObject() - if nil != err { - stepErrChan <- fmt.Errorf("volumeHandle.ProvisionObject() failed: %v\n", err) - return - } - return -} - -func deleteObject(objectPath string) (err error) { - accountName, containerName, objectName, err := utils.PathToAcctContObj(objectPath) - if nil != err { - stepErrChan <- fmt.Errorf("utils.PathToAcctContObj(\"%v\") failed: %v\n", objectPath, err) - return - } - - err = swiftclient.ObjectDelete(accountName, containerName, objectName, swiftclient.SkipRetry) - if nil != err { - stepErrChan <- fmt.Errorf("swiftclient.ObjectDelete(\"%v\", \"%v\", \"%v\", swiftclient.SkipRetry) failed: %v\n", accountName, containerName, objectName, err) - return - } - return -} - -func swiftclientWorkout(rwSizeEach *rwSizeEachStruct, threadIndex uint64, doSameFile bool) { - var err error - var objectPath string - if !doSameFile { - err, objectPath = createObject() - if nil != err { - return - } - } else { - objectPath = rwSizeEach.ObjectPath - } - - accountName, containerName, objectName, err := utils.PathToAcctContObj(objectPath) - if nil != err { - stepErrChan <- fmt.Errorf("utils.PathToAcctContObj(\"%v\") failed: %v\n", objectPath, err) - return - } - - rwSizeRequested := rwSizeEach.KiB * 1024 - - bufWritten := make([]byte, rwSizeRequested) - for i := uint64(0); i < rwSizeRequested; i++ { - bufWritten[i] = 0 - } - - stepErrChan <- nil - _ = <-doNextStepChan - - chunkedPutContext, err := swiftclient.ObjectFetchChunkedPutContext(accountName, containerName, objectName, "") - if nil != err { - stepErrChan <- fmt.Errorf("swiftclient.ObjectFetchChunkedPutContext(\"%v\", \"%v\", \"%v\") failed: %v\n", accountName, containerName, objectName, err) - return - } - - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - err = chunkedPutContext.SendChunk(bufWritten) - if nil != err { - stepErrChan <- fmt.Errorf("chunkedPutContext.SendChunk(bufWritten) failed: %v\n", err) - return - } - } - - err = chunkedPutContext.Close() - if nil != err { - stepErrChan <- fmt.Errorf("chunkedPutContext.Close() failed: %v\n", err) - return - } - - stepErrChan <- nil - _ = <-doNextStepChan - - for rwOffset := uint64(0); rwOffset < rwSizeTotal; rwOffset += rwSizeRequested { - bufRead, err := swiftclient.ObjectGet(accountName, containerName, objectName, rwOffset, rwSizeRequested) - if nil != err { - stepErrChan <- fmt.Errorf("swiftclient.ObjectGet(\"%v\", \"%v\", \"%v\", rwOffset, rwSizeRequested) failed: %v\n", accountName, containerName, objectName, err) - return - } - if rwSizeRequested != uint64(len(bufRead)) { - stepErrChan <- fmt.Errorf("swiftclient.ObjectGet(\"%v\", \"%v\", \"%v\", rwOffset, rwSizeRequested) failed to transfer all requested bytes\n", accountName, containerName, objectName) - return - } - } - - stepErrChan <- nil - _ = <-doNextStepChan - - if !doSameFile { - err = deleteObject(objectPath) - } - - stepErrChan <- nil -} diff --git a/proxyfsd/Makefile b/proxyfsd/Makefile deleted file mode 100644 index 850db1fa..00000000 --- a/proxyfsd/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/proxyfsd - -include ../GoMakefile diff --git a/proxyfsd/cluster_1_peer.conf b/proxyfsd/cluster_1_peer.conf deleted file mode 100644 index 408e8480..00000000 --- a/proxyfsd/cluster_1_peer.conf +++ /dev/null @@ -1,22 +0,0 @@ -# Cluster .conf file for 1 Peer -# -# Following .include, caller should define Cluster.WhoAmI as Peer0 - -[Peer:Peer0] -PublicIPAddr: 192.168.22.40 -PrivateIPAddr: 192.168.23.40 -ReadCacheQuotaFraction: 0.20 - -[Cluster] -Peers: Peer0 -ServerGuid: 0bb51164-258f-4e04-a417-e16d736ca41c -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 diff --git a/proxyfsd/cluster_3_peers.conf b/proxyfsd/cluster_3_peers.conf deleted file mode 100644 index fbe9faf6..00000000 --- a/proxyfsd/cluster_3_peers.conf +++ /dev/null @@ -1,32 +0,0 @@ -# Cluster .conf file for 3 Peers -# -# Following .include, caller should define Cluster.WhoAmI as Peer1, Peer2, or Peer3 - -[Peer:Peer1] -PublicIPAddr: 192.168.22.41 -PrivateIPAddr: 192.168.23.41 -ReadCacheQuotaFraction: 0.20 - -[Peer:Peer2] -PublicIPAddr: 192.168.22.42 -PrivateIPAddr: 192.168.23.42 -ReadCacheQuotaFraction: 0.20 - -[Peer:Peer3] -PublicIPAddr: 192.168.22.43 -PrivateIPAddr: 192.168.23.43 -ReadCacheQuotaFraction: 0.20 - -[Cluster] -Peers: Peer1 Peer2 Peer3 -ServerGuid: 0bb51164-258f-4e04-a417-e16d736ca41c -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 diff --git a/proxyfsd/daemon.go b/proxyfsd/daemon.go deleted file mode 100644 index 0b9002b9..00000000 --- a/proxyfsd/daemon.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package proxyfsd - -import ( - "fmt" - "net/http" - _ "net/http/pprof" - "os" - "os/signal" - "strings" - "sync" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/transitions" - "github.com/NVIDIA/proxyfs/version" - - // Force importing of the following "top-most" package - _ "github.com/NVIDIA/proxyfs/httpserver" -) - -// Daemon is launched as a GoRoutine that launches ProxyFS. During startup, the parent should read errChan -// to await Daemon getting to the point where it is ready to handle the specified signal set. Any errors -// encountered before or after this point will be sent to errChan (and be non-nil of course). -func Daemon(confFile string, confStrings []string, errChan chan error, wg *sync.WaitGroup, execArgs []string, signals ...os.Signal) { - var ( - confMap conf.ConfMap - err error - signalReceived os.Signal - ) - - // Compute confMap - - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - errChan <- err - - return - } - - err = confMap.UpdateFromStrings(confStrings) - if nil != err { - errChan <- err - - return - } - - // Optionally launch an embedded HTTP Server for Golang runtime access; - // this should be done before transitions.Up() is called so it is - // available if transitions.Up() hangs (the embedded http server will - // return http.StatusServiceUnavailable (503) during the transition. - debugServerPortAsUint16, err := confMap.FetchOptionValueUint16("ProxyfsDebug", "DebugServerPort") - if nil != err && debugServerPortAsUint16 != 0 { - - debugServerPortAsString := fmt.Sprintf("%d", debugServerPortAsUint16) - logger.Infof("proxyfsd.Daemon() starting debug HTTP Server on localhost:%s", debugServerPortAsString) - go http.ListenAndServe("localhost:"+debugServerPortAsString, nil) - } - - // Arm signal handler used to catch signals - // - // Note: signalChan must be buffered to avoid race with window between - // arming handler and blocking on the chan read when signals might - // otherwise be lost. No signals will be processed until - // transitions.Up() finishes, but an incoming SIGHUP will not cause the - // process to immediately exit. - signalChan := make(chan os.Signal, 16) - - // if signals is empty it means "catch all signals" it is possible to catch - signal.Notify(signalChan, signals...) - - // Start up dæmon packages - - err = transitions.Up(confMap) - if nil != err { - errChan <- err - return - } - wg.Add(1) - logger.Infof("proxyfsd is starting up (version %s) (PID %d); invoked as '%s'", - version.ProxyFSVersion, os.Getpid(), strings.Join(execArgs, "' '")) - defer func() { - logger.Infof("proxyfsd logger is shutting down (PID %d)", os.Getpid()) - err = transitions.Down(confMap) - if nil != err { - logger.Errorf("transitions.Down() failed: %v", err) // Oddly, if logger.Down() fails, will this work? - } - errChan <- err - wg.Done() - }() - - // indicate transitions finished and signal handlers have been armed successfully - errChan <- nil - - // Await a signal - reloading confFile each SIGHUP - exiting otherwise - for { - signalReceived = <-signalChan - logger.Infof("Received signal: '%v'", signalReceived) - - // these signals are normally ignored, but if "signals..." above is empty - // they are delivered via the channel. we should simply ignore them. - if signalReceived == unix.SIGCHLD || signalReceived == unix.SIGURG || - signalReceived == unix.SIGWINCH || signalReceived == unix.SIGCONT { - logger.Infof("Ignored signal: '%v'", signalReceived) - continue - } - - // we can get SIGPIPE whenever an HTTP or other client closes a - // socket on us, so ignore it - if signalReceived == unix.SIGPIPE { - logger.Infof("Ignored signal: '%v'", signalReceived) - continue - } - - // SIGHUP means reconfig but any other signal means time to exit - if unix.SIGHUP != signalReceived { - logger.Infof("signal catcher is shutting down proxyfsd (PID %d)", os.Getpid()) - - if signalReceived != unix.SIGTERM && signalReceived != unix.SIGINT { - logger.Errorf("proxyfsd received unexpected signal: %v", signalReceived) - } - - return - } - - // caught SIGHUP -- recompute confMap and re-apply - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - err = fmt.Errorf("failed to load updated config: %v", err) - errChan <- err - return - } - - err = confMap.UpdateFromStrings(confStrings) - if nil != err { - err = fmt.Errorf("failed to reapply config overrides: %v", err) - errChan <- err - return - } - - err = transitions.Signaled(confMap) - if nil != err { - err = fmt.Errorf("transitions.Signaled() failed: %v", err) - errChan <- err - return - } - } -} diff --git a/proxyfsd/daemon_test.go b/proxyfsd/daemon_test.go deleted file mode 100644 index 84980b22..00000000 --- a/proxyfsd/daemon_test.go +++ /dev/null @@ -1,468 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package proxyfsd - -import ( - "bytes" - "encoding/json" - "io/ioutil" - "net/http" - "os" - "strconv" - "sync" - "testing" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/fs" - "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/ramswift" -) - -func TestMain(m *testing.M) { - mRunReturn := m.Run() - os.Exit(mRunReturn) -} - -func TestDaemon(t *testing.T) { - var ( - bytesWritten uint64 - confMapStrings []string - confMapStringsWithAutoFormatTrueAdded []string - createdFileInodeNumber inode.InodeNumber - err error - errChan chan error - execArgs []string - ramswiftDoneChan chan bool - ramswiftSignalHandlerIsArmedWG sync.WaitGroup - readData []byte - testVersion uint64 - testVersionConfFile *os.File - testVersionConfFileName string - toReadFileInodeNumber inode.InodeNumber - volumeHandle fs.VolumeHandle - wg sync.WaitGroup - ) - - // Setup a ramswift instance leveraging test config - - confMapStrings = []string{ - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - - "StatsLogger.Period=0m", - "StatsLogger.Verbose=false", - - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - - "Peer:Peer0.PublicIPAddr=127.0.0.1", - "Peer:Peer0.PrivateIPAddr=127.0.0.1", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - - "Cluster.WhoAmI=Peer0", - "Cluster.Peers=Peer0", - "Cluster.ServerGuid=a66488e9-a051-4ff7-865d-87bfb84cc2ae", - "Cluster.PrivateClusterUDPPort=18123", - "Cluster.UDPPacketSendSize=1400", - "Cluster.UDPPacketRecvSize=1500", - "Cluster.UDPPacketCapPerMessage=5", - "Cluster.HeartBeatDuration=1s", - "Cluster.HeartBeatMissLimit=3", - "Cluster.MessageQueueDepthPerPeer=4", - "Cluster.MaxRequestDuration=1s", - "Cluster.LivenessCheckerEnabled=true", - "Cluster.LivenessCheckRedundancy=2", - - "ProxyfsDebug.DebugServerPort=6058", - - "HTTPServer.TCPPort=53461", - - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=35262", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=1", - "SwiftClient.RetryLimitObject=1", - "SwiftClient.RetryDelay=10ms", - "SwiftClient.RetryDelayObject=10ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=64", - "SwiftClient.NonChunkedConnectionPoolSize=32", - - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000", - - "Volume:CommonVolume.FSID=1", - "Volume:CommonVolume.FUSEMountPointName=CommonMountPoint", - "Volume:CommonVolume.NFSExportClientMapList=CommonVolumeNFSClient0", - "Volume:CommonVolume.SMBShareName=CommonShare", - "Volume:CommonVolume.PrimaryPeer=Peer0", - "Volume:CommonVolume.AccountName=AUTH_CommonAccount", - "Volume:CommonVolume.CheckpointContainerName=.__checkpoint__", - "Volume:CommonVolume.CheckpointContainerStoragePolicy=gold", - "Volume:CommonVolume.CheckpointInterval=10s", - "Volume:CommonVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way", - "Volume:CommonVolume.MaxFlushSize=10485760", - "Volume:CommonVolume.MaxFlushTime=10s", - "Volume:CommonVolume.FileDefragmentChunkSize=10485760", - "Volume:CommonVolume.FileDefragmentChunkDelay=10ms", - "Volume:CommonVolume.NonceValuesToReserve=100", - "Volume:CommonVolume.MaxEntriesPerDirNode=32", - "Volume:CommonVolume.MaxExtentsPerFileNode=32", - "Volume:CommonVolume.MaxInodesPerMetadataNode=32", - "Volume:CommonVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:CommonVolume.MaxDirFileNodesPerMetadataNode=16", - "Volume:CommonVolume.MaxBytesInodeCache=100000", - "Volume:CommonVolume.InodeCacheEvictInterval=1s", - "Volume:CommonVolume.ActiveLeaseEvictLowLimit=5000", - "Volume:CommonVolume.ActiveLeaseEvictHighLimit=5010", - - "NFSClientMap:CommonVolumeNFSClient0.ClientPattern=*", - "NFSClientMap:CommonVolumeNFSClient0.AccessMode=rw", - "NFSClientMap:CommonVolumeNFSClient0.RootSquash=no_root_squash", - "NFSClientMap:CommonVolumeNFSClient0.Secure=insecure", - - "VolumeGroup:CommonVolumeGroup.VolumeList=CommonVolume", - "VolumeGroup:CommonVolumeGroup.VirtualIPAddr=", - "VolumeGroup:CommonVolumeGroup.PrimaryPeer=Peer0", - "VolumeGroup:CommonVolumeGroup.ReadCacheLineSize=1000000", - "VolumeGroup:CommonVolumeGroup.ReadCacheWeight=100", - - "FSGlobals.VolumeGroupList=CommonVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.DirEntryCacheEvictLowLimit=10000", - "FSGlobals.DirEntryCacheEvictHighLimit=10010", - "FSGlobals.FileExtentMapEvictLowLimit=10000", - "FSGlobals.FileExtentMapEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - - "JSONRPCServer.TCPPort=12346", // 12346 instead of 12345 so that test can run if proxyfsd is already running - "JSONRPCServer.FastTCPPort=32346", // ...and similarly here... - "JSONRPCServer.DataPathLogging=false", - "JSONRPCServer.MinLeaseDuration=250ms", - "JSONRPCServer.LeaseInterruptInterval=250ms", - "JSONRPCServer.LeaseInterruptLimit=20", - - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - } - - testVersionConfFile, err = ioutil.TempFile(os.TempDir(), "proxyfsdTest_") - if nil != err { - t.Fatalf("ioutil.TempFile() failed: %v", err) - } - testVersionConfFileName = testVersionConfFile.Name() - - _, err = testVersionConfFile.WriteString("[TestVersionSection]\n") - if nil != err { - t.Fatalf("testVersionConfFile.WriteString() [case 1] failed: %v", err) - } - _, err = testVersionConfFile.WriteString("Version: 1\n") - if nil != err { - t.Fatalf("testVersionConfFile.WriteString() [case 2] failed: %v", err) - } - - err = testVersionConfFile.Close() - if nil != err { - t.Fatalf("testVersionConfFile.Close() [case 1] failed: %v", err) - } - - ramswiftSignalHandlerIsArmedWG.Add(1) - ramswiftDoneChan = make(chan bool, 1) - - go ramswift.Daemon(testVersionConfFileName, confMapStrings, &ramswiftSignalHandlerIsArmedWG, ramswiftDoneChan, unix.SIGINT) - - ramswiftSignalHandlerIsArmedWG.Wait() - - // Launch an instance of proxyfsd using above config with Volume:CommonVolume.AutoFormat=true - - errChan = make(chan error, 1) // Must be buffered to avoid race - - confMapStringsWithAutoFormatTrueAdded = append(confMapStrings, "Volume:CommonVolume.AutoFormat=true") - execArgs = append([]string{"daemon_test", "/dev/null"}, confMapStringsWithAutoFormatTrueAdded...) - go Daemon("/dev/null", confMapStringsWithAutoFormatTrueAdded, errChan, &wg, - execArgs, unix.SIGTERM, unix.SIGHUP) - - err = <-errChan - if nil != err { - t.Fatalf("Daemon() startup failed [case 1a]: %v", err) - } - - // Write to the volume (with no flush so that only time-based/restart flush is performed) - - volumeHandle, err = fs.FetchVolumeHandleByVolumeName("CommonVolume") - if nil != err { - t.Fatalf("fs.FetchVolumeHandleByVolumeName() failed [case 1]: %v", err) - } - - createdFileInodeNumber, err = volumeHandle.Create( - inode.InodeRootUserID, - inode.InodeGroupID(0), - nil, - inode.RootDirInodeNumber, - "TestFile", - inode.R_OK|inode.W_OK, - ) - if nil != err { - t.Fatalf("fs.Create() failed: %v", err) - } - - bytesWritten, err = volumeHandle.Write( - inode.InodeRootUserID, - inode.InodeGroupID(0), - nil, - createdFileInodeNumber, - 0, - []byte{0x00, 0x01, 0x02, 0x03}, - nil, - ) - if nil != err { - t.Fatalf("fs.Write() failed: %v", err) - } - if 4 != bytesWritten { - t.Fatalf("fs.Write() returned unexpected bytesWritten") - } - - // Verify written data before restart - - toReadFileInodeNumber, err = volumeHandle.Lookup( - inode.InodeRootUserID, - inode.InodeGroupID(0), - nil, - inode.RootDirInodeNumber, - "TestFile", - ) - if nil != err { - t.Fatalf("fs.Lookup() failed [case 1]: %v", err) - } - - readData, err = volumeHandle.Read( - inode.InodeRootUserID, - inode.InodeGroupID(0), - nil, - toReadFileInodeNumber, - 0, - 4, - nil, - ) - if nil != err { - t.Fatalf("fs.Read() failed [case 1]: %v", err) - } - if 0 != bytes.Compare([]byte{0x00, 0x01, 0x02, 0x03}, readData) { - t.Fatalf("fs.Read() returned unexpected readData [case 1]") - } - - // Send ourself a SIGTERM to signal normal termination of mainWithArgs() - - unix.Kill(unix.Getpid(), unix.SIGTERM) - - err = <-errChan - - wg.Wait() // wait for services to go Down() - - if nil != err { - t.Fatalf("Daemon() exited with error [case 1b]: == %v", err) - } - - // Relaunch an instance of proxyfsd (without Volume:TestVolume.AutoFormat=true) - - errChan = make(chan error, 1) // Must be buffered to avoid race - - execArgs = append([]string{"daemon_test", testVersionConfFileName}, confMapStrings...) - go Daemon(testVersionConfFileName, confMapStrings, errChan, &wg, - execArgs, unix.SIGTERM, unix.SIGHUP) - - err = <-errChan - if nil != err { - t.Fatalf("Daemon() startup failed [case 2a]: %v", err) - } - - // Verify written data after restart - - volumeHandle, err = fs.FetchVolumeHandleByVolumeName("CommonVolume") - if nil != err { - t.Fatalf("fs.FetchVolumeHandleByVolumeName() failed [case 2]: %v", err) - } - - toReadFileInodeNumber, err = volumeHandle.Lookup( - inode.InodeRootUserID, - inode.InodeGroupID(0), - nil, - inode.RootDirInodeNumber, - "TestFile", - ) - if nil != err { - t.Fatalf("fs.Lookup() failed [case 2]: %v", err) - } - - readData, err = volumeHandle.Read( - inode.InodeRootUserID, - inode.InodeGroupID(0), - nil, - toReadFileInodeNumber, - 0, - 4, - nil, - ) - if nil != err { - t.Fatalf("fs.Read() failed [case 2]: %v", err) - } - if 0 != bytes.Compare([]byte{0x00, 0x01, 0x02, 0x03}, readData) { - t.Fatalf("fs.Read() returned unexpected readData [case 2]") - } - - // Verify [TestVersionSection]Version == 1 - - testVersion = fetchTestVersionSectionDotVersion(t) - if 1 != testVersion { - t.Fatalf("Before SIGHUP, fetchTestVersionSectionDotVersion() should have returned 1") - } - - // Update testVersionConfFileName to bump [TestVersionSection]Version to 2 - - testVersionConfFile, err = os.OpenFile(testVersionConfFileName, os.O_TRUNC|os.O_WRONLY, 0) - if nil != err { - t.Fatalf("os.OpenFile() failed: %v", err) - } - - _, err = testVersionConfFile.WriteString("[TestVersionSection]\n") - if nil != err { - t.Fatalf("testVersionConfFile.WriteString() [case 3] failed: %v", err) - } - _, err = testVersionConfFile.WriteString("Version: 2\n") - if nil != err { - t.Fatalf("testVersionConfFile.WriteString() [case 4] failed: %v", err) - } - - err = testVersionConfFile.Close() - if nil != err { - t.Fatalf("testVersionConfFile.Close() [case 2] failed: %v", err) - } - - // Send ourself a SIGHUP to signal reload of testVersionConfFileName - // and wait a sufficient amount of time for the reload to complete - - unix.Kill(unix.Getpid(), unix.SIGHUP) - time.Sleep(time.Second) - - // Verify [TestVersionSection]Version == 2 - - testVersion = fetchTestVersionSectionDotVersion(t) - if 2 != testVersion { - t.Fatalf("After SIGHUP, fetchTestVersionSectionDotVersion() should have returned 2") - } - - // Send ourself a SIGTERM to signal normal termination of mainWithArgs() - - unix.Kill(unix.Getpid(), unix.SIGTERM) - - err = <-errChan - - wg.Wait() // wait for services to go Down() - - if nil != err { - t.Fatalf("Daemon() exited with error [case 2b]: %v", err) - } - - // Send ourself a SIGINT to also terminate ramswift - - unix.Kill(unix.Getpid(), unix.SIGINT) - - _ = <-ramswiftDoneChan - - // Clean up - - err = os.Remove(testVersionConfFileName) - if nil != err { - t.Fatalf("os.Remove(testVersionConfFileName) failed: %v", err) - } -} - -func fetchTestVersionSectionDotVersion(t *testing.T) (version uint64) { - var ( - body []byte - bodySections map[string]interface{} - err error - ok bool - resp *http.Response - testVersionSection interface{} - testVersionSectionMap map[string]interface{} - testVersionSectionMapVersion interface{} - testVersionSectionMapVersionSlice []interface{} - testVersionSectionMapVersionSliceElementZero interface{} - versionAsString string - ) - - resp, err = http.Get("http://127.0.0.1:53461/config") - if nil != err { - t.Fatalf("http.Get() failed: %v", err) - } - body, err = ioutil.ReadAll(resp.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() failed: %v", err) - } - err = resp.Body.Close() - if nil != err { - t.Fatalf("resp.Body.Close() failed: %v", err) - } - bodySections = make(map[string]interface{}) - err = json.Unmarshal(body, &bodySections) - if nil != err { - t.Fatalf("json.Unmarshal() failed: %v", err) - } - testVersionSection, ok = bodySections["TestVersionSection"] - if !ok { - t.Fatalf("bodySections[\"TestVersionSection\"] not found") - } - testVersionSectionMap, ok = testVersionSection.(map[string]interface{}) - if !ok { - t.Fatalf("testVersionSection.(map[string]interface{}) failed") - } - testVersionSectionMapVersion, ok = testVersionSectionMap["Version"] - if !ok { - t.Fatalf("testVersionSectionMap[\"Version\"] not found") - } - testVersionSectionMapVersionSlice, ok = testVersionSectionMapVersion.([]interface{}) - if !ok { - t.Fatalf("testVersionSectionMapVersion.([]interface{}) failed") - } - if 1 != len(testVersionSectionMapVersionSlice) { - t.Fatalf("testVersionSectionMapVersionSlice should have a single element") - } - testVersionSectionMapVersionSliceElementZero = testVersionSectionMapVersionSlice[0] - versionAsString, ok = testVersionSectionMapVersionSliceElementZero.(string) - if !ok { - t.Fatalf("testVersionSectionMapVersionSliceElementZero.(string) failed") - } - version, err = strconv.ParseUint(versionAsString, 10, 64) - if nil != err { - t.Fatalf("strconv(versionAsString, 10, 64) failed: %v", err) - } - - return -} diff --git a/proxyfsd/debug.conf b/proxyfsd/debug.conf deleted file mode 100644 index dbee9382..00000000 --- a/proxyfsd/debug.conf +++ /dev/null @@ -1,13 +0,0 @@ -# Debug-related parameters for proxyfs -# -# ProfileType is used to enable performance profiling of proxyfsd using pkg/profile. -# The possible values for ProfileType are: Memory, Block, CPU, Mutex or None. -# Only one of these value can be set. If not specified, the default is no profiling. -# -# DebugServerPort will be served by an embedded HTTP Server responding to requests -# as described in package net/http/pprof documentation. If not defined, the embedded -# HTTP Server is not launched. -# -[ProxyfsDebug] -ProfileType: None -DebugServerPort: 6058 diff --git a/proxyfsd/default.conf b/proxyfsd/default.conf deleted file mode 100644 index db7a27d2..00000000 --- a/proxyfsd/default.conf +++ /dev/null @@ -1,277 +0,0 @@ -# Default .conf file - -# Each "peer" in the cluster is listed here... but, for now, there should only be one (for a given node) -[Peer:Peer0] -PublicIPAddr: 192.168.22.40 -PrivateIPAddr: 192.168.23.40 -ReadCacheQuotaFraction: 0.20 - -# Identifies what "peers" make up the cluster (there should only be one for now) and which one "we" are -# -# For LogLevel, valid values are: -# 0 == None (the default) -# 1 == State changes -# 2 == Messages -# 3 == Message details -# 4 == Max -[Cluster] -WhoAmI: Peer0 -Peers: Peer0 -ServerGuid: 0bb51164-258f-4e04-a417-e16d736ca41c -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 - -# Specifies the path particulars to the "NoAuth" WSGI pipeline -[SwiftClient] -NoAuthIPAddr: 127.0.0.1 -NoAuthTCPPort: 8090 - -RetryDelay: 1s -RetryExpBackoff: 1.5 -RetryLimit: 11 - -RetryDelayObject: 1s -RetryExpBackoffObject: 1.95 -RetryLimitObject: 8 - -ChunkedConnectionPoolSize: 512 -NonChunkedConnectionPoolSize: 128 - -SwiftReconNoWriteThreshold: 80 -SwiftReconNoWriteErrno: ENOSPC -SwiftReconReadOnlyThreshold: 90 -SwiftReconReadOnlyErrno: EROFS -SwiftConfDir: /etc/swift -SwiftReconChecksPerConfCheck: 10 - -# A storage policy into which the chunks of files and directories will go -[PhysicalContainerLayout:CommonVolumePhysicalContainerLayoutReplicated3Way] -ContainerStoragePolicy: silver -ContainerNamePrefix: Replicated3Way_ -ContainersPerPeer: 10 -MaxObjectsPerContainer: 1000000 - -# A set of snapshot schedules in crontab format + keep count -# -# CronTab fields are: -# -# field allowed values -# ----- -------------- -# minute 0-59 -# hour 0-23 -# day of month 1-31 -# month 1-12 -# day of week 0-6 (0 == Sunday) -# -# Note that full crontab parsing is not supported... only single values are allowed for each field -[SnapShotSchedule:MinutelySnapShotSchedule] -CronTab: * * * * * # Every minute -Keep: 59 - -[SnapShotSchedule:HourlySnapShotSchedule] -CronTab: 0 * * * * # At the top of every hour -Keep: 23 - -[SnapShotSchedule:DailySnapShotSchedule] -CronTab: 0 0 * * * # At midnight every day -Keep: 6 - -[SnapShotSchedule:WeeklySnapShotSchedule] -CronTab: 0 0 * * 0 # At midnight every Sunday -Keep: 8 - -[SnapShotSchedule:MonthlySnapShotSchedule] -CronTab: 0 0 1 * * # At midnight on the first of every month -Keep: 11 - -[SnapShotSchedule:YearlySnapShotSchedule] -CronTab: 0 0 1 1 * # At midnight on the January 1st of every year -Keep: 4 - -# A snapshot policy referencing one or more snapshot schedules -# -# If TimeZone is not specified or is either "" or "UTC", Etc/UTC is used -# If TimeZone is "Local", the local time zone is used -# Otherwise, the TimeZone must be from the IANA Time Zone database, such as "America/Los_Angeles" -# -# TimeZone is used to both compute the time for each scheduled snapshot -# as well as how to name each taken snapshot using the format specified in RFC3339 -[SnapShotPolicy:CommonSnapShotPolicy] -ScheduleList: MinutelySnapShotSchedule,HourlySnapShotSchedule,DailySnapShotSchedule,WeeklySnapShotSchedule,MonthlySnapShotSchedule,YearlySnapShotSchedule -TimeZone: America/Los_Angeles - -# A description of a volume / file system -[Volume:CommonVolume] -FSID: 1 -FUSEMountPointName: CommonMountPoint -NFSExportClientMapList: CommonVolumeNFSClient0 -SMBShareName: CommonShare -AccountName: AUTH_test -AutoFormat: false -NonceValuesToReserve: 100 -MaxEntriesPerDirNode: 32 -MaxExtentsPerFileNode: 32 -MaxInodesPerMetadataNode: 32 -MaxLogSegmentsPerMetadataNode: 64 -MaxDirFileNodesPerMetadataNode: 16 -MaxCreatedDeletedObjectsPerMetadataNode: 256 -CheckpointEtcdKeyName: ProxyFS:Volume:CommonVolume:Checkpoint -CheckpointContainerName: .__checkpoint__ -CheckpointContainerStoragePolicy: gold -CheckpointInterval: 10s -#ReplayLogFileName: CommonVolume.rlog -DefaultPhysicalContainerLayout: CommonVolumePhysicalContainerLayoutReplicated3Way -MaxFlushSize: 10485760 -MaxFlushTime: 10s -FileDefragmentChunkSize: 10485760 -FileDefragmentChunkDelay: 10ms -ReportedBlockSize: 65536 -ReportedFragmentSize: 65536 -ReportedNumBlocks: 1677721600 -ReportedNumInodes: 107374182400 -SnapShotIDNumBits: 10 -MaxBytesInodeCache: 10485760 -InodeCacheEvictInterval: 1s -#SnapShotPolicy: CommonSnapShotPolicy # Optional -SMBValidUserList: swift -SMBBrowseable: true -SMBStrictSync: yes -SMBAuditLogging: false -SMBEncryptionRequired: false -ActiveLeaseEvictLowLimit: 500000 -ActiveLeaseEvictHighLimit: 500010 - -[NFSClientMap:CommonVolumeNFSClient0] -ClientPattern: * -AccessMode: rw -RootSquash: no_root_squash -Secure: insecure - -# A description of a volume group -# -# PrimaryPeer should be the lone Peer in Cluster.Peers that will serve this set of volumes -[VolumeGroup:CommonVolumeGroup] -VolumeList: CommonVolume -VirtualIPAddr: -PrimaryPeer: Peer0 -ReadCacheLineSize: 1048576 -ReadCacheWeight: 100 -SMBWorkgroup: # If missing or blank, defaults to WORKGROUP -SMBActiveDirectoryEnabled: false # If true, all other SMBActiveDirectory* Key:Values are required (defaults to false) -SMBActiveDirectoryRealm: -SMBActiveDirectoryIDMapDefaultMin: -SMBActiveDirectoryIDMapDefaultMax: -SMBActiveDirectoryIDMapWorkgroupMin: -SMBActiveDirectoryIDMapWorkgroupMax: - -# Describes the set of volumes of the file system listed above -[FSGlobals] -VolumeGroupList: CommonVolumeGroup -CheckpointHeaderConsensusAttempts: 5 -MountRetryLimit: 6 -MountRetryDelay: 1s -MountRetryExpBackoff: 2 -LogCheckpointHeaderPosts: true -TryLockBackoffMin: 10ms -TryLockBackoffMax: 50ms -TryLockSerializationThreshhold: 5 -SymlinkMax: 32 -CoalesceElementChunkSize: 16 -InodeRecCacheEvictLowLimit: 10000 -InodeRecCacheEvictHighLimit: 10010 -LogSegmentRecCacheEvictLowLimit: 10000 -LogSegmentRecCacheEvictHighLimit: 10010 -BPlusTreeObjectCacheEvictLowLimit: 10000 -BPlusTreeObjectCacheEvictHighLimit: 10010 -CreatedDeletedObjectsCacheEvictLowLimit: 10000 -CreatedDeletedObjectsCacheEvictHighLimit: 10010 -DirEntryCacheEvictLowLimit: 10000 -DirEntryCacheEvictHighLimit: 10010 -FileExtentMapEvictLowLimit: 10000 -FileExtentMapEvictHighLimit: 10010 -EtcdEnabled: false -EtcdEndpoints: 127.0.0.1:2379 -EtcdAutoSyncInterval: 1m -EtcdCertDir: /etc/ssl/etcd/ssl/ -EtcdDialTimeout: 10s -EtcdOpTimeout: 20s -MetadataRecycleBin: false -SMBUserList: swift - -# Each SMBUser is s Key who's Value is (e.g. in Python) the base64.standard_b64encode() of the SMBUser's Password -[SMBUsers] -swift: c3dpZnQ= - -# RPC path from file system clients (both Samba and "normal" WSGI stack)... needs to be shared with them -[JSONRPCServer] -TCPPort: 12345 -FastTCPPort: 32345 -DataPathLogging: false -Debug: false -RetryRPCPort: 32356 -RetryRPCTTLCompleted: 10m -RetryRPCAckTrim: 100ms -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCertFilePath: -RetryRPCKeyFilePath: -MinLeaseDuration: 250ms -LeaseInterruptInterval: 250ms -LeaseInterruptLimit: 20 - -# Log reporting parameters -[Logging] -LogFilePath: proxyfsd.log -LogToConsole: false # when true, log to stderr even when LogFilePath is set -TraceLevelLogging: none # Enable trace logging on a per-package basis. Supported values: dlm, fs, fuse, headhunter, inode, jrpcfs, logger, proxyfsd, sortedmap, swiftclient, and transitions...or none (default). -DebugLevelLogging: none # Enable debug logging on a per-package basis. Supported values: ldlm, fs, jrpcfs, and inode...or none (default). -# NOTE: Log levels other than Trace and Debug are always on. - -[EventLog] -Enabled: false -BufferKey: 1234 -BufferLength: 1048576 # 1MiB -MinBackoff: 1us -MaxBackoff: 2us -DaemonPollDelay: 10ms -DaemonOutputPath: # If blank, os.Stdout is used - -# Stats reporting parameters (must contain either a UDPPort or TCPPort) -[Stats] -UDPPort: 8125 -BufferLength: 1000 -MaxLatency: 1s - -# HTTP server -[HTTPServer] -TCPPort: 15346 -JobHistoryMaxSize: 5 - -# Write selected memory, connection, and Swift operation statistics -# to the log once each Period. The minimum Period is 10 min. Use -# 0 to disable statistic logging. -# -# Verbose setting will dump all available stats each Period. -[StatsLogger] -Period: 10m -Verbose: true - -# Debug-related parameters for proxyfs -# -# ProfileType is used to enable performance profiling of proxyfsd using pkg/profile. -# The possible values for ProfileType are: Memory, Block, CPU, Mutex or None. -# Only one of these value can be set. If not specified, the default is no profiling. -# -# DebugServerPort will be served by an embedded HTTP Server responding to requests -# as described in package net/http/pprof documentation. -[ProxyfsDebug] -ProfileType: None -DebugServerPort: 6060 diff --git a/proxyfsd/file_server.conf b/proxyfsd/file_server.conf deleted file mode 100644 index bad9d1c2..00000000 --- a/proxyfsd/file_server.conf +++ /dev/null @@ -1,147 +0,0 @@ -# File Server description - -[PhysicalContainerLayout:CommonVolumePhysicalContainerLayoutReplicated3Way] -ContainerStoragePolicy: silver # bronze -ContainerNamePrefix: Replicated3Way_ # ErasureCoded_ -ContainersPerPeer: 10 -MaxObjectsPerContainer: 1000000 - -[SnapShotSchedule:MinutelySnapShotSchedule] -CronTab: * * * * * # Every minute -Keep: 59 - -[SnapShotSchedule:HourlySnapShotSchedule] -CronTab: 0 * * * * # At the top of every hour -Keep: 23 - -[SnapShotSchedule:DailySnapShotSchedule] -CronTab: 0 0 * * * # At midnight every day -Keep: 6 - -[SnapShotSchedule:WeeklySnapShotSchedule] -CronTab: 0 0 * * 0 # At midnight every Sunday -Keep: 8 - -[SnapShotSchedule:MonthlySnapShotSchedule] -CronTab: 0 0 1 * * # At midnight on the first of every month -Keep: 11 - -[SnapShotSchedule:YearlySnapShotSchedule] -CronTab: 0 0 1 1 * # At midnight on the January 1st of every year -Keep: 4 - -[SnapShotPolicy:CommonSnapShotPolicy] -ScheduleList: MinutelySnapShotSchedule,HourlySnapShotSchedule,DailySnapShotSchedule,WeeklySnapShotSchedule,MonthlySnapShotSchedule,YearlySnapShotSchedule -TimeZone: America/Los_Angeles - -[SnapShotSchedule:TestSnapShotSchedule] -CronTab: * * * * * # Every minute -Keep: 3 - -[SnapShotPolicy:TestSnapShotPolicy] -ScheduleList: TestSnapShotSchedule -TimeZone: America/Los_Angeles - -# A description of a volume / file system -[Volume:CommonVolume] -FSID: 1 -FUSEMountPointName: CommonMountPoint -NFSExportClientMapList: CommonVolumeNFSClient0 -SMBShareName: CommonShare -AccountName: AUTH_test -AutoFormat: false -NonceValuesToReserve: 100 -MaxEntriesPerDirNode: 32 -MaxExtentsPerFileNode: 32 -MaxInodesPerMetadataNode: 32 -MaxLogSegmentsPerMetadataNode: 64 -MaxDirFileNodesPerMetadataNode: 16 -MaxCreatedDeletedObjectsPerMetadataNode: 256 -CheckpointEtcdKeyName: ProxyFS:Volume:CommonVolume:Checkpoint -CheckpointContainerName: .__checkpoint__ -CheckpointContainerStoragePolicy: gold -CheckpointInterval: 10s -#ReplayLogFileName: CommonVolume.rlog -DefaultPhysicalContainerLayout: CommonVolumePhysicalContainerLayoutReplicated3Way -MaxFlushSize: 10485760 -MaxFlushTime: 10s -FileDefragmentChunkSize: 10485760 -FileDefragmentChunkDelay: 10ms -ReportedBlockSize: 65536 -ReportedFragmentSize: 65536 -ReportedNumBlocks: 1677721600 -ReportedNumInodes: 107374182400 -SnapShotIDNumBits: 10 -MaxBytesInodeCache: 10485760 -InodeCacheEvictInterval: 1s -#SnapShotPolicy: CommonSnapShotPolicy # Optional -#SnapShotPolicy: TestSnapShotPolicy -SMBValidUserList: swift -SMBBrowseable: true -SMBStrictSync: yes -SMBAuditLogging: false -SMBEncryptionRequired: false -ActiveLeaseEvictLowLimit: 500000 -ActiveLeaseEvictHighLimit: 500010 - -[NFSClientMap:CommonVolumeNFSClient0] -ClientPattern: * -AccessMode: rw -RootSquash: no_root_squash -Secure: insecure - -# A description of a volume group -# -# PrimaryPeer should be the lone Peer in Cluster.Peers that will serve this set of volumes -# All VolumeGroups sharing a VirtualIPAddr (empty or not) containing SMB Volumes should agree on SMB* Key:Values -[VolumeGroup:CommonVolumeGroup] -VolumeList: CommonVolume -VirtualIPAddr: -PrimaryPeer: Peer0 -ReadCacheLineSize: 1048576 -ReadCacheWeight: 100 -SMBWorkgroup: # If missing or blank, defaults to WORKGROUP -SMBActiveDirectoryEnabled: false # If true, all other SMBActiveDirectory* Key:Values are required (defaults to false) -SMBActiveDirectoryRealm: -SMBActiveDirectoryIDMapDefaultMin: -SMBActiveDirectoryIDMapDefaultMax: -SMBActiveDirectoryIDMapWorkgroupMin: -SMBActiveDirectoryIDMapWorkgroupMax: - -[FSGlobals] -VolumeGroupList: CommonVolumeGroup -CheckpointHeaderConsensusAttempts: 5 -MountRetryLimit: 6 -MountRetryDelay: 1s -MountRetryExpBackoff: 2 -LogCheckpointHeaderPosts: true -TryLockBackoffMin: 10ms -TryLockBackoffMax: 50ms -TryLockSerializationThreshhold: 5 -SymlinkMax: 32 -CoalesceElementChunkSize: 16 -InodeRecCacheEvictLowLimit: 10000 -InodeRecCacheEvictHighLimit: 10010 -LogSegmentRecCacheEvictLowLimit: 10000 -LogSegmentRecCacheEvictHighLimit: 10010 -BPlusTreeObjectCacheEvictLowLimit: 10000 -BPlusTreeObjectCacheEvictHighLimit: 10010 -CreatedDeletedObjectsCacheEvictLowLimit: 10000 -CreatedDeletedObjectsCacheEvictHighLimit: 10010 -DirEntryCacheEvictLowLimit: 10000 -DirEntryCacheEvictHighLimit: 10010 -FileExtentMapEvictLowLimit: 10000 -FileExtentMapEvictHighLimit: 10010 -EtcdEnabled: false -EtcdEndpoints: 127.0.0.1:2379 -EtcdAutoSyncInterval: 1m -EtcdCertDir: /etc/ssl/etcd/ssl/ -EtcdDialTimeout: 10s -EtcdOpTimeout: 20s -MetadataRecycleBin: false -SMBUserList: swift -SMBMapToGuest: # One of Never, Bad User, Bad Password, or Bad Uid (case insensitive)... defaults to Never -SMBNetBiosName: # Defaults to `hostname -s` (i.e. short host name) - -[SMBUsers] -swift: c3dpZnQ= # base64.standard_b64encode("swift") diff --git a/proxyfsd/file_server_mac_3_peers.conf b/proxyfsd/file_server_mac_3_peers.conf deleted file mode 100644 index 13c85680..00000000 --- a/proxyfsd/file_server_mac_3_peers.conf +++ /dev/null @@ -1,155 +0,0 @@ -# File Server description - -[PhysicalContainerLayout:CommonVolumePhysicalContainerLayoutReplicated3Way] -ContainerStoragePolicy: silver -ContainerNamePrefix: Replicated3Way_ -ContainersPerPeer: 10 -MaxObjectsPerContainer: 1000000 - -[SnapShotSchedule:MinutelySnapShotSchedule] -CronTab: * * * * * # Every minute -Keep: 59 - -[SnapShotSchedule:HourlySnapShotSchedule] -CronTab: 0 * * * * # At the top of every hour -Keep: 23 - -[SnapShotSchedule:DailySnapShotSchedule] -CronTab: 0 0 * * * # At midnight every day -Keep: 6 - -[SnapShotSchedule:WeeklySnapShotSchedule] -CronTab: 0 0 * * 0 # At midnight every Sunday -Keep: 8 - -[SnapShotSchedule:MonthlySnapShotSchedule] -CronTab: 0 0 1 * * # At midnight on the first of every month -Keep: 11 - -[SnapShotSchedule:YearlySnapShotSchedule] -CronTab: 0 0 1 1 * # At midnight on the January 1st of every year -Keep: 4 - -[SnapShotPolicy:CommonSnapShotPolicy] -ScheduleList: MinutelySnapShotSchedule,HourlySnapShotSchedule,DailySnapShotSchedule,WeeklySnapShotSchedule,MonthlySnapShotSchedule,YearlySnapShotSchedule -TimeZone: America/Los_Angeles - -[SnapShotSchedule:TestSnapShotSchedule] -CronTab: * * * * * # Every minute -Keep: 3 - -[SnapShotPolicy:TestSnapShotPolicy] -ScheduleList: TestSnapShotSchedule -TimeZone: America/Los_Angeles - -# A description of a volume / file system -[Volume:CommonVolume] -FSID: 1 -FUSEMountPointName: CommonMountPoint -NFSExportClientMapList: CommonVolumeNFSClient0 -SMBShareName: CommonShare -AccountName: AUTH_test -AutoFormat: false -NonceValuesToReserve: 100 -MaxEntriesPerDirNode: 32 -MaxExtentsPerFileNode: 32 -MaxInodesPerMetadataNode: 32 -MaxLogSegmentsPerMetadataNode: 64 -MaxDirFileNodesPerMetadataNode: 16 -MaxCreatedDeletedObjectsPerMetadataNode: 256 -CheckpointEtcdKeyName: ProxyFS:Volume:CommonVolume:Checkpoint -CheckpointContainerName: .__checkpoint__ -CheckpointContainerStoragePolicy: gold -CheckpointInterval: 10s -#ReplayLogFileName: CommonVolume.rlog -DefaultPhysicalContainerLayout: CommonVolumePhysicalContainerLayoutReplicated3Way -MaxFlushSize: 10485760 -MaxFlushTime: 10s -FileDefragmentChunkSize: 10485760 -FileDefragmentChunkDelay: 10ms -ReportedBlockSize: 65536 -ReportedFragmentSize: 65536 -ReportedNumBlocks: 1677721600 -ReportedNumInodes: 107374182400 -SnapShotIDNumBits: 10 -MaxBytesInodeCache: 10485760 -InodeCacheEvictInterval: 1s -#SnapShotPolicy: CommonSnapShotPolicy # Optional -#SnapShotPolicy: TestSnapShotPolicy -SMBValidUserList: swift -SMBBrowseable: true -SMBStrictSync: yes -SMBAuditLogging: false -SMBEncryptionRequired: false -ActiveLeaseEvictLowLimit: 500000 -ActiveLeaseEvictHighLimit: 500010 - -[NFSClientMap:CommonVolumeNFSClient0] -ClientPattern: * -AccessMode: rw -RootSquash: no_root_squash -Secure: insecure - -# A description of a volume group -# -# PrimaryPeer should be the lone Peer in Cluster.Peers that will serve this set of volumes -# All VolumeGroups sharing a VirtualIPAddr (empty or not) containing SMB Volumes should agree on SMB* Key:Values -[VolumeGroup:CommonVolumeGroup] -VolumeList: CommonVolume -VirtualIPAddr: -PrimaryPeer: Peer1 -ReadCacheLineSize: 1048576 -ReadCacheWeight: 100 -SMBWorkgroup: # If missing or blank, defaults to WORKGROUP -SMBActiveDirectoryEnabled: false # If true, all other SMBActiveDirectory* Key:Values are required (defaults to false) -SMBActiveDirectoryRealm: -SMBActiveDirectoryIDMapDefaultMin: -SMBActiveDirectoryIDMapDefaultMax: -SMBActiveDirectoryIDMapWorkgroupMin: -SMBActiveDirectoryIDMapWorkgroupMax: - -[VolumeGroup:EmptyVolumeGroup] -VolumeList: -VirtualIPAddr: -PrimaryPeer: Peer2 -ReadCacheLineSize: 1048576 -ReadCacheWeight: 100 -SMBActiveDirectoryEnabled: false - -[FSGlobals] -VolumeGroupList: CommonVolumeGroup EmptyVolumeGroup -CheckpointHeaderConsensusAttempts: 5 -MountRetryLimit: 6 -MountRetryDelay: 1s -MountRetryExpBackoff: 2 -LogCheckpointHeaderPosts: true -TryLockBackoffMin: 10ms -TryLockBackoffMax: 50ms -TryLockSerializationThreshhold: 5 -SymlinkMax: 32 -CoalesceElementChunkSize: 16 -InodeRecCacheEvictLowLimit: 10000 -InodeRecCacheEvictHighLimit: 10010 -LogSegmentRecCacheEvictLowLimit: 10000 -LogSegmentRecCacheEvictHighLimit: 10010 -BPlusTreeObjectCacheEvictLowLimit: 10000 -BPlusTreeObjectCacheEvictHighLimit: 10010 -CreatedDeletedObjectsCacheEvictLowLimit: 10000 -CreatedDeletedObjectsCacheEvictHighLimit: 10010 -DirEntryCacheEvictLowLimit: 10000 -DirEntryCacheEvictHighLimit: 10010 -FileExtentMapEvictLowLimit: 10000 -FileExtentMapEvictHighLimit: 10010 -EtcdEnabled: false -EtcdEndpoints: 127.0.0.1:2379 -EtcdAutoSyncInterval: 1m -EtcdCertDir: /etc/ssl/etcd/ssl/ -EtcdDialTimeout: 10s -EtcdOpTimeout: 20s -MetadataRecycleBin: false -SMBUserList: swift -SMBMapToGuest: # One of Never, Bad User, Bad Password, or Bad Uid (case insensitive)... defaults to Never -SMBNetBiosName: # Defaults to `hostname -s` (i.e. short host name) - -[SMBUsers] -swift: c3dpZnQ= # base64.standard_b64encode("swift") diff --git a/proxyfsd/httpserver.conf b/proxyfsd/httpserver.conf deleted file mode 100644 index de2f9a9b..00000000 --- a/proxyfsd/httpserver.conf +++ /dev/null @@ -1,6 +0,0 @@ -# HTTP server - -[HTTPServer] -TCPPort: 15346 -JobHistoryMaxSize: 5 - diff --git a/proxyfsd/logging.conf b/proxyfsd/logging.conf deleted file mode 100644 index 0542b538..00000000 --- a/proxyfsd/logging.conf +++ /dev/null @@ -1,29 +0,0 @@ -[Logging] -LogFilePath: proxyfsd.log - -# NOTE: Log levels other than Trace and Debug are always on. - -# Enable trace logging on a per-package basis. Trace logs are disabled -# by default unless enabled here. -# -# Supported values: dlm, fs, fuse, headhunter, inode, jrpcfs, logger, -# proxyfsd, sortedmap, swiftclient, and transitions...or none (default). -TraceLevelLogging: none - -# Enable debug logging on a per-package basis. Debug logs are disabled -# by default unless enabled here. -# -# Supported values: dlm, fs, inode, and jrpcfs...or none (default). -DebugLevelLogging: none - -# when true, log to stderr even when LogFilePath is set -LogToConsole: false - -[EventLog] -Enabled: false -BufferKey: 1234 -BufferLength: 1048576 # 1MiB -MinBackoff: 1us -MaxBackoff: 2us -DaemonPollDelay: 10ms -DaemonOutputPath: # If blank, os.Stdout is used diff --git a/proxyfsd/mac_cluster_1_peer.conf b/proxyfsd/mac_cluster_1_peer.conf deleted file mode 100644 index a74b620d..00000000 --- a/proxyfsd/mac_cluster_1_peer.conf +++ /dev/null @@ -1,22 +0,0 @@ -# Mac Cluster .conf file for 1 Peer -# -# Following .include, caller should define Cluster.WhoAmI as Peer0 - -[Peer:Peer0] -PublicIPAddr: 127.0.0.1 # Normally 192.168.22.40 -PrivateIPAddr: 127.0.0.1 # Normally 192.168.23.40 -ReadCacheQuotaFraction: 0.20 - -[Cluster] -Peers: Peer0 -ServerGuid: 30ae4a7e-b28b-4fcf-b8c4-b65dbe25b5e7 -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 diff --git a/proxyfsd/mac_cluster_3_peers.conf b/proxyfsd/mac_cluster_3_peers.conf deleted file mode 100644 index 0882e834..00000000 --- a/proxyfsd/mac_cluster_3_peers.conf +++ /dev/null @@ -1,39 +0,0 @@ -# Mac Cluster .conf file for 3 Peers -# -# Following .include, caller should define Cluster.WhoAmI as Peer1, Peer2, or Peer3 -# -# MacOS only provided 127.0.0.1 (& [::1]) by default -# To enable multiple "localhost" IPv4 Addresses to enable the following: -# -# sudo ifconfig lo0 alias 127.0.0.2 up -# sudo ifconfig lo0 alias 127.0.0.3 up -# sudo ifconfig lo0 alias 127.0.0.4 up - -[Peer:Peer1] -PublicIPAddr: 127.0.0.2 # Normally 192.168.22.41 -PrivateIPAddr: 127.0.0.2 # Normally 192.168.23.41 -ReadCacheQuotaFraction: 0.20 - -[Peer:Peer2] # sudo ifconfig lo0 alias 127.0.0.2 -PublicIPAddr: 127.0.0.3 # Normally 192.168.22.42 -PrivateIPAddr: 127.0.0.3 # Normally 192.168.23.42 -ReadCacheQuotaFraction: 0.20 - -[Peer:Peer3] # sudo ifconfig lo0 alias 127.0.0.3 -PublicIPAddr: 127.0.0.4 # Normally 192.168.22.43 -PrivateIPAddr: 127.0.0.4 # Normally 192.168.23.43 -ReadCacheQuotaFraction: 0.20 - -[Cluster] -Peers: Peer1 Peer2 Peer3 -ServerGuid: 30ae4a7e-b28b-4fcf-b8c4-b65dbe25b5e7 -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 diff --git a/proxyfsd/macproxyfsd0.conf b/proxyfsd/macproxyfsd0.conf deleted file mode 100644 index 071d314a..00000000 --- a/proxyfsd/macproxyfsd0.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Peer 0's .conf file - -[Cluster] -WhoAmI: Peer0 - -.include ./mac_cluster_1_peer.conf -.include ./swift_client.conf -.include ./file_server.conf -.include ./rpc_server.conf -.include ./logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf diff --git a/proxyfsd/macproxyfsd1.conf b/proxyfsd/macproxyfsd1.conf deleted file mode 100644 index 2f9e3f5e..00000000 --- a/proxyfsd/macproxyfsd1.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Peer 1's .conf file - -[Cluster] -WhoAmI: Peer1 - -.include ./mac_cluster_3_peers.conf -.include ./swift_client.conf -.include ./file_server_mac_3_peers.conf -.include ./rpc_server.conf -.include ./logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf diff --git a/proxyfsd/macproxyfsd2.conf b/proxyfsd/macproxyfsd2.conf deleted file mode 100644 index d82dd4b7..00000000 --- a/proxyfsd/macproxyfsd2.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Peer 2's .conf file - -[Cluster] -WhoAmI: Peer2 - -.include ./mac_cluster_3_peers.conf -.include ./swift_client.conf -.include ./file_server_mac_3_peers.conf -.include ./rpc_server.conf -.include ./logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf diff --git a/proxyfsd/macproxyfsd3.conf b/proxyfsd/macproxyfsd3.conf deleted file mode 100644 index f1520677..00000000 --- a/proxyfsd/macproxyfsd3.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Peer 3's .conf file - -[Cluster] -WhoAmI: Peer3 - -.include ./mac_cluster_3_peers.conf -.include ./swift_client.conf -.include ./file_server_mac_3_peers.conf -.include ./rpc_server.conf -.include ./logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf diff --git a/proxyfsd/proxyfsd/Makefile b/proxyfsd/proxyfsd/Makefile deleted file mode 100644 index 83d40620..00000000 --- a/proxyfsd/proxyfsd/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/proxyfsd/proxyfsd - -include ../../GoMakefile diff --git a/proxyfsd/proxyfsd/dummy_test.go b/proxyfsd/proxyfsd/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/proxyfsd/proxyfsd/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/proxyfsd/proxyfsd/main.go b/proxyfsd/proxyfsd/main.go deleted file mode 100644 index 82933f7f..00000000 --- a/proxyfsd/proxyfsd/main.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// The proxyfsd program is the main ProxyFS dæmon and is named accordingly. -package main - -import ( - "fmt" - "log" - "log/syslog" - "os" - "sync" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/proxyfsd" -) - -func main() { - if len(os.Args) < 2 { - log.Fatalf("no .conf file specified") - } - - errChan := make(chan error, 1) // Must be buffered to avoid race - var wg sync.WaitGroup - - syslogger, err := syslog.Dial("", "", syslog.LOG_DAEMON, "proxyfsd") - if err != nil { - fmt.Fprintf(os.Stderr, "proxyfsd/Daemon(): syslog.Dial() failed: %v\n", err) - syslogger = nil - } else { - syslogger.Info(fmt.Sprintf("starting up: calling Daemon()")) - } - - // empty signal list (final argument) means "catch all signals" its possible to catch - go proxyfsd.Daemon(os.Args[1], os.Args[2:], errChan, &wg, os.Args, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) - - // read errChan to indicate when proxyfsd.Daemon() is up and running... or - // that an error during startup has been encountered - - err = <-errChan - if nil != err { - if nil != syslogger { - syslogger.Err(fmt.Sprintf("shutting down: Daemon startup error: %v", err)) - } - } else { - // startup worked... now just wait for proxyfsd.Daemon() to exit - - err = <-errChan - - if nil != syslogger { - if nil == err { - syslogger.Info(fmt.Sprintf("shutting down: Daemon() finished")) - } else { - syslogger.Err(fmt.Sprintf("shutting down: Daemon() returned error: %v", err)) - } - } - - wg.Wait() // wait for services to go Down() - } - - // how ever we get to here, exit abnormally if nil != err - - if nil != err { - fmt.Fprintf(os.Stderr, "proxyfsd: Daemon(): returned error: %v\n", err) // Can't use logger.*() as it's not currently "up" - os.Exit(1) // Exit with non-success status that can be checked from scripts - } -} diff --git a/proxyfsd/proxyfsd0.conf b/proxyfsd/proxyfsd0.conf deleted file mode 100644 index 665645ec..00000000 --- a/proxyfsd/proxyfsd0.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Peer 0's .conf file - -[Cluster] -WhoAmI: Peer0 - -.include ./cluster_1_peer.conf -.include ./swift_client.conf -.include ./file_server.conf -.include ./rpc_server.conf -.include ./logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf diff --git a/proxyfsd/proxyfsd1.conf b/proxyfsd/proxyfsd1.conf deleted file mode 100644 index 6a75baf0..00000000 --- a/proxyfsd/proxyfsd1.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Peer 1's .conf file - -[Cluster] -WhoAmI: Peer1 - -.include ./cluster_3_peers.conf -.include ./swift_client.conf -.include ./file_server.conf -.include ./rpc_server.conf -.include ./logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf diff --git a/proxyfsd/proxyfsd2.conf b/proxyfsd/proxyfsd2.conf deleted file mode 100644 index 51d7636a..00000000 --- a/proxyfsd/proxyfsd2.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Peer 2's .conf file - -[Cluster] -WhoAmI: Peer2 - -.include ./cluster_3_peers.conf -.include ./swift_client.conf -.include ./file_server.conf -.include ./rpc_server.conf -.include ./logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf diff --git a/proxyfsd/proxyfsd3.conf b/proxyfsd/proxyfsd3.conf deleted file mode 100644 index 3051b67c..00000000 --- a/proxyfsd/proxyfsd3.conf +++ /dev/null @@ -1,15 +0,0 @@ -# Peer 3's .conf file - -[Cluster] -WhoAmI: Peer3 - -.include ./cluster_3_peers.conf -.include ./swift_client.conf -.include ./file_server.conf -.include ./rpc_server.conf -.include ./logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf diff --git a/proxyfsd/rpc_server.conf b/proxyfsd/rpc_server.conf deleted file mode 100644 index fe5ea332..00000000 --- a/proxyfsd/rpc_server.conf +++ /dev/null @@ -1,17 +0,0 @@ -# FS JSON RPC server for use with swift middleware and samba vfs - -[JSONRPCServer] -TCPPort: 12345 -FastTCPPort: 32345 -DataPathLogging: false -Debug: false -RetryRPCPort: 32356 -RetryRPCTTLCompleted: 10m -RetryRPCAckTrim: 100ms -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCertFilePath: -RetryRPCKeyFilePath: -MinLeaseDuration: 250ms -LeaseInterruptInterval: 250ms -LeaseInterruptLimit: 20 diff --git a/proxyfsd/saio_cluster_1_peer.conf b/proxyfsd/saio_cluster_1_peer.conf deleted file mode 100644 index 966bc1d6..00000000 --- a/proxyfsd/saio_cluster_1_peer.conf +++ /dev/null @@ -1,22 +0,0 @@ -# SAIO Cluster .conf file for 1 Peer -# -# Following .include, caller should define Cluster.WhoAmI as Peer0 - -[Peer:Peer0] -PublicIPAddr: 127.0.0.1 -PrivateIPAddr: 0.0.0.0 -ReadCacheQuotaFraction: 0.20 - -[Cluster] -Peers: Peer0 -ServerGuid: 0bb51164-258f-4e04-a417-e16d736ca41c -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 diff --git a/proxyfsd/saio_logging.conf b/proxyfsd/saio_logging.conf deleted file mode 100644 index ff268ee6..00000000 --- a/proxyfsd/saio_logging.conf +++ /dev/null @@ -1,24 +0,0 @@ -[Logging] -LogFilePath: /var/log/proxyfsd/proxyfsd.log - -# NOTE: Log levels other than Trace and Debug are always on. - -# Enable trace logging on a per-package basis. Trace logs are disabled by default unless enabled here. -# Supported values: dlm, fs, fuse, headhunter, inode, jrpcfs, logger, proxyfsd, sortedmap, swiftclient, and transitions...or none (default). -TraceLevelLogging: none - -# Enable debug logging on a per-package basis. Debug logs are disabled by default unless enabled here. -# Supported values: ldlm, fs, jrpcfs, and inode...or none (default). -DebugLevelLogging: none - -# when true, log to stderr even when LogFilePath is set— -LogToConsole: false - -[EventLog] -Enabled: false -BufferKey: 1234 -BufferLength: 1048576 # 1MiB -MinBackoff: 1us -MaxBackoff: 2us -DaemonPollDelay: 10ms -DaemonOutputPath: # If blank, os.Stdout is used diff --git a/proxyfsd/saioproxyfsd0.conf b/proxyfsd/saioproxyfsd0.conf deleted file mode 100644 index 9e05f5e9..00000000 --- a/proxyfsd/saioproxyfsd0.conf +++ /dev/null @@ -1,19 +0,0 @@ -# Peer 0's .conf file - -[Cluster] -WhoAmI: Peer0 - -.include ./saio_cluster_1_peer.conf -.include ./swift_client.conf -.include ./file_server.conf -.include ./rpc_server.conf -.include ./saio_logging.conf -.include ./stats.conf -.include ./statslogger.conf -.include ./trackedlock.conf -.include ./httpserver.conf -.include ./debug.conf - -# put it here to have the last word (override previous) -#[Logging] -#TraceLevelLogging: headhunter inode sortedmap diff --git a/proxyfsd/stats.conf b/proxyfsd/stats.conf deleted file mode 100644 index 20f97c92..00000000 --- a/proxyfsd/stats.conf +++ /dev/null @@ -1,5 +0,0 @@ -# Stats reporting parameters (must contain either a UDPPort or TCPPort) -[Stats] -UDPPort: 48125 # To use nc to test, use "TCPPort" rather than UDPPort. -BufferLength: 1000 -MaxLatency: 1s diff --git a/proxyfsd/statslogger.conf b/proxyfsd/statslogger.conf deleted file mode 100644 index 152eca19..00000000 --- a/proxyfsd/statslogger.conf +++ /dev/null @@ -1,8 +0,0 @@ -# Write selected memory, connection, and Swift operation statistics -# to the log once each Period. The minimum Period is 10 min. Use -# 0 to disable statistic logging. -# -# Verbose setting will dump all available stats each Period. -[StatsLogger] -Period: 10m -Verbose: true diff --git a/proxyfsd/swift_client.conf b/proxyfsd/swift_client.conf deleted file mode 100644 index 857ddc2f..00000000 --- a/proxyfsd/swift_client.conf +++ /dev/null @@ -1,33 +0,0 @@ -# swiftclient settings -# -# RetryDelay, RetryExpBackoff, RetryLimit control the behavior of -# swiftclient when operations (GET, PUT, etc.) fail for a container or -# object and RetryDelayObject, RetryLimitObject, RetryExpBackoffObject -# control the behavior when operations fail for an object. -# -# RetryDelay is the time before the first retry; RetryLimit is the -# maximum number of retries, and RetryExpBackoff is an exponential -# backoff which is multiplied with the previous RetryDelay to -# determine how long to wait for the second, third, fourth, -# etc. retry. -[SwiftClient] -NoAuthIPAddr: 127.0.0.1 -NoAuthTCPPort: 8090 - -RetryDelay: 1s -RetryExpBackoff: 1.5 -RetryLimit: 11 - -RetryDelayObject: 1s -RetryExpBackoffObject: 1.95 -RetryLimitObject: 8 - -ChunkedConnectionPoolSize: 512 -NonChunkedConnectionPoolSize: 128 - -SwiftReconNoWriteThreshold: 80 -SwiftReconNoWriteErrno: ENOSPC -SwiftReconReadOnlyThreshold: 90 -SwiftReconReadOnlyErrno: EROFS -SwiftConfDir: /etc/swift -SwiftReconChecksPerConfCheck: 10 diff --git a/proxyfsd/trackedlock.conf b/proxyfsd/trackedlock.conf deleted file mode 100644 index f1853b13..00000000 --- a/proxyfsd/trackedlock.conf +++ /dev/null @@ -1,19 +0,0 @@ -[TrackedLock] - -# If non-zero check how long each lock was held when it is unlocked -# and log a warning message if it was held for more then the specified -# number of seconds. Set to "0s" to disable checking. -# -# If enabled, "40s" is the suggested value. -# -LockHoldTimeLimit: 0s - -# If non-zero, schedule a daemon, the trackedlock watcher, to -# periodically check each lock to see if it has been held for longer -# then LockHoldTimeLimit and, if so, log a warning message about the -# lock. Set to "0s" to disable the daemon (the daemon is also -# disabled if LockHoldTimeLimit is 0). -# -# If enabled, "20s" is the suggested value. -# -LockCheckPeriod: 0s diff --git a/ramswift/Makefile b/ramswift/Makefile deleted file mode 100644 index fa8948bd..00000000 --- a/ramswift/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/ramswift - -include ../GoMakefile diff --git a/ramswift/chaos_settings.conf b/ramswift/chaos_settings.conf deleted file mode 100644 index 04c8ab92..00000000 --- a/ramswift/chaos_settings.conf +++ /dev/null @@ -1,22 +0,0 @@ -# Chaos injection settings -# -# If FailureHTTPStatus is omitted, 500 (Internal Server Error ) is assumed -# If a ...FailureRate is omitted, 0 (meaning no errors injected) is assumed - -[RamSwiftChaos] -#FailureHTTPStatus: 500 -#AccountDeleteFailureRate: 0 -#AccountGetFailureRate: 0 -#AccountHeadFailureRate: 0 -#AccountPostFailureRate: 0 -#AccountPutFailureRate: 0 -#ContainerDeleteFailureRate: 0 -#ContainerGetFailureRate: 0 -#ContainerHeadFailureRate: 0 -#ContainerPostFailureRate: 0 -#ContainerPutFailureRate: 0 -#ObjectDeleteFailureRate: 0 -#ObjectGetFailureRate: 0 -#ObjectHeadFailureRate: 0 -#ObjectPostFailureRate: 0 -#ObjectPutFailureRate: 0 diff --git a/ramswift/daemon.go b/ramswift/daemon.go deleted file mode 100644 index 48108c6d..00000000 --- a/ramswift/daemon.go +++ /dev/null @@ -1,1667 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package ramswift provides an in-memory emulation of the Swift object storage -// API, which can be run as a goroutine from another package, or as a standalone -// binary. -package ramswift - -import ( - "fmt" - "io/ioutil" - "log" - "math/rand" - "net/http" - "os" - "os/signal" - "strconv" - "strings" - "sync" - "syscall" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/transitions" - "github.com/NVIDIA/proxyfs/utils" -) - -type swiftAccountStruct struct { - sync.Mutex // protects swiftAccountStruct.headers & swiftContainerTree - name string - headers http.Header - swiftContainerTree sortedmap.LLRBTree // key is swiftContainerStruct.name, value is *swiftContainerStruct -} - -type swiftAccountContext struct { - swiftAccount *swiftAccountStruct -} - -type swiftContainerStruct struct { - sync.Mutex // protects swiftContainerStruct.headers & swiftContainerStruct.swiftObjectTree - name string - swiftAccount *swiftAccountStruct // back-reference to swiftAccountStruct - headers http.Header - swiftObjectTree sortedmap.LLRBTree // key is swiftObjectStruct.name, value is *swiftObjectStruct -} - -type swiftContainerContext struct { - swiftContainer *swiftContainerStruct -} - -type swiftObjectStruct struct { - sync.Mutex // protects swiftObjectStruct.contents - name string - swiftContainer *swiftContainerStruct // back-reference to swiftContainerStruct - headers http.Header - contents []byte -} - -type methodStruct struct { - delete uint64 - get uint64 - head uint64 - post uint64 - put uint64 -} - -type globalsStruct struct { - sync.Mutex // protects globalsStruct.swiftAccountMap - whoAmI string - noAuthTCPPort uint16 - noAuthAddr string - noAuthHTTPServer *http.Server - noAuthHTTPServerWG sync.WaitGroup - swiftAccountMap map[string]*swiftAccountStruct // key is swiftAccountStruct.name, value is *swiftAccountStruct - accountMethodCount methodStruct - containerMethodCount methodStruct - objectMethodCount methodStruct - chaosFailureHTTPStatus int - accountMethodChaosFailureRate methodStruct // fail method if corresponding accountMethodCount divisible by accountMethodChaosFailureRate - containerMethodChaosFailureRate methodStruct // fail method if corresponding containerMethodCount divisible by containerMethodChaosFailureRate - objectMethodChaosFailureRate methodStruct // fail method if corresponding objectMethodCount divisible by objectMethodChaosFailureRate - maxAccountNameLength uint64 - maxContainerNameLength uint64 - maxObjectNameLength uint64 - accountListingLimit uint64 - containerListingLimit uint64 -} - -var globals *globalsStruct - -type httpRequestHandler struct{} - -type rangeStruct struct { - startOffset uint64 - stopOffset uint64 -} - -type stringSet map[string]bool - -var headerNameIgnoreSet = stringSet{"Accept": true, "Accept-Encoding": true, "User-Agent": true, "Content-Length": true} - -func (context *swiftAccountContext) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsString, ok := key.(string) - if ok { - err = nil - } else { - err = fmt.Errorf("swiftAccountContext.DumpKey() could not parse key as a string") - } - return -} - -func (context *swiftAccountContext) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - swiftContainer, ok := value.(*swiftContainerStruct) - if !ok { - err = fmt.Errorf("swiftAccountContext.DumpValue() could not parse key as a *swiftContainerStruct") - return - } - valueAsString = fmt.Sprintf("@%p: %#v", swiftContainer, swiftContainer) - err = nil - return -} - -func (context *swiftContainerContext) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsString, ok := key.(string) - if ok { - err = nil - } else { - err = fmt.Errorf("swiftContainerContext.DumpKey() could not parse key as a string") - } - return -} - -func (context *swiftContainerContext) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - swiftObject, ok := value.(*swiftObjectStruct) - if !ok { - err = fmt.Errorf("swiftContainerContext.DumpValue() could not parse key as a *swiftObjectStruct") - return - } - valueAsString = fmt.Sprintf("@%p: %#v", swiftObject, swiftObject) - err = nil - return -} - -func parsePath(request *http.Request) (infoOnly bool, swiftAccountName string, swiftContainerName string, swiftObjectName string) { - infoOnly = false - swiftAccountName = "" - swiftContainerName = "" - swiftObjectName = "" - - if "/info" == request.URL.Path { - infoOnly = true - return - } - - if strings.HasPrefix(request.URL.Path, "/v1/") { - pathSplit := strings.SplitN(request.URL.Path[4:], "/", 3) - swiftAccountName = pathSplit[0] - if 1 == len(pathSplit) { - swiftContainerName = "" - swiftObjectName = "" - } else { - swiftContainerName = pathSplit[1] - if 2 == len(pathSplit) { - swiftObjectName = "" - } else { - swiftObjectName = pathSplit[2] - } - } - } - - return -} - -// parseRangeHeader takes an HTTP Range header (e.g. bytes=100-200, bytes=10-20,30-40) and returns a slice of start and stop indices if any. -func parseRangeHeader(request *http.Request, objectLen int) (ranges []rangeStruct, err error) { - var ( - off int - rangeHeaderValueSuffix string - rangeString string - rangesStrings []string - rangesStringsIndex int - startOffset int64 - stopOffset int64 - ) - - rangeHeaderValue := request.Header.Get("Range") - if "" == rangeHeaderValue { - ranges = make([]rangeStruct, 0) - err = nil - return - } - - if !strings.HasPrefix(rangeHeaderValue, "bytes=") { - err = fmt.Errorf("rangeHeaderValue (%v) does not start with expected \"bytes=\"", rangeHeaderValue) - return - } - - rangeHeaderValueSuffix = rangeHeaderValue[len("bytes="):] - - rangesStrings = strings.SplitN(rangeHeaderValueSuffix, ",", 2) - - ranges = make([]rangeStruct, len(rangesStrings)) - - for rangesStringsIndex, rangeString = range rangesStrings { - rangeStringSlice := strings.SplitN(rangeString, "-", 2) - if 2 != len(rangeStringSlice) { - err = fmt.Errorf("rangeHeaderValue (%v) malformed", rangeHeaderValue) - return - } - if "" == rangeStringSlice[0] { - startOffset = int64(-1) - } else { - off, err = strconv.Atoi(rangeStringSlice[0]) - if nil != err { - err = fmt.Errorf("rangeHeaderValue (%v) malformed (strconv.Atoi() failure: %v)", rangeHeaderValue, err) - return - } - startOffset = int64(off) - } - - if "" == rangeStringSlice[1] { - stopOffset = int64(-1) - } else { - off, err = strconv.Atoi(rangeStringSlice[1]) - if nil != err { - err = fmt.Errorf("rangeHeaderValue (%v) malformed (strconv.Atoi() failure: %v)", rangeHeaderValue, err) - return - } - stopOffset = int64(off) - } - - if ((0 > startOffset) && (0 > stopOffset)) || (startOffset > stopOffset) { - err = fmt.Errorf("rangeHeaderValue (%v) malformed", rangeHeaderValue) - return - } - - if startOffset < 0 { - startOffset = int64(objectLen) - stopOffset - if startOffset < 0 { - err = fmt.Errorf("rangeHeaderValue (%v) malformed...computed startOffset negative", rangeHeaderValue) - return - } - stopOffset = int64(objectLen - 1) - } else if stopOffset < 0 { - stopOffset = int64(objectLen - 1) - } else { - if stopOffset > int64(objectLen-1) { - stopOffset = int64(objectLen - 1) - } - } - - ranges[rangesStringsIndex].startOffset = uint64(startOffset) - ranges[rangesStringsIndex].stopOffset = uint64(stopOffset) - } - - err = nil - return -} - -func locateSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, errno syscall.Errno) { - globals.Lock() - swiftAccount, ok := globals.swiftAccountMap[swiftAccountName] - if !ok { - globals.Unlock() - errno = unix.ENOENT - return - } - globals.Unlock() - errno = 0 - return -} - -func createSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, errno syscall.Errno) { - globals.Lock() - _, ok := globals.swiftAccountMap[swiftAccountName] - if ok { - globals.Unlock() - errno = unix.EEXIST - return - } - context := &swiftAccountContext{} - swiftAccount = &swiftAccountStruct{ - name: swiftAccountName, - headers: make(http.Header), - swiftContainerTree: sortedmap.NewLLRBTree(sortedmap.CompareString, context), - } - context.swiftAccount = swiftAccount - globals.swiftAccountMap[swiftAccountName] = swiftAccount - globals.Unlock() - errno = 0 - return -} - -func createOrLocateSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, wasCreated bool) { - globals.Lock() - swiftAccount, ok := globals.swiftAccountMap[swiftAccountName] - if ok { - wasCreated = false - } else { - context := &swiftAccountContext{} - swiftAccount = &swiftAccountStruct{ - name: swiftAccountName, - headers: make(http.Header), - swiftContainerTree: sortedmap.NewLLRBTree(sortedmap.CompareString, context), - } - context.swiftAccount = swiftAccount - globals.swiftAccountMap[swiftAccountName] = swiftAccount - wasCreated = true - } - globals.Unlock() - return -} - -func deleteSwiftAccount(swiftAccountName string, force bool) (errno syscall.Errno) { - globals.Lock() - swiftAccount, ok := globals.swiftAccountMap[swiftAccountName] - if ok { - if force { - // ok if account contains data... we'll forget it - } else { - swiftAccount.Lock() - swiftswiftAccountContainerCount, nonShadowingErr := swiftAccount.swiftContainerTree.Len() - if nil != nonShadowingErr { - panic(nonShadowingErr) - } - if 0 != swiftswiftAccountContainerCount { - swiftAccount.Unlock() - globals.Unlock() - errno = unix.ENOTEMPTY - return - } - swiftAccount.Unlock() - } - delete(globals.swiftAccountMap, swiftAccountName) - } else { - globals.Unlock() - errno = unix.ENOENT - return - } - globals.Unlock() - errno = 0 - return -} - -func locateSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, errno syscall.Errno) { - swiftAccount.Lock() - swiftContainerAsValue, ok, err := swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) - if nil != err { - panic(err) - } - if ok { - swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) - } else { - swiftAccount.Unlock() - errno = unix.ENOENT - return - } - swiftAccount.Unlock() - errno = 0 - return -} - -func createSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, errno syscall.Errno) { - swiftAccount.Lock() - _, ok, err := swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) - if nil != err { - panic(err) - } - if ok { - swiftAccount.Unlock() - errno = unix.EEXIST - return - } else { - context := &swiftContainerContext{} - swiftContainer = &swiftContainerStruct{ - name: swiftContainerName, - swiftAccount: swiftAccount, - headers: make(http.Header), - swiftObjectTree: sortedmap.NewLLRBTree(sortedmap.CompareString, context), - } - context.swiftContainer = swiftContainer - _, err = swiftAccount.swiftContainerTree.Put(swiftContainerName, swiftContainer) - if nil != err { - panic(err) - } - } - swiftAccount.Unlock() - errno = 0 - return -} - -func createOrLocateSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, wasCreated bool) { - swiftAccount.Lock() - swiftContainerAsValue, ok, err := swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) - if nil != err { - panic(err) - } - if ok { - swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) - wasCreated = false - } else { - context := &swiftContainerContext{} - swiftContainer = &swiftContainerStruct{ - name: swiftContainerName, - swiftAccount: swiftAccount, - headers: make(http.Header), - swiftObjectTree: sortedmap.NewLLRBTree(sortedmap.CompareString, context), - } - context.swiftContainer = swiftContainer - _, err = swiftAccount.swiftContainerTree.Put(swiftContainerName, swiftContainer) - if nil != err { - panic(err) - } - wasCreated = true - } - swiftAccount.Unlock() - return -} - -func deleteSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (errno syscall.Errno) { - swiftAccount.Lock() - swiftContainerAsValue, ok, err := swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) - if nil != err { - panic(err) - } - if ok { - swiftContainer := swiftContainerAsValue.(*swiftContainerStruct) - swiftContainer.Lock() - swiftContainerObjectCount, nonShadowingErr := swiftContainer.swiftObjectTree.Len() - if nil != nonShadowingErr { - panic(nonShadowingErr) - } - if 0 != swiftContainerObjectCount { - swiftContainer.Unlock() - swiftAccount.Unlock() - errno = unix.ENOTEMPTY - return - } - swiftContainer.Unlock() - _, err = swiftAccount.swiftContainerTree.DeleteByKey(swiftContainerName) - if nil != err { - panic(err) - } - } else { - swiftAccount.Unlock() - errno = unix.ENOENT - return - } - swiftAccount.Unlock() - errno = 0 - return -} - -func locateSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, errno syscall.Errno) { - swiftContainer.Lock() - swiftObjectAsValue, ok, err := swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) - if nil != err { - panic(err) - } - if ok { - swiftObject = swiftObjectAsValue.(*swiftObjectStruct) - } else { - swiftContainer.Unlock() - errno = unix.ENOENT - return - } - swiftContainer.Unlock() - errno = 0 - return -} - -func createSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, errno syscall.Errno) { - swiftContainer.Lock() - _, ok, err := swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) - if nil != err { - panic(err) - } - if ok { - swiftContainer.Unlock() - errno = unix.EEXIST - return - } else { - swiftObject = &swiftObjectStruct{name: swiftObjectName, swiftContainer: swiftContainer, contents: []byte{}} - _, err = swiftContainer.swiftObjectTree.Put(swiftObjectName, swiftObject) - if nil != err { - panic(err) - } - } - swiftContainer.Unlock() - errno = 0 - return -} - -func createOrLocateSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, wasCreated bool) { - swiftContainer.Lock() - swiftObjectAsValue, ok, err := swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) - if nil != err { - panic(err) - } - if ok { - swiftObject = swiftObjectAsValue.(*swiftObjectStruct) - wasCreated = false - } else { - swiftObject = &swiftObjectStruct{name: swiftObjectName, swiftContainer: swiftContainer, contents: []byte{}} - _, err = swiftContainer.swiftObjectTree.Put(swiftObjectName, swiftObject) - if nil != err { - panic(err) - } - wasCreated = true - } - swiftContainer.Unlock() - return -} - -func deleteSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (errno syscall.Errno) { - swiftContainer.Lock() - _, ok, err := swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) - if nil != err { - panic(err) - } - if ok { - _, err = swiftContainer.swiftObjectTree.DeleteByKey(swiftObjectName) - if nil != err { - panic(err) - } - } else { - swiftContainer.Unlock() - errno = unix.ENOENT - return - } - swiftContainer.Unlock() - errno = 0 - return -} - -func doDelete(responseWriter http.ResponseWriter, request *http.Request) { - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName := parsePath(request) - if infoOnly || ("" == swiftAccountName) { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - if "" == swiftContainerName { - // DELETE SwiftAccount - globals.Lock() - globals.accountMethodCount.delete++ - if (0 != globals.accountMethodChaosFailureRate.delete) && (0 == globals.accountMethodCount.delete%globals.accountMethodChaosFailureRate.delete) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - errno := deleteSwiftAccount(swiftAccountName, false) - switch errno { - case 0: - responseWriter.WriteHeader(http.StatusNoContent) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - case unix.ENOTEMPTY: - responseWriter.WriteHeader(http.StatusConflict) - default: - err := fmt.Errorf("deleteSwiftAccount(\"%v\", false) returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } - } else { - // DELETE SwiftContainer or SwiftObject - swiftAccount, errno := locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftObjectName { - // DELETE SwiftContainer - globals.Lock() - globals.containerMethodCount.delete++ - if (0 != globals.containerMethodChaosFailureRate.delete) && (0 == globals.containerMethodCount.delete%globals.containerMethodChaosFailureRate.delete) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - errno := deleteSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - responseWriter.WriteHeader(http.StatusNoContent) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - case unix.ENOTEMPTY: - responseWriter.WriteHeader(http.StatusConflict) - default: - err := fmt.Errorf("deleteSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - } else { - // DELETE SwiftObject - globals.Lock() - globals.objectMethodCount.delete++ - if (0 != globals.objectMethodChaosFailureRate.delete) && (0 == globals.objectMethodCount.delete%globals.objectMethodChaosFailureRate.delete) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftContainer, errno := locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - errno := deleteSwiftObject(swiftContainer, swiftObjectName) - switch errno { - case 0: - responseWriter.WriteHeader(http.StatusNoContent) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("deleteSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) - panic(err) - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } - } -} - -func doGet(responseWriter http.ResponseWriter, request *http.Request) { - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName := parsePath(request) - if infoOnly { - _, _ = responseWriter.Write(utils.StringToByteSlice("{")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"swift\": {")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_account_name_length\": " + strconv.Itoa(int(globals.maxAccountNameLength)) + ",")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_container_name_length\": " + strconv.Itoa(int(globals.maxContainerNameLength)) + ",")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_object_name_length\": " + strconv.Itoa(int(globals.maxObjectNameLength)) + ",")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"account_listing_limit\": " + strconv.Itoa(int(globals.accountListingLimit)) + ",")) - _, _ = responseWriter.Write(utils.StringToByteSlice("\"container_listing_limit\": " + strconv.Itoa(int(globals.containerListingLimit)))) - _, _ = responseWriter.Write(utils.StringToByteSlice("}")) - _, _ = responseWriter.Write(utils.StringToByteSlice("}")) - } else { - if "" == swiftAccountName { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - swiftAccount, errno := locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftContainerName { - // GET SwiftAccount - globals.Lock() - globals.accountMethodCount.get++ - if (0 != globals.accountMethodChaosFailureRate.get) && (0 == globals.accountMethodCount.get%globals.accountMethodChaosFailureRate.get) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftAccount.Lock() - for headerName, headerValueSlice := range swiftAccount.headers { - for _, headerValue := range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - numContainers, err := swiftAccount.swiftContainerTree.Len() - if nil != err { - panic(err) - } - if 0 == numContainers { - responseWriter.WriteHeader(http.StatusNoContent) - } else { - marker := "" - markerSlice, ok := request.URL.Query()["marker"] - if ok && (0 < len(markerSlice)) { - marker = markerSlice[0] - } - containerIndex, found, err := swiftAccount.swiftContainerTree.BisectRight(marker) - if nil != err { - panic(err) - } - if found { - containerIndex++ - } - if containerIndex < numContainers { - containerIndexLimit := numContainers - if (containerIndexLimit - containerIndex) > int(globals.accountListingLimit) { - containerIndexLimit = containerIndex + int(globals.accountListingLimit) - } - for containerIndex < containerIndexLimit { - swiftContainerNameAsKey, _, _, err := swiftAccount.swiftContainerTree.GetByIndex(containerIndex) - if nil != err { - panic(err) - } - swiftContainerName := swiftContainerNameAsKey.(string) - _, _ = responseWriter.Write(utils.StringToByteSlice(swiftContainerName)) - _, _ = responseWriter.Write([]byte{'\n'}) - containerIndex++ - } - } else { - responseWriter.WriteHeader(http.StatusNoContent) - } - } - swiftAccount.Unlock() - } - } else { - // GET SwiftContainer or SwiftObject - swiftContainer, errno := locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - if "" == swiftObjectName { - // GET SwiftContainer - globals.Lock() - globals.containerMethodCount.get++ - if (0 != globals.containerMethodChaosFailureRate.get) && (0 == globals.containerMethodCount.get%globals.containerMethodChaosFailureRate.get) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftContainer.Lock() - for headerName, headerValueSlice := range swiftContainer.headers { - for _, headerValue := range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - numObjects, err := swiftContainer.swiftObjectTree.Len() - if nil != err { - panic(err) - } - if 0 == numObjects { - responseWriter.WriteHeader(http.StatusNoContent) - } else { - marker := "" - markerSlice, ok := request.URL.Query()["marker"] - if ok && (0 < len(markerSlice)) { - marker = markerSlice[0] - } - objectIndex, found, err := swiftContainer.swiftObjectTree.BisectRight(marker) - if nil != err { - panic(err) - } - if found { - objectIndex++ - } - if objectIndex < numObjects { - objectIndexLimit := numObjects - if (objectIndexLimit - objectIndex) > int(globals.containerListingLimit) { - objectIndexLimit = objectIndex + int(globals.containerListingLimit) - } - for objectIndex < objectIndexLimit { - swiftObjectNameAsKey, _, _, err := swiftContainer.swiftObjectTree.GetByIndex(objectIndex) - if nil != err { - panic(err) - } - swiftObjectName := swiftObjectNameAsKey.(string) - _, _ = responseWriter.Write(utils.StringToByteSlice(swiftObjectName)) - _, _ = responseWriter.Write([]byte{'\n'}) - objectIndex++ - } - } else { - responseWriter.WriteHeader(http.StatusNoContent) - } - } - swiftContainer.Unlock() - } - } else { - // GET SwiftObject - globals.Lock() - globals.objectMethodCount.get++ - if (0 != globals.objectMethodChaosFailureRate.get) && (0 == globals.objectMethodCount.get%globals.objectMethodChaosFailureRate.get) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftObject, errno := locateSwiftObject(swiftContainer, swiftObjectName) - switch errno { - case 0: - swiftObject.Lock() - for headerName, headerValueSlice := range swiftObject.headers { - for _, headerValue := range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - ranges, err := parseRangeHeader(request, len(swiftObject.contents)) - if nil == err { - switch len(ranges) { - case 0: - responseWriter.Header().Add("Content-Type", "application/octet-stream") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(swiftObject.contents) - case 1: - responseWriter.Header().Add("Content-Type", "application/octet-stream") - responseWriter.Header().Add("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ranges[0].startOffset, ranges[0].stopOffset, len(swiftObject.contents))) - responseWriter.WriteHeader(http.StatusPartialContent) - _, _ = responseWriter.Write(swiftObject.contents[ranges[0].startOffset:(ranges[0].stopOffset + 1)]) - default: - boundaryString := fmt.Sprintf("%016x%016x", rand.Uint64(), rand.Uint64()) - responseWriter.Header().Add("Content-Type", fmt.Sprintf("multipart/byteranges; boundary=%v", boundaryString)) - responseWriter.WriteHeader(http.StatusPartialContent) - for _, rS := range ranges { - _, _ = responseWriter.Write([]byte("--" + boundaryString + "\r\n")) - _, _ = responseWriter.Write([]byte("Content-Type: application/octet-stream\r\n")) - _, _ = responseWriter.Write([]byte(fmt.Sprintf("Content-Range: bytes %d-%d/%d\r\n", rS.startOffset, rS.stopOffset, len(swiftObject.contents)))) - _, _ = responseWriter.Write([]byte("\r\n")) - _, _ = responseWriter.Write(swiftObject.contents[rS.startOffset:(rS.stopOffset + 1)]) - _, _ = responseWriter.Write([]byte("\r\n")) - } - _, _ = responseWriter.Write([]byte("--" + boundaryString + "--")) - } - } else { - responseWriter.WriteHeader(http.StatusBadRequest) - } - swiftObject.Unlock() - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) - panic(err) - } - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } - } -} - -func doHead(responseWriter http.ResponseWriter, request *http.Request) { - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName := parsePath(request) - if infoOnly || ("" == swiftAccountName) { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - swiftAccount, errno := locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftContainerName { - // HEAD SwiftAccount - globals.Lock() - globals.accountMethodCount.head++ - if (0 != globals.accountMethodChaosFailureRate.head) && (0 == globals.accountMethodCount.head%globals.accountMethodChaosFailureRate.head) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftAccount.Lock() - for headerName, headerValueSlice := range swiftAccount.headers { - for _, headerValue := range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - swiftAccount.Unlock() - responseWriter.WriteHeader(http.StatusNoContent) - } - } else { - // HEAD SwiftContainer or SwiftObject - swiftContainer, errno := locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - if "" == swiftObjectName { - // HEAD SwiftContainer - globals.Lock() - globals.containerMethodCount.head++ - if (0 != globals.containerMethodChaosFailureRate.head) && (0 == globals.containerMethodCount.head%globals.containerMethodChaosFailureRate.head) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftContainer.Lock() - for headerName, headerValueSlice := range swiftContainer.headers { - for _, headerValue := range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - swiftContainer.Unlock() - responseWriter.WriteHeader(http.StatusNoContent) - } - } else { - // HEAD SwiftObject - globals.Lock() - globals.objectMethodCount.head++ - if (0 != globals.objectMethodChaosFailureRate.head) && (0 == globals.objectMethodCount.head%globals.objectMethodChaosFailureRate.head) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftObject, errno := locateSwiftObject(swiftContainer, swiftObjectName) - switch errno { - case 0: - swiftObject.Lock() - for headerName, headerValueSlice := range swiftObject.headers { - for _, headerValue := range headerValueSlice { - responseWriter.Header().Add(headerName, headerValue) - } - } - responseWriter.Header().Set("Content-Length", strconv.Itoa(len(swiftObject.contents))) - responseWriter.WriteHeader(http.StatusOK) - swiftObject.Unlock() - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) - panic(err) - } - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } -} - -func doPost(responseWriter http.ResponseWriter, request *http.Request) { - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName := parsePath(request) - if infoOnly || ("" == swiftAccountName) { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - swiftAccount, errno := locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftContainerName { - // POST SwiftAccount - globals.Lock() - globals.accountMethodCount.post++ - if (0 != globals.accountMethodChaosFailureRate.post) && (0 == globals.accountMethodCount.post%globals.accountMethodChaosFailureRate.post) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftAccount.Lock() - for headerName, headerValueSlice := range request.Header { - _, ignoreHeader := headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen := len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftAccount.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue := range headerValueSlice { - if 0 < len(headerValue) { - swiftAccount.headers[headerName] = append(swiftAccount.headers[headerName], headerValue) - } - } - if 0 == len(swiftAccount.headers[headerName]) { - delete(swiftAccount.headers, headerName) - } - } - } - } - swiftAccount.Unlock() - responseWriter.WriteHeader(http.StatusNoContent) - } - } else { - // POST SwiftContainer or SwiftObject - swiftContainer, errno := locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - if "" == swiftObjectName { - // POST SwiftContainer - globals.Lock() - globals.containerMethodCount.post++ - if (0 != globals.containerMethodChaosFailureRate.post) && (0 == globals.containerMethodCount.post%globals.containerMethodChaosFailureRate.post) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftContainer.Lock() - for headerName, headerValueSlice := range request.Header { - _, ignoreHeader := headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen := len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftContainer.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue := range headerValueSlice { - if 0 < len(headerValue) { - swiftContainer.headers[headerName] = append(swiftContainer.headers[headerName], headerValue) - } - } - if 0 == len(swiftContainer.headers[headerName]) { - delete(swiftContainer.headers, headerName) - } - } - } - } - swiftContainer.Unlock() - responseWriter.WriteHeader(http.StatusNoContent) - } - } else { - // POST SwiftObject - globals.Lock() - globals.objectMethodCount.head++ - if (0 != globals.objectMethodChaosFailureRate.post) && (0 == globals.objectMethodCount.head%globals.objectMethodChaosFailureRate.post) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftObject, errno := locateSwiftObject(swiftContainer, swiftObjectName) - switch errno { - case 0: - swiftObject.Lock() - for headerName, headerValueSlice := range request.Header { - _, ignoreHeader := headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen := len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftObject.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue := range headerValueSlice { - if 0 < len(headerValue) { - swiftObject.headers[headerName] = append(swiftObject.headers[headerName], headerValue) - } - } - if 0 == len(swiftObject.headers[headerName]) { - delete(swiftObject.headers, headerName) - } - } - } - } - swiftObject.Unlock() - responseWriter.WriteHeader(http.StatusNoContent) - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) - panic(err) - } - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusNotFound) - default: - err := fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } -} - -func doPut(responseWriter http.ResponseWriter, request *http.Request) { - infoOnly, swiftAccountName, swiftContainerName, swiftObjectName := parsePath(request) - if infoOnly || ("" == swiftAccountName) { - responseWriter.WriteHeader(http.StatusForbidden) - } else { - if "" == swiftContainerName { - // PUT SwiftAccount - globals.Lock() - globals.accountMethodCount.put++ - if (0 != globals.accountMethodChaosFailureRate.put) && (0 == globals.accountMethodCount.put%globals.accountMethodChaosFailureRate.put) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftAccount, wasCreated := createOrLocateSwiftAccount(swiftAccountName) - swiftAccount.Lock() - if wasCreated { - swiftAccount.headers = make(http.Header) - } - for headerName, headerValueSlice := range request.Header { - _, ignoreHeader := headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen := len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftAccount.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue := range headerValueSlice { - if 0 < len(headerValue) { - swiftAccount.headers[headerName] = append(swiftAccount.headers[headerName], headerValue) - } - } - if 0 == len(swiftAccount.headers[headerName]) { - delete(swiftAccount.headers, headerName) - } - } - } - } - swiftAccount.Unlock() - if wasCreated { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusAccepted) - } - } - } else { - // PUT SwiftContainer or SwiftObject - swiftAccount, errno := locateSwiftAccount(swiftAccountName) - switch errno { - case 0: - if "" == swiftObjectName { - // PUT SwiftContainer - globals.Lock() - globals.containerMethodCount.put++ - if (0 != globals.containerMethodChaosFailureRate.put) && (0 == globals.containerMethodCount.put%globals.containerMethodChaosFailureRate.put) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - } else { - globals.Unlock() - swiftContainer, wasCreated := createOrLocateSwiftContainer(swiftAccount, swiftContainerName) - swiftContainer.Lock() - if wasCreated { - swiftContainer.headers = make(http.Header) - } - for headerName, headerValueSlice := range request.Header { - _, ignoreHeader := headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen := len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftContainer.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue := range headerValueSlice { - if 0 < len(headerValue) { - swiftContainer.headers[headerName] = append(swiftContainer.headers[headerName], headerValue) - } - } - if 0 == len(swiftContainer.headers[headerName]) { - delete(swiftContainer.headers, headerName) - } - } - } - } - swiftContainer.Unlock() - if wasCreated { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusAccepted) - } - } - } else { - // PUT SwiftObject - globals.Lock() - globals.objectMethodCount.put++ - if (0 != globals.objectMethodChaosFailureRate.put) && (0 == globals.objectMethodCount.put%globals.objectMethodChaosFailureRate.put) { - globals.Unlock() - responseWriter.WriteHeader(globals.chaosFailureHTTPStatus) - - // consume the PUT so the sender can finish sending - // and read the response - _, _ = ioutil.ReadAll(request.Body) - } else { - globals.Unlock() - swiftContainer, errno := locateSwiftContainer(swiftAccount, swiftContainerName) - switch errno { - case 0: - swiftObject, wasCreated := createOrLocateSwiftObject(swiftContainer, swiftObjectName) - swiftObject.Lock() - if wasCreated { - swiftObject.headers = make(http.Header) - } - for headerName, headerValueSlice := range request.Header { - _, ignoreHeader := headerNameIgnoreSet[headerName] - if !ignoreHeader { - headerValueSliceLen := len(headerValueSlice) - if 0 < headerValueSliceLen { - swiftObject.headers[headerName] = make([]string, 0, headerValueSliceLen) - for _, headerValue := range headerValueSlice { - if 0 < len(headerValue) { - swiftObject.headers[headerName] = append(swiftObject.headers[headerName], headerValue) - } - } - if 0 == len(swiftObject.headers[headerName]) { - delete(swiftObject.headers, headerName) - } - } - } - } - swiftObject.contents, _ = ioutil.ReadAll(request.Body) - swiftObject.Unlock() - if wasCreated { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusCreated) - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusForbidden) - default: - err := fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) - panic(err) - } - } - } - case unix.ENOENT: - responseWriter.WriteHeader(http.StatusForbidden) - default: - err := fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) - panic(err) - } - } - } -} - -func (h httpRequestHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - switch request.Method { - case http.MethodDelete: - doDelete(responseWriter, request) - case http.MethodGet: - doGet(responseWriter, request) - case http.MethodHead: - doHead(responseWriter, request) - case http.MethodPost: - doPost(responseWriter, request) - case http.MethodPut: - doPut(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func setupNoAuthSwift(confMap conf.ConfMap) { - var ( - err error - errno syscall.Errno - primaryPeerList []string - swiftAccountName string - volumeGroupNameList []string - volumeGroupName string - volumeGroupSectionName string - volumeName string - volumeNameList []string - volumeSectionName string - ) - - // Fetch and configure volumes for which "we" are the PrimaryPeer - - volumeGroupNameList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - log.Fatalf("failed fetch of FSGlobals.VolumeGroupList: %v", err) - } - - for _, volumeGroupName = range volumeGroupNameList { - volumeGroupSectionName = "VolumeGroup:" + volumeGroupName - - primaryPeerList, err = confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "PrimaryPeer") - if nil != err { - log.Fatalf("failed fetch of %v.PrimaryPeer: %v", volumeGroupSectionName, err) - } - if 0 == len(primaryPeerList) { - continue - } else if 1 == len(primaryPeerList) { - if globals.whoAmI != primaryPeerList[0] { - continue - } - } else { - log.Fatalf("fetch of %v.PrimaryPeer returned multiple values", volumeGroupSectionName) - } - - volumeNameList, err = confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "VolumeList") - if nil != err { - log.Fatalf("failed fetch of %v.VolumeList: %v", volumeGroupSectionName, err) - } - - for _, volumeName = range volumeNameList { - volumeSectionName = "Volume:" + volumeName - - swiftAccountName, err = confMap.FetchOptionValueString(volumeSectionName, "AccountName") - if nil != err { - log.Fatalf("failed fetch of %v.AccountName: %v", volumeSectionName, err) - } - - _, errno = createSwiftAccount(swiftAccountName) - if 0 != errno { - log.Fatalf("failed create of %v: %v", swiftAccountName, err) - } - } - } - - // Fetch chaos settings - - fetchChaosSettings(confMap) - - // Fetch responses for GETs on /info - - fetchSwiftInfo(confMap) - - // Create HTTP Server on the requested noAuthTCPPort - - globals.noAuthHTTPServer = &http.Server{ - Addr: globals.noAuthAddr, - Handler: httpRequestHandler{}, - } -} - -func serveNoAuthSwift() { - _ = globals.noAuthHTTPServer.ListenAndServe() - - globals.noAuthHTTPServerWG.Done() -} - -func updateConf(confMap conf.ConfMap) { - var ( - err error - noAuthTCPPortUpdate uint16 - ok bool - primaryPeerList []string - swiftAccountNameListCurrent []string // element == swiftAccountName - swiftAccountNameListUpdate map[string]bool // key == swiftAccountName; value is ignored - swiftAccountName string - volumeGroupName string - volumeGroupNameList []string - volumeGroupSectionName string - volumeListUpdate []string - volumeName string - volumeNameList []string - volumeSectionName string - whoAmIUpdate string - ) - - // First validate "we" didn't change - - whoAmIUpdate, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - log.Fatalf("failed fetch of Cluster.WhoAmI: %v", err) - } - if whoAmIUpdate != globals.whoAmI { - log.Fatal("update of whoAmI not allowed") - } - - noAuthTCPPortUpdate, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") - if nil != err { - log.Fatalf("failed fetch of Swift.NoAuthTCPPort: %v", err) - } - if noAuthTCPPortUpdate != globals.noAuthTCPPort { - log.Fatal("update of noAuthTCPPort not allowed") - } - - // Compute current list of accounts being served - - swiftAccountNameListCurrent = make([]string, 0) - - globals.Lock() - - for swiftAccountName = range globals.swiftAccountMap { - swiftAccountNameListCurrent = append(swiftAccountNameListCurrent, swiftAccountName) - } - - globals.Unlock() - - // Fetch list of accounts for which "we" are the PrimaryPeer - - volumeListUpdate = make([]string, 0) - - volumeGroupNameList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - log.Fatalf("failed fetch of FSGlobals.VolumeGroupList: %v", err) - } - - for _, volumeGroupName = range volumeGroupNameList { - volumeGroupSectionName = "VolumeGroup:" + volumeGroupName - - primaryPeerList, err = confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "PrimaryPeer") - if nil != err { - log.Fatalf("failed fetch of %v.PrimaryPeer: %v", volumeGroupSectionName, err) - } - if 0 == len(primaryPeerList) { - continue - } else if 1 == len(primaryPeerList) { - if globals.whoAmI != primaryPeerList[0] { - continue - } - } else { - log.Fatalf("fetch of %v.PrimaryPeer returned multiple values", volumeGroupSectionName) - } - - volumeNameList, err = confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "VolumeList") - if nil != err { - log.Fatalf("failed fetch of %v.VolumeList: %v", volumeGroupSectionName, err) - } - - volumeListUpdate = append(volumeListUpdate, volumeNameList...) - } - - swiftAccountNameListUpdate = make(map[string]bool) - - for _, volumeName = range volumeListUpdate { - volumeSectionName = "Volume:" + volumeName - swiftAccountName, err = confMap.FetchOptionValueString(volumeSectionName, "AccountName") - if nil != err { - log.Fatalf("failed fetch of %v.AccountName: %v", volumeSectionName, err) - } - swiftAccountNameListUpdate[swiftAccountName] = true - } - - // Delete accounts not found in accountListUpdate - - for _, swiftAccountName = range swiftAccountNameListCurrent { - _, ok = swiftAccountNameListUpdate[swiftAccountName] - if !ok { - _ = deleteSwiftAccount(swiftAccountName, true) - } - } - - // Add accounts in accountListUpdate not found in globals.swiftAccountMap - - for swiftAccountName = range swiftAccountNameListUpdate { - _, _ = createOrLocateSwiftAccount(swiftAccountName) - } - - // Fetch (potentially updated) chaos settings - - fetchChaosSettings(confMap) - - // Fetch (potentially updated) responses for GETs on /info - - fetchSwiftInfo(confMap) -} - -func fetchChaosSettings(confMap conf.ConfMap) { - var ( - chaosFailureHTTPStatus uint16 - err error - ) - - chaosFailureHTTPStatus, err = confMap.FetchOptionValueUint16("RamSwiftChaos", "FailureHTTPStatus") - if nil == err { - globals.chaosFailureHTTPStatus = int(chaosFailureHTTPStatus) - } else { - globals.chaosFailureHTTPStatus = http.StatusInternalServerError - } - - globals.accountMethodChaosFailureRate.delete, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "AccountDeleteFailureRate") - if nil != err { - globals.accountMethodChaosFailureRate.delete = 0 - } - globals.accountMethodChaosFailureRate.get, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "AccountGetFailureRate") - if nil != err { - globals.accountMethodChaosFailureRate.get = 0 - } - globals.accountMethodChaosFailureRate.head, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "AccountHeadFailureRate") - if nil != err { - globals.accountMethodChaosFailureRate.head = 0 - } - globals.accountMethodChaosFailureRate.post, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "AccountPostFailureRate") - if nil != err { - globals.accountMethodChaosFailureRate.post = 0 - } - globals.accountMethodChaosFailureRate.put, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "AccountPutFailureRate") - if nil != err { - globals.accountMethodChaosFailureRate.put = 0 - } - - globals.containerMethodChaosFailureRate.delete, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ContainerDeleteFailureRate") - if nil != err { - globals.containerMethodChaosFailureRate.delete = 0 - } - globals.containerMethodChaosFailureRate.get, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ContainerGetFailureRate") - if nil != err { - globals.containerMethodChaosFailureRate.get = 0 - } - globals.containerMethodChaosFailureRate.head, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ContainerHeadFailureRate") - if nil != err { - globals.containerMethodChaosFailureRate.head = 0 - } - globals.containerMethodChaosFailureRate.post, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ContainerPostFailureRate") - if nil != err { - globals.containerMethodChaosFailureRate.post = 0 - } - globals.containerMethodChaosFailureRate.put, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ContainerPutFailureRate") - if nil != err { - globals.containerMethodChaosFailureRate.put = 0 - } - - globals.objectMethodChaosFailureRate.delete, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ObjectDeleteFailureRate") - if nil != err { - globals.objectMethodChaosFailureRate.delete = 0 - } - globals.objectMethodChaosFailureRate.get, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ObjectGetFailureRate") - if nil != err { - globals.objectMethodChaosFailureRate.get = 0 - } - globals.objectMethodChaosFailureRate.head, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ObjectHeadFailureRate") - if nil != err { - globals.objectMethodChaosFailureRate.head = 0 - } - globals.objectMethodChaosFailureRate.post, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ObjectPostFailureRate") - if nil != err { - globals.objectMethodChaosFailureRate.post = 0 - } - globals.objectMethodChaosFailureRate.put, err = confMap.FetchOptionValueUint64("RamSwiftChaos", "ObjectPutFailureRate") - if nil != err { - globals.objectMethodChaosFailureRate.put = 0 - } -} - -func fetchSwiftInfo(confMap conf.ConfMap) { - var ( - err error - ) - - maxIntAsUint64 := uint64(^uint(0) >> 1) - - globals.maxAccountNameLength, err = confMap.FetchOptionValueUint64("RamSwiftInfo", "MaxAccountNameLength") - if nil != err { - log.Fatalf("failed fetch of RamSwiftInfo.MaxAccountNameLength: %v", err) - } - if globals.maxAccountNameLength > maxIntAsUint64 { - log.Fatal("RamSwiftInfo.MaxAccountNameLength too large... must fit in a Go int") - } - globals.maxContainerNameLength, err = confMap.FetchOptionValueUint64("RamSwiftInfo", "MaxContainerNameLength") - if nil != err { - log.Fatalf("failed fetch of RamSwiftInfo.MaxContainerNameLength: %v", err) - } - if globals.maxContainerNameLength > maxIntAsUint64 { - log.Fatal("RamSwiftInfo.MaxContainerNameLength too large... must fit in a Go int") - } - globals.maxObjectNameLength, err = confMap.FetchOptionValueUint64("RamSwiftInfo", "MaxObjectNameLength") - if nil != err { - log.Fatalf("failed fetch of RamSwiftInfo.MaxObjectNameLength: %v", err) - } - if globals.maxObjectNameLength > maxIntAsUint64 { - log.Fatal("RamSwiftInfo.MaxObjectNameLength too large... must fit in a Go int") - } - globals.accountListingLimit, err = confMap.FetchOptionValueUint64("RamSwiftInfo", "AccountListingLimit") - if nil != err { - log.Fatalf("failed fetch of RamSwiftInfo.AccountListingLimit: %v", err) - } - if globals.accountListingLimit > maxIntAsUint64 { - log.Fatal("RamSwiftInfo.AccountListingLimit too large... must fit in a Go int") - } - globals.containerListingLimit, err = confMap.FetchOptionValueUint64("RamSwiftInfo", "ContainerListingLimit") - if nil != err { - log.Fatalf("failed fetch of RamSwiftInfo.ContainerListingLimit: %v", err) - } - if globals.containerListingLimit > maxIntAsUint64 { - log.Fatal("RamSwiftInfo.ContainerListingLimit too large... must fit in a Go int") - } -} - -func Daemon(confFile string, confStrings []string, signalHandlerIsArmedWG *sync.WaitGroup, doneChan chan bool, signals ...os.Signal) { - var ( - confMap conf.ConfMap - err error - resp *http.Response - signalChan chan os.Signal - signalReceived os.Signal - ) - - // Initialization - - globals = &globalsStruct{} - - globals.swiftAccountMap = make(map[string]*swiftAccountStruct) - - // Compute confMap - - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - log.Fatalf("failed to load config: %v", err) - } - - err = confMap.UpdateFromStrings(confStrings) - if nil != err { - log.Fatalf("failed to apply config overrides: %v", err) - } - - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - log.Fatalf("failed to upgrade confMap: %v", err) - } - - // Find out who "we" are - - globals.whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - log.Fatalf("failed fetch of Cluster.WhoAmI: %v", err) - } - - globals.noAuthTCPPort, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") - if nil != err { - log.Fatalf("failed fetch of Swift.NoAuthTCPPort: %v", err) - } - - globals.noAuthAddr = "127.0.0.1:" + strconv.Itoa(int(globals.noAuthTCPPort)) - - // Kick off NoAuth Swift Proxy Emulator - - setupNoAuthSwift(confMap) - globals.noAuthHTTPServerWG.Add(1) - go serveNoAuthSwift() - - // Wait for serveNoAuthSwift() to begin serving - - for { - resp, err = http.Get("http://" + globals.noAuthAddr + "/info") - if nil != err { - log.Printf("failed GET of \"/info\": %v", err) - continue - } - if http.StatusOK == resp.StatusCode { - break - } - log.Printf("GET of \"/info\" returned Status %v", resp.Status) - time.Sleep(100 * time.Millisecond) - } - - // Arm signal handler used to indicate termination and wait on it - // - // Note: signalled chan must be buffered to avoid race with window between - // arming handler and blocking on the chan read - - signalChan = make(chan os.Signal, 1) - - signal.Notify(signalChan, signals...) - - if nil != signalHandlerIsArmedWG { - signalHandlerIsArmedWG.Done() - } - - // Await a signal - reloading confFile each SIGHUP - exiting otherwise - - for { - signalReceived = <-signalChan - - if unix.SIGHUP == signalReceived { - // recompute confMap and re-apply - - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - log.Fatalf("failed to load updated config: %v", err) - } - - err = confMap.UpdateFromStrings(confStrings) - if nil != err { - log.Fatalf("failed to reapply config overrides: %v", err) - } - - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - log.Fatalf("failed to upgrade confMap: %v", err) - } - - updateConf(confMap) - } else { - // signalReceived either SIGINT or SIGTERM... so just exit - - err = globals.noAuthHTTPServer.Close() - - globals.noAuthHTTPServerWG.Wait() - - globals = nil - - doneChan <- true - - return - } - } -} diff --git a/ramswift/daemon_test.go b/ramswift/daemon_test.go deleted file mode 100644 index dcfff25c..00000000 --- a/ramswift/daemon_test.go +++ /dev/null @@ -1,1274 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package ramswift - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "strings" - "sync" - "testing" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/utils" -) - -func TestViaNoAuthClient(t *testing.T) { - const ( - noAuthTCPPort = "9999" - ) - var ( - confStrings = []string{ - "SwiftClient.NoAuthTCPPort=" + noAuthTCPPort, - "SwiftClient.NoAuthIPAddr=127.0.0.1", - - "TrackedLock.LockHoldTimeLimit=0s", - "TrackedLock.LockCheckPeriod=0s", - - "Cluster.WhoAmI=Peer0", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - } - contentType string - contentTypeMultiPartBoundary string - doneChan chan bool - err error - errChan chan error - expectedBuf []byte - expectedInfo string - httpClient *http.Client - httpRequest *http.Request - httpResponse *http.Response - mouseHeaderPresent bool - pipeReader *io.PipeReader - pipeWriter *io.PipeWriter - readBuf []byte - signalHandlerIsArmedWG sync.WaitGroup - urlForInfo string - urlPrefix string - ) - - signalHandlerIsArmedWG.Add(1) - doneChan = make(chan bool, 1) // Must be buffered to avoid race - - go Daemon("/dev/null", confStrings, &signalHandlerIsArmedWG, doneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - // Setup urlPrefix to be "http://127.0.0.1:/v1/" - - urlForInfo = "http://127.0.0.1:" + noAuthTCPPort + "/info" - urlPrefix = "http://127.0.0.1:" + noAuthTCPPort + "/v1/" - - // Setup http.Client that we will use for all HTTP requests - - httpClient = &http.Client{} - - // Send a GET for "/info" expecting [RamSwiftInfo] data in compact JSON form - - httpRequest, err = http.NewRequest("GET", urlForInfo, nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - expectedInfo = "{\"swift\": {\"max_account_name_length\": 256,\"max_container_name_length\": 256,\"max_object_name_length\": 1024,\"account_listing_limit\": 10000,\"container_listing_limit\": 10000}}" - if int64(len(expectedInfo)) != httpResponse.ContentLength { - t.Fatalf("GET of /info httpResponse.ContentLength unexpected") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if expectedInfo != utils.ByteSliceToString(readBuf) { - t.Fatalf("GET of /info httpResponse.Body contents unexpected") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for account "TestAccount" and header Cat: Dog - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Cat", "Dog") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for account "TestAccount" and header Mouse: Bird - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestAccount should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for account "TestAccount" deleting header Mouse - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestAccount should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for account "TestAccount" and header Mouse: Bird - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusAccepted != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestAccount should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for account "TestAccount" deleting header Mouse - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusAccepted != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestAccount should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for container "TestContainer" and header Cat: Dog - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Cat", "Dog") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" expecting "TestContainer\n" and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if int64(len("TestContainer\n")) != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "TestContainer\n" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" with marker "AAA" expecting "TestContainer\n" and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount?marker=AAA", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if int64(len("TestContainer\n")) != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "TestContainer\n" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" with marker "ZZZ" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount?marker=ZZZ", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for container "TestContainer" and header Mouse: Bird - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestContainer should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for container "TestContainer" deleting header Mouse - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestContainer should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for container "TestContainer" and header Mouse: Bird - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusAccepted != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestContainer should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for container "TestContainer" deleting header Mouse - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusAccepted != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestContainer should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a non-chunked PUT for object "Foo" to contain []byte{0x00, 0x01, 0x02} - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/Foo", bytes.NewReader([]byte{0x00, 0x01, 0x02})) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a chunked PUT for object "Bar"" with 1st chunk being []byte{0xAA, 0xBB} & 2nd chunk being []byte{0xCC, 0xDD, 0xEE} - - pipeReader, pipeWriter = io.Pipe() - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/Bar", pipeReader) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.ContentLength = -1 - httpRequest.Header.Del("Content-Length") - errChan = make(chan error, 1) - go func() { - var ( - nonShadowingErr error - nonShadowingHTTPResponse *http.Response - ) - nonShadowingHTTPResponse, nonShadowingErr = httpClient.Do(httpRequest) - if nil == nonShadowingErr { - httpResponse = nonShadowingHTTPResponse - } - errChan <- nonShadowingErr - }() - _, err = pipeWriter.Write([]byte{0xAA, 0xBB}) - if nil != err { - t.Fatalf("pipeWriter.Write() returned unexpected error: %v", err) - } - _, err = pipeWriter.Write([]byte{0xCC, 0xDD, 0xEE}) - if nil != err { - t.Fatalf("pipeWriter.Write() returned unexpected error: %v", err) - } - err = pipeWriter.Close() - if nil != err { - t.Fatalf("pipeWriter.Close() returned unexpected error: %v", err) - } - err = <-errChan - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" expecting "Bar\nFoo\n" and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if int64(len("Bar\nFoo\n")) != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "Bar\nFoo\n" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" with marker "AAA" expecting "Bar\nFoo\n" and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer?marker=AAA", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if int64(len("Bar\nFoo\n")) != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if "Bar\nFoo\n" != utils.ByteSliceToString(readBuf) { - t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" with marker "ZZZ" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer?marker=ZZZ", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "Foo" expecting Content-Length: 3 - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/Foo", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if 3 != httpResponse.ContentLength { - t.Fatalf("httpResponse.ContentLength contained unexpected value: %v", httpResponse.ContentLength) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a full object GET for object "Foo" expecting []byte{0x00, 0x01, 0x02} - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Foo", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if int64(len([]byte{0x00, 0x01, 0x02})) != httpResponse.ContentLength { - t.Fatalf("Foo should contain precisely []byte{0x00, 0x01, 0x02}") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if 0 != bytes.Compare([]byte{0x00, 0x01, 0x02}, readBuf) { - t.Fatalf("Foo should contain precisely []byte{0x00, 0x01, 0x02}") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a range GET of bytes at offset 1 for length 3 for object "Bar" expecting []byte{0xBB, 0xCC, 0xDD} - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Range", "bytes=1-3") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusPartialContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if int64(len([]byte{0xBB, 0xCC, 0xDD})) != httpResponse.ContentLength { - t.Fatalf("Bar's bytes 1-3 should contain precisely []byte{0xBB, 0xCC, 0xDD}") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, readBuf) { - t.Fatalf("Bar's bytes 1-3 should contain precisely []byte{0xBB, 0xCC, 0xDD}") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a range GET of bytes at offset 0 for length 2 - // and offset 3 for length of 1 for object "Bar" - // expecting two MIME parts: []byte{0xAA, 0xBB} and []byte{0xDD} - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Range", "bytes=0-1,3-3") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusPartialContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - contentType = httpResponse.Header.Get("Content-Type") - contentTypeMultiPartBoundary = strings.TrimPrefix(contentType, "multipart/byteranges; boundary=") - if (len(contentType) == len(contentTypeMultiPartBoundary)) || (0 == len(contentTypeMultiPartBoundary)) { - t.Fatalf("httpReponse.Header[\"Content-Type\"] contained unexpected value: \"%v\"", contentType) - } - expectedBuf = make([]byte, 0, httpResponse.ContentLength) - expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"\r\n")...) - expectedBuf = append(expectedBuf, []byte("Content-Type: application/octet-stream\r\n")...) - expectedBuf = append(expectedBuf, []byte("Content-Range: bytes 0-1/5\r\n")...) - expectedBuf = append(expectedBuf, []byte("\r\n")...) - expectedBuf = append(expectedBuf, []byte{0xAA, 0xBB}...) - expectedBuf = append(expectedBuf, []byte("\r\n")...) - expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"\r\n")...) - expectedBuf = append(expectedBuf, []byte("Content-Type: application/octet-stream\r\n")...) - expectedBuf = append(expectedBuf, []byte("Content-Range: bytes 3-3/5\r\n")...) - expectedBuf = append(expectedBuf, []byte("\r\n")...) - expectedBuf = append(expectedBuf, []byte{0xDD}...) - expectedBuf = append(expectedBuf, []byte("\r\n")...) - expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"--")...) - if int64(len(expectedBuf)) != httpResponse.ContentLength { - t.Fatalf("Unexpected multi-part GET response Content-Length") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if 0 != bytes.Compare(expectedBuf, readBuf) { - t.Fatalf("Unexpected payload of multi-part GET response") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a tail GET of the last 2 bytes for object "Bar" expecting []byte{0xDD, 0xEE} - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Range", "bytes=-2") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusPartialContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if int64(len([]byte{0xDD, 0xEE})) != httpResponse.ContentLength { - t.Fatalf("Bar's last 2 bytes should contain precisely []byte{0xDD, 0xEE}") - } - readBuf, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) - } - if 0 != bytes.Compare([]byte{0xDD, 0xEE}, readBuf) { - t.Fatalf("Bar's last 2 bytes should contain precisely []byte{0xDD, 0xEE}") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a PUT for object "ZigZag" and header Cat: Dog - - httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Cat", "Dog") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusCreated != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "ZigZag" expecting header Cat: Dog - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "ZigZag" expecting header Cat: Dog - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for object "ZigZag" and header Mouse: Bird - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "Bird") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "ZigZag" expecting header Cat: Dog & Mouse: Bird - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if httpResponse.Header.Get("Mouse") != "Bird" { - t.Fatalf("TestAccount should have header Mouse: Bird") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a POST for object "ZigZag" deleting header Mouse - - httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpRequest.Header.Add("Mouse", "") - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a HEAD for object "ZigZag" expecting header Cat: Dog & no Mouse header - - httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - _, mouseHeaderPresent = httpResponse.Header["Mouse"] - if mouseHeaderPresent { - t.Fatalf("TestAccount should not have header Mouse") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for object "Foo" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/Foo", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for object "Bar" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/Bar", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for object "ZigZag" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for container "TestContainer" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestContainer should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestContainer should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for container "TestContainer" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a GET for account "TestAccount" expecting Content-Length: 0 and header Cat: Dog - - httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - if httpResponse.Header.Get("Cat") != "Dog" { - t.Fatalf("TestAccount should have header Cat: Dog") - } - if 0 != httpResponse.ContentLength { - t.Fatalf("TestAccount should contain no elements at this point") - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send a DELETE for account "TestAccount" - - httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount", nil) - if nil != err { - t.Fatalf("http.NewRequest() returned unexpected error: %v", err) - } - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - t.Fatalf("httpClient.Do() returned unexpected error: %v", err) - } - if http.StatusNoContent != httpResponse.StatusCode { - t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) - } - err = httpResponse.Body.Close() - if nil != err { - t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) - } - - // Send ourself a SIGTERM to signal normal termination of mainWithArgs() - - unix.Kill(unix.Getpid(), unix.SIGTERM) - - _ = <-doneChan -} diff --git a/ramswift/macramswift0.conf b/ramswift/macramswift0.conf deleted file mode 100644 index c0895f90..00000000 --- a/ramswift/macramswift0.conf +++ /dev/null @@ -1,10 +0,0 @@ -# Peer 0's .conf file - -.include ../proxyfsd/macproxyfsd0.conf - -.include ./chaos_settings.conf - -.include ./swift_info.conf - -[SwiftClient] -SwiftReconChecksPerConfCheck: 0 diff --git a/ramswift/ramswift/Makefile b/ramswift/ramswift/Makefile deleted file mode 100644 index 3c34d068..00000000 --- a/ramswift/ramswift/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/ramswift/ramswift - -include ../../GoMakefile diff --git a/ramswift/ramswift/dummy_test.go b/ramswift/ramswift/dummy_test.go deleted file mode 100644 index d4299172..00000000 --- a/ramswift/ramswift/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/ramswift/ramswift/main.go b/ramswift/ramswift/main.go deleted file mode 100644 index a94373f1..00000000 --- a/ramswift/ramswift/main.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "log" - "os" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/ramswift" -) - -func main() { - if len(os.Args) < 2 { - log.Fatalf("no .conf file specified") - } - - doneChan := make(chan bool, 1) // Must be buffered to avoid race - - go ramswift.Daemon(os.Args[1], os.Args[2:], nil, doneChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) - - _ = <-doneChan -} diff --git a/ramswift/ramswift0.conf b/ramswift/ramswift0.conf deleted file mode 100644 index ec97388d..00000000 --- a/ramswift/ramswift0.conf +++ /dev/null @@ -1,10 +0,0 @@ -# Peer 0's .conf file - -.include ../proxyfsd/proxyfsd0.conf - -.include ./chaos_settings.conf - -.include ./swift_info.conf - -[SwiftClient] -SwiftReconChecksPerConfCheck: 0 diff --git a/ramswift/saioramswift0.conf b/ramswift/saioramswift0.conf deleted file mode 100644 index 152f9f11..00000000 --- a/ramswift/saioramswift0.conf +++ /dev/null @@ -1,10 +0,0 @@ -# Peer 0's .conf file - -.include ../proxyfsd/saioproxyfsd0.conf - -.include ./chaos_settings.conf - -.include ./swift_info.conf - -[SwiftClient] -SwiftReconChecksPerConfCheck: 0 diff --git a/ramswift/swift_info.conf b/ramswift/swift_info.conf deleted file mode 100644 index e64ede67..00000000 --- a/ramswift/swift_info.conf +++ /dev/null @@ -1,20 +0,0 @@ -# Swift BackEnd Info Response .conf file -# -# These name:value pairs are sent in JSON format in response to a query on /info (no "v1" in the path) -# -# JSON Object should look like: -# -# { -# 'swift' : { -# 'max_account_name_length' : , -# 'max_container_name_length' : , -# 'max_object_name_length' : -# } -# } - -[RamSwiftInfo] -MaxAccountNameLength: 256 -MaxContainerNameLength: 256 -MaxObjectNameLength: 1024 -AccountListingLimit: 10000 -ContainerListingLimit: 10000 diff --git a/readdir-stress/.gitignore b/readdir-stress/.gitignore deleted file mode 100644 index d9fc8fd9..00000000 --- a/readdir-stress/.gitignore +++ /dev/null @@ -1 +0,0 @@ -readdir-stress diff --git a/readdir-stress/Makefile b/readdir-stress/Makefile deleted file mode 100644 index bb903850..00000000 --- a/readdir-stress/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -TARGET = readdir-stress -LIBS = -lm -lpthread -CC = gcc -CFLAGS = -Wall -Werror -LFLAGS = -Wall - -.PHONY: default all clean - -default: $(TARGET) -all: default - -OBJECTS = $(patsubst %.c, %.o, $(wildcard *.c)) -HEADERS = $(wildcard *.h) - -%.o: %.c $(HEADERS) - $(CC) $(CFLAGS) -c $< -o $@ - -.PRECIOUS: $(TARGET) $(OBJECTS) - -$(TARGET): $(OBJECTS) - $(CC) $(OBJECTS) $(LFLAGS) $(LIBS) -o $@ - -clean: - -rm -f *.o - -rm -f $(TARGET) diff --git a/readdir-stress/readdir-stress.c b/readdir-stress/readdir-stress.c deleted file mode 100644 index cbe01147..00000000 --- a/readdir-stress/readdir-stress.c +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -typedef struct { - pthread_t thread; - int passes_completed; -} readdir_thread_t; - -#define MAX_DIR_ENTRIES 1000 -#define MAX_PASSES_PER_THREAD 100000 -#define MAX_THREADS 80 // Note: Line-length of status display limited - -#define SECONDS_PER_POLL 1 - -#define DIR_CREATION_MODE (S_IRWXU | S_IRWXG | S_IRWXO) -#define FILE_CREATION_MODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH) - -static const char convert_int_modulo_0_thru_9_A_thru_Z[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - -char *dir_path; -size_t dirent_malloc_size; -pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; -int num_dir_entries; -int num_passes_in_total; -int num_passes_per_thread; -int num_threads; -int passes_completed_in_total; - -void usage(const char *argv_0) { - fprintf(stderr, "%s <# dir-entries> <# threads> <# passes/thread>\n", argv_0); - fprintf(stderr, " : directory to create (must not exist; parent must exist)\n"); - fprintf(stderr, " <# dir-entries>: # of files created in -specified directory\n"); - fprintf(stderr, " <# threads: # of threads simultaneously performing readdir_r(3)'s\n"); - fprintf(stderr, " <# passes/thread>: # of times to read the -specified directory\n"); -} - -void *readdir_start_routine(void *arg) { - DIR *dir; - struct dirent *entry; - int file_index; - int pass_index; - int posix_ret; - readdir_thread_t *readdir_thread = (readdir_thread_t *)arg; - struct dirent *result; - - entry = (struct dirent *)malloc(dirent_malloc_size); - if (NULL == entry) { - fprintf(stderr, "entry = malloc(dirent_malloc_size) failed: %s\n", strerror(errno)); - exit(1); - } - - for (pass_index = 0; pass_index < num_passes_per_thread; pass_index++) { - dir = opendir(dir_path); - if (NULL == dir) { - fprintf(stderr, "opendir(%s) failed: %s\n", dir_path, strerror(errno)); - exit(1); - } - - // Read ".", "..", and all but the last entry - for (file_index = -1; file_index <= num_dir_entries; file_index++) { - result = NULL; - posix_ret = readdir_r(dir, entry, &result); - if (0 != posix_ret) { - fprintf(stderr, "readdir_r(%s,,) failed: %s\n", dir_path, strerror(posix_ret)); - exit(1); - } - if (NULL == result) { // For all but the last entry - fprintf(stderr, "readdir_r(%s,,) should have returned non-NULL result\n", dir_path); - exit(1); - } - } - - // Read what should be the last entry - result = NULL; - posix_ret = readdir_r(dir, entry, &result); - if (0 != posix_ret) { - fprintf(stderr, "readdir_r(%s,,) failed: %s\n", dir_path, strerror(posix_ret)); - exit(1); - } - if (NULL != result) { // For the last entry - fprintf(stderr, "readdir_r(%s,,) should have returned NULL result\n", dir_path); - exit(1); - } - - posix_ret = closedir(dir); - if (0 != posix_ret) { - fprintf(stderr, "closedir(%s) failed: %s\n", dir_path, strerror(errno)); - exit(1); - } - - readdir_thread->passes_completed = 1 + pass_index; - - posix_ret = pthread_mutex_lock(&mutex); - if (0 != posix_ret) { - fprintf(stderr, "pthread_mutex_lock(&mutex) failed: %s\n", strerror(posix_ret)); - exit(1); - } - - passes_completed_in_total++; - - posix_ret = pthread_mutex_unlock(&mutex); - if (0 != posix_ret) { - fprintf(stderr, "pthread_mutex_unlock(&mutex) failed: %s\n", strerror(posix_ret)); - exit(1); - } - } - - free(entry); - - return NULL; -} - -int main(int argc, const char * argv[]) { - DIR *dir_path_dir; - char *dir_path_dirname; - char *dir_path_dirname_buf; - size_t dirent_d_name_offset; - int fd; - int file_index; - char *file_path; - size_t file_path_len; - int posix_ret; - readdir_thread_t *readdir_thread; - readdir_thread_t *readdir_thread_array; - struct stat stat_buf; - char *status_string; - int thread_index; - - // Validate arguments - - if (5 != argc) { - usage(argv[0]); - exit(1); - } - - dir_path = strdup(argv[1]); - if (NULL == dir_path) { - fprintf(stderr, "strdup(%s) failed: %s\n", argv[1], strerror(errno)); - exit(1); - } - - num_dir_entries = atoi(argv[2]); - num_threads = atoi(argv[3]); - num_passes_per_thread = atoi(argv[4]); - - dir_path_dirname_buf = strdup(dir_path); - if (NULL == dir_path_dirname_buf) { - fprintf(stderr, "strdup(%s) failed: %s\n", dir_path, strerror(errno)); - exit(1); - } - dir_path_dirname = dirname(dir_path_dirname_buf); - - dir_path_dir = opendir(dir_path_dirname); - if (NULL == dir_path_dir) { - fprintf(stderr, "%s must exist\n", dir_path_dirname); - exit(1); - } else { - posix_ret = closedir(dir_path_dir); - if (0 != posix_ret) { - fprintf(stderr, "closedir(%s) failed: %s\n", dir_path_dirname, strerror(errno)); - exit(1); - } - } - - posix_ret = stat(dir_path, &stat_buf); - if (0 == posix_ret) { - fprintf(stderr, "%s must not pre-exist\n", dir_path); - exit(1); - } else { - if (ENOENT != errno) { - fprintf(stderr, "stat(%s,) failed: %s\n", dir_path, strerror(errno)); - exit(1); - } - } - - if ((1 > num_dir_entries) || (MAX_DIR_ENTRIES < num_dir_entries)) { - fprintf(stderr, "num_dir_entries (%d) must be between 1 and %d (inclusive)\n", num_dir_entries, MAX_DIR_ENTRIES); - exit(1); - } - if ((1 > num_threads) || (MAX_THREADS < num_threads)) { - fprintf(stderr, "num_threads (%d) must be between 1 and %d (inclusive)\n", num_threads, MAX_THREADS); - exit(1); - } - if ((1 > num_passes_per_thread) || (MAX_PASSES_PER_THREAD < num_passes_per_thread)) { - fprintf(stderr, "num_passes_per_thread (%d) must be between 1 and %d (inclusive)\n", num_passes_per_thread, MAX_PASSES_PER_THREAD); - exit(1); - } - - // Build test directory - - posix_ret = mkdir(dir_path, DIR_CREATION_MODE); - if (0 != posix_ret) { - fprintf(stderr, "mkdir(%s,) failed: %s\n", dir_path, strerror(errno)); - exit(1); - } - - file_path_len = strlen(dir_path) + 1 + 5 + 4; // "/file_XXXX" - file_path = (char *)malloc(file_path_len + 1); - if (NULL == file_path) { - fprintf(stderr, "file_path = malloc(file_path_len + 1) failed: %s\n", strerror(errno)); - exit(1); - } - - for (file_index = 0; file_index < num_dir_entries; file_index++) { - posix_ret = sprintf(file_path, "%s/file_%04X", dir_path, file_index); - if (file_path_len != posix_ret) { - fprintf(stderr, "sprintf(,,,) returned unexpected length: %d\n", posix_ret); - exit(1); - } - - fd = open(file_path, O_WRONLY|O_CREAT|O_EXCL, FILE_CREATION_MODE); - if (-1 == fd) { - fprintf(stderr, "open(%s,,) failed: %s\n", file_path, strerror(errno)); - exit(1); - } - - posix_ret = close(fd); - if (0 != posix_ret) { - fprintf(stderr, "close(fd[%s]) failed: %s\n", file_path, strerror(errno)); - exit(1); - } - } - - // Start pthreads - - dirent_d_name_offset = offsetof(struct dirent, d_name); - dirent_malloc_size = dirent_d_name_offset + 5 + 4 + 1; // "file_XXXX" - - num_passes_in_total = num_threads * num_passes_per_thread; - passes_completed_in_total = 0; - - readdir_thread_array = (readdir_thread_t *)malloc(num_threads * sizeof(readdir_thread_t)); - if (NULL == readdir_thread_array) { - fprintf(stderr, "readdir_thread_array = malloc(%d * sizeof(readdir_thread_t)) failed: %s\n", num_threads, strerror(errno)); - exit(1); - } - - for (thread_index = 0; thread_index < num_threads; thread_index++) { - readdir_thread = &readdir_thread_array[thread_index]; - - readdir_thread->passes_completed = 0; - - posix_ret = pthread_create(&readdir_thread->thread, NULL, readdir_start_routine, readdir_thread); - if (0 != posix_ret) { - fprintf(stderr, "pthread_create(,,,) failed: %s\n", strerror(posix_ret)); - exit(1); - } - } - - // Poll reader_thread's until they are done - - status_string = (char *)malloc(num_threads + 1); - if (NULL == status_string) { - fprintf(stderr, "status_string = malloc(num_threads + 1) failed: %s\n", strerror(errno)); - exit(1); - } - status_string[num_threads] = '\0'; - - for (;;) { - (void)sleep(SECONDS_PER_POLL); - - for (thread_index = 0; thread_index < num_threads; thread_index++) { - readdir_thread = &readdir_thread_array[thread_index]; - status_string[thread_index] = convert_int_modulo_0_thru_9_A_thru_Z[readdir_thread->passes_completed % 36]; - } - - printf("%s\n", status_string); - - if (num_passes_in_total == passes_completed_in_total) { - break; - } - } - - // Clean-up - - for (thread_index = 0; thread_index < num_threads; thread_index++) { - readdir_thread = &readdir_thread_array[thread_index]; - - posix_ret = pthread_join(readdir_thread->thread, NULL); - if (0 != posix_ret) { - fprintf(stderr, "pthread_join(,) failed: %s\n", strerror(posix_ret)); - exit(1); - } - } - - for (file_index = 0; file_index < num_dir_entries; file_index++) { - posix_ret = sprintf(file_path, "%s/file_%04X", dir_path, file_index); - if (file_path_len != posix_ret) { - fprintf(stderr, "sprintf(,,,) returned unexpected length: %d\n", posix_ret); - exit(1); - } - - posix_ret = unlink(file_path); - if (0 != posix_ret) { - fprintf(stderr, "unlink(%s) failed: %s\n", file_path, strerror(errno)); - exit(1); - } - } - - posix_ret = rmdir(dir_path); - if (0 != posix_ret) { - fprintf(stderr, "rmdir(%s) failed: %s\n", dir_path, strerror(errno)); - exit(1); - } - - free(status_string); - free(readdir_thread_array); - free(file_path); - free(dir_path_dirname_buf); - free(dir_path); - - exit(0); -} diff --git a/saio/Vagrantfile b/saio/Vagrantfile deleted file mode 100644 index f3916289..00000000 --- a/saio/Vagrantfile +++ /dev/null @@ -1,36 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Notes: -# -# 1) vboxnet0 is assumed to be a host-only network @ address 172.28.128.1 (DHCP disabled) -# 2) Though not required, GOPATH is assumed to be the ../../../../../ directory -# 3) The directory on the VM Host will be /vagrant on the VM and be the path in GOPATH -# 4) ProxyFS repo git clone'd to $GOPATH/src/github.com/NVIDIA/ -# 5) Swift repos et. al. git clone'd to $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ -# 6) ../Makefile will be ready to be executed after `cdpfs` inside the VM -# 7) As GOPATH is effectively shared between Host and VM, builds in the two environments -# will collide in the contents of the $GOPATH/bin (only executables, not libraries) -# 8) With GOPATH now deprecated, much of the above concerns need to be modified (or ignored) - -Vagrant.configure(2) do |config| - config.vm.box = "centos-74-minimal-20180727" - config.vm.box_url = "https://o.swiftstack.org/v1/AUTH_misc/vagrant_boxes/centos-74-minimal-20180727.box" - config.vm.define "saiopfs" do |saiopfs| - end - config.vm.provider :virtualbox do |vb| - vb.name = "SAIO for ProxyFS" - vb.cpus = Integer(ENV['VAGRANT_CPUS'] || 2) - vb.memory = Integer(ENV['VAGRANT_RAM'] || 6144) - vb.customize ["modifyvm", :id, "--audio", "none"] - end - config.vm.synced_folder "../../../../../", "/vagrant", type: "virtualbox" - config.vm.network "private_network", ip: "172.28.128.2", :name => 'vboxnet0', :adapter => 2 - config.vm.network "forwarded_port", guest: 15346, host: 15346 - config.vm.network "forwarded_port", guest: 8080, host: 8080 - config.vm.network "forwarded_port", guest: 8443, host: 8443 - config.vm.network "forwarded_port", guest: 32356, host: 32356 - config.vm.network "forwarded_port", guest: 9090, host: 9091 - config.vm.network "private_network", ip: "192.168.22.113", :name => 'vboxnet1', :adapter => 3 - config.vm.provision "shell", path: "vagrant_provision.sh" -end diff --git a/saio/bin/flush_caches b/saio/bin/flush_caches deleted file mode 100755 index 441a5cb2..00000000 --- a/saio/bin/flush_caches +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -sudo su -c 'sync; echo 2 > /proc/sys/vm/drop_caches' diff --git a/saio/bin/provision_middleware b/saio/bin/provision_middleware deleted file mode 100755 index 90eff428..00000000 --- a/saio/bin/provision_middleware +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -cd /vagrant/src/github.com/NVIDIA/proxyfs/pfs_middleware -sudo python setup.py develop -cd /vagrant/src/github.com/NVIDIA/proxyfs/meta_middleware -sudo python setup.py develop diff --git a/saio/bin/reset_etcd b/saio/bin/reset_etcd deleted file mode 100755 index 5b548cc2..00000000 --- a/saio/bin/reset_etcd +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -sudo rm -rf /etcd/proxyfs.etcd diff --git a/saio/bin/start_etcd b/saio/bin/start_etcd deleted file mode 100755 index e3790f2c..00000000 --- a/saio/bin/start_etcd +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -sudo systemctl start proxyfs-etcd - -# To access from cmdline: -# -# ETCDCTL_API=3 etcdctl --endpoints=http://localhost:2379 put foo bar -# ETCDCTL_API=3 etcdctl --endpoints=http://localhost:2379 get foo -# ETCDCTL_API=3 etcdctl --endpoints=http://localhost:2379 txn -i -# value("foo") = "bar" -# -# put foo cat -# -# put foo dog -# diff --git a/saio/bin/start_proxyfs_and_swift b/saio/bin/start_proxyfs_and_swift deleted file mode 100755 index 6bf87bb3..00000000 --- a/saio/bin/start_proxyfs_and_swift +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -# A simple script to start the swift services only -# The PATH, etc should already be setup by systemctl environment - -function await_proxyfsd_startup { - while true - do - /usr/bin/systemctl -q is-active proxyfsd - if [ $? -ne 0 ] - then - echo "ProxyFS failed to start. Exiting..." - exit 1 - fi - curl http://127.0.0.1:15346/ 2>/dev/null >/dev/null - if [ $? -eq 0 ] - then - break - fi - sleep 1 - done -} - -function await_swift_startup { - while true - do - curl http://127.0.0.1:8090/info 2>/dev/null >/dev/null - if [ $? -eq 0 ] - then - break - fi - echo "Waiting for Swift to be started..." - sleep 1 - done -} - -function format_volume_if_necessary { - sudo /vagrant/bin/mkproxyfs -I $1 /vagrant/src/github.com/NVIDIA/proxyfs/saio/proxyfs.conf SwiftClient.RetryLimit=1 - if [ $? -ne 0 ] - then - echo "Could not pre-format $1" - exit 1 - fi -} - -sudo mount -a - -echo "Shutting down services and mount points..." -/vagrant/src/github.com/NVIDIA/proxyfs/saio/bin/unmount_and_stop_pfs -echo -echo "Bringing up services..." -if [ -f /usr/bin/systemctl ]; then - # Centos - sudo /usr/bin/systemctl start memcached - sudo /usr/bin/swift-init main start - await_swift_startup - format_volume_if_necessary CommonVolume - sudo /usr/bin/systemctl start proxyfsd - await_proxyfsd_startup -else - # Ubuntu (not tested!) - sudo /usr/sbin/service memcached start - sudo /usr/bin/swift-init main start - await_swift_startup - format_volume_if_necessary CommonVolume - sudo /usr/sbin/service proxyfsd start - await_proxyfsd_startup -fi diff --git a/saio/bin/start_swift_only b/saio/bin/start_swift_only deleted file mode 100755 index 45edb519..00000000 --- a/saio/bin/start_swift_only +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -# A simple script to start the swift services only -# The PATH, etc should already be setup by systemctl environment - -sudo mount -a - -echo "Shutting down services and mount points..." -/vagrant/src/github.com/NVIDIA/proxyfs/saio/bin/unmount_and_stop_pfs -echo -echo "Bringing up services..." -if [ -f /usr/bin/systemctl ]; then - # Centos - sudo /usr/bin/systemctl start memcached - sudo /usr/bin/swift-init main start -else - # Ubuntu (not tested!) - sudo /usr/sbin/service memcached start - sudo /usr/bin/swift-init main start -fi diff --git a/saio/bin/stop_etcd b/saio/bin/stop_etcd deleted file mode 100755 index 5b109a39..00000000 --- a/saio/bin/stop_etcd +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -sudo systemctl stop proxyfs-etcd diff --git a/saio/bin/unmount_and_stop_pfs b/saio/bin/unmount_and_stop_pfs deleted file mode 100755 index 9f72570e..00000000 --- a/saio/bin/unmount_and_stop_pfs +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -# A simple script to unmount the sample mount point and stop the services -# The PATH, etc should already be setup by systemctl environment - -function await_proxyfsd_shutdown { - while true - do - pidof proxyfsd > /dev/null - if [ $? -ne 0 ] - then - break - fi - echo "Waiting for ProxyFS to be stopped..." - sleep 1 - done -} - -function await_pfsagentd_shutdown { - while true - do - pidof pfsagentd > /dev/null - if [ $? -ne 0 ] - then - break - fi - echo "Waiting for PFSAgent to be stopped..." - sleep 1 - done -} - -MOUNT_POINT_NFS=127.0.0.1:/CommonMountPoint -SHARE_NFS=/mnt/nfs_proxyfs_mount/ - -MOUNT_POINT_SMB=//127.0.0.1/proxyfs -SHARE_SMB=/mnt/smb_proxyfs_mount/ - -if [ $# = 0 ]; then - SHOULD_UNMOUNT=1 -else - if [ $1 = "keepmounts" ]; then - SHOULD_UNMOUNT=0 - else - echo "Invalid argument: $1" - exit 1 - fi -fi - -if [ $SHOULD_UNMOUNT = 1 ]; then - mountpoint -q $SHARE_NFS - if [ $? -eq 0 ]; then - echo "Unmounting NFS mount point at '${SHARE_NFS}'..." - sudo umount $SHARE_NFS - fi - - mountpoint -q $SHARE_SMB - if [ $? -eq 0 ]; then - echo "Unmounting SMB mount point at '${SHARE_SMB}'..." - sudo umount $SHARE_SMB - fi -fi - -if [ -f /usr/bin/systemctl ]; then - # Centos - sudo /usr/bin/systemctl stop pfsagentd - sudo /usr/bin/systemctl stop nfs-idmap - sudo /usr/bin/systemctl stop nfs-lock - sudo /usr/bin/systemctl stop nfs-server - sudo /usr/bin/systemctl stop rpcbind - sudo /usr/bin/systemctl stop smb - # We need to make sure PFSAgent is stopped before we stop ProxyFS, but we - # don't care if other services are stopped in the meantime. - await_pfsagentd_shutdown - sudo /usr/bin/systemctl stop proxyfsd - await_proxyfsd_shutdown - sudo /usr/bin/swift-init main stop - sudo /usr/bin/systemctl stop memcached -else - # Ubuntu (not tested!) - # Here we should stop pfsagentd, but we don't support Ubuntu - sudo /usr/sbin/service nfs-idmap stop - sudo /usr/sbin/service nfs-lock stop - sudo /usr/sbin/service nfs-server stop - sudo /usr/sbin/service rpcbind stop - sudo /usr/sbin/service smbd stop - sudo /usr/sbin/service proxyfsd stop - await_proxyfsd_shutdown - sudo /usr/bin/swift-init main stop - sudo /usr/sbin/service memcached stop -fi diff --git a/saio/container/.gitignore b/saio/container/.gitignore deleted file mode 100644 index 58817cb0..00000000 --- a/saio/container/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -meta_middleware/ -pfs_middleware/ diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile deleted file mode 100644 index 7903f620..00000000 --- a/saio/container/Dockerfile +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -FROM centos:7.4.1708 - -ARG SwiftVersion -ARG GolangVersion=1.16.6 -ARG ProxyFS_Version=stable - -ENV GolangBasename "go${GolangVersion}.linux-amd64.tar.gz" -ENV GolangURL "https://golang.org/dl/${GolangBasename}" - -RUN yum install -y yum-utils - -RUN yum-config-manager --disable CentOS-Base -RUN yum-config-manager --disable CentOS-CR -RUN yum-config-manager --disable CentOS-Debuginfo -RUN yum-config-manager --disable CentOS-fasttrack -RUN yum-config-manager --disable CentOS-Media -RUN yum-config-manager --disable CentOS-Sources -RUN yum-config-manager --disable CentOS-Vault - -RUN rm -rf /etc/yum.repos.d/CentOS-Base.repo -RUN rm -rf /etc/yum.repos.d/CentOS-CR.repo -RUN rm -rf /etc/yum.repos.d/CentOS-Debuginfo.repo -RUN rm -rf /etc/yum.repos.d/CentOS-fasttrack.repo -RUN rm -rf /etc/yum.repos.d/CentOS-Media.repo -RUN rm -rf /etc/yum.repos.d/CentOS-Sources.repo -RUN rm -rf /etc/yum.repos.d/CentOS-Vault.repo - -RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/os/x86_64/ -RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/updates/x86_64/ -RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/extras/x86_64/ -RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/centosplus/x86_64/ -RUN yum-config-manager --enable vault.centos.org_centos_7.4.1708_os_x86_64_ -RUN yum-config-manager --enable vault.centos.org_centos_7.4.1708_updates_x86_64_ -RUN yum-config-manager --enable vault.centos.org_centos_7.4.1708_extras_x86_64_ -RUN yum-config-manager --enable vault.centos.org_centos_7.4.1708_centosplus_x86_64_ - -RUN yum clean all - -RUN yum install -y wget -RUN yum install -y git - -RUN yum install -y --disableexcludes=all glibc-common gcc - -RUN useradd --user-group --groups wheel swift -RUN chmod 755 ~swift - -RUN yum install -y attr -RUN yum install -y e2fsprogs - -RUN mkdir -p /srv -RUN truncate -s 1GB /srv/swift-disk -RUN echo "y" | mkfs.ext4 -I 4096 /srv/swift-disk -RUN mkdir -p /srv/1/node/sdb1 -RUN echo "/srv/swift-disk /srv/1/node/sdb1 ext4 loop,noatime,nodiratime,nobarrier 0 0" >> /etc/fstab - -RUN mkdir -p /var/cache/swift /var/cache/swift2 /var/cache/swift3 /var/cache/swift4 -RUN chown swift:swift /var/cache/swift* -RUN mkdir -p /var/run/swift -RUN chown swift:swift /var/run/swift - -RUN yum install -y \ - memcached \ - sqlite \ - xfsprogs \ - libffi-devel \ - xinetd \ - openssl-devel \ - python-setuptools \ - python-coverage \ - python-devel \ - python-nose \ - pyxattr \ - python-eventlet \ - python-greenlet \ - python-paste-deploy \ - python-netifaces \ - python-dns \ - python-mock - -RUN yum install -y centos-release-scl -RUN yum install -y rh-python36 -RUN ln -s /opt/rh/rh-python36/root/bin/python3.6 /bin/python3.6 -RUN ln -s /bin/python3.6 /bin/python3 -RUN ln -s /opt/rh/rh-python36/root/usr/include /opt/rh/rh-python36/root/include - -RUN yum install -y epel-release -RUN yum install -y python-pip -RUN pip install --upgrade 'pip<21.0' -RUN pip install --upgrade setuptools - -WORKDIR /home/swift -RUN git clone https://github.com/openstack/liberasurecode.git -WORKDIR /home/swift/liberasurecode -RUN yum install -y gcc make autoconf automake libtool -RUN ./autogen.sh -RUN ./configure -RUN make -RUN make install -RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/liberasurecode.conf -RUN ldconfig - -WORKDIR /home/swift -RUN git clone https://github.com/openstack/pyeclib.git -WORKDIR /home/swift/pyeclib -RUN pip install -e . -RUN pip install -r test-requirements.txt - -WORKDIR /home/swift -RUN git clone https://github.com/NVIDIA/swift.git -WORKDIR /home/swift/swift -RUN if [[ -n "$SwiftVersion" ]] ; then git checkout $SwiftVersion ; else git checkout $( git describe --tags --abbrev=0 $(git rev-list --tags --max-count=1) ) ; fi -RUN pip install wheel -RUN python setup.py bdist_wheel -RUN yum remove -y python-greenlet -RUN pip install --constraint py2-constraints.txt -r requirements.txt -RUN python setup.py develop -# The following avoid dependency on pip-installed pyOpenSSL being newer than required -RUN pip install python-openstackclient==3.12.0 python-glanceclient==2.7.0 -# This is a temporary fix while bandit gets added to py2-constraints.txt -RUN pip install bandit==1.6.2 -RUN pip install --constraint py2-constraints.txt -r test-requirements.txt - -# TODO: May need to do something about: -# systemctl enable memcached.service -# systemctl start memcached.service - -WORKDIR /etc/swift - -RUN rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz -RUN swift-ring-builder object.builder create 10 1 1 -RUN swift-ring-builder object.builder add r1z1-127.0.0.1:8010/sdb1 1 -RUN swift-ring-builder object.builder rebalance -RUN swift-ring-builder object-1.builder create 10 1 1 -RUN swift-ring-builder object-1.builder add r1z1-127.0.0.1:8010/sdb1 1 -RUN swift-ring-builder object-1.builder rebalance -RUN swift-ring-builder object-2.builder create 10 1 1 -RUN swift-ring-builder object-2.builder add r1z1-127.0.0.1:8010/sdb1 1 -RUN swift-ring-builder object-2.builder rebalance -RUN swift-ring-builder container.builder create 10 1 1 -RUN swift-ring-builder container.builder add r1z1-127.0.0.1:8011/sdb1 1 -RUN swift-ring-builder container.builder rebalance -RUN swift-ring-builder account.builder create 10 1 1 -RUN swift-ring-builder account.builder add r1z1-127.0.0.1:8012/sdb1 1 -RUN swift-ring-builder account.builder rebalance - -COPY etc/swift . -RUN chown -R swift:swift . - -WORKDIR /opt/ProxyFS - -RUN rm -rf /var/log/proxyfsd -RUN mkdir -p /var/log/proxyfsd -RUN touch /var/log/proxyfsd/proxyfsd.log -RUN chmod 777 /var -RUN chmod 777 /var/log -RUN chmod 777 /var/log/proxyfsd -RUN chmod 666 /var/log/proxyfsd/proxyfsd.log - -RUN wget -nv $GolangURL -RUN tar -C /usr/local -xzf $GolangBasename - -ENV PATH $PATH:/usr/local/go/bin - -RUN yum install -y make fuse - -RUN mkdir -p /opt/ProxyFS/GOPATH/src/github.com/NVIDIA -WORKDIR /opt/ProxyFS/GOPATH/src/github.com/NVIDIA -RUN git clone https://github.com/NVIDIA/proxyfs.git -WORKDIR /opt/ProxyFS/GOPATH/src/github.com/NVIDIA/proxyfs -RUN git checkout $ProxyFS_Version - -RUN cd pfs_middleware && python setup.py develop -RUN cd meta_middleware && python setup.py develop - -RUN make version generate build - -WORKDIR /opt/ProxyFS - -RUN mkdir CommonMountPoint -RUN chmod 777 CommonMountPoint - -RUN yum install -y rsyslog -RUN echo "\$ModLoad imudp" >> /etc/rsyslog.conf -RUN echo "\$UDPServerRun 514" >> /etc/rsyslog.conf - -COPY docker_startup.sh . -COPY proxyfs.conf . -RUN chmod +x docker_startup.sh - -CMD /opt/ProxyFS/docker_startup.sh - -# To build this image: -# -# docker build \ -# [--build-arg SwiftVersion=] \ -# [--build-arg ProxyFS_Version=] \ -# [-t :] . - -# To run a container for this image: -# -# docker run -d --rm --device /dev/fuse --device /dev/loop0 --cap-add SYS_ADMIN -p 15346:15346 saio-pfs diff --git a/saio/container/Makefile b/saio/container/Makefile deleted file mode 100644 index db38d9c2..00000000 --- a/saio/container/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -all : clone-pfs clone-meta docker-build - -.PHONY: all clone-pfs clone-meta docker-build - -clone-pfs: - rm -rf pfs_middleware - cp -R ../../pfs_middleware . - -clone-meta: - rm -rf meta_middleware - cp -R ../../meta_middleware . - -docker-build: - docker build -t saio-pfs:latest . diff --git a/saio/container/docker_startup.sh b/saio/container/docker_startup.sh deleted file mode 100644 index e2bdbe73..00000000 --- a/saio/container/docker_startup.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -mount /srv/1/node/sdb1 -chown -R swift:swift /srv/1/node/sdb1 - -rsyslogd - -memcached -u swift -d - -swift-init main start - -while true -do - curl http://127.0.0.1:8090/info 2>/dev/null >/dev/null - if [ $? -eq 0 ] - then - break - fi - echo "Waiting for Swift to be started..." - sleep 1 -done - -echo "Swift is up... starting ProxyFS..." - -myPublicIPAddr=`ip addr | grep inet | grep eth0 | awk '{print $2}' | awk 'BEGIN {FS="/"} {print $1}'` - -echo "myPublicIPAddr == $myPublicIPAddr" - -proxyfsd proxyfs.conf Peer:Peer0.PublicIPAddr=$myPublicIPAddr diff --git a/saio/container/etc/swift/account-server/1.conf b/saio/container/etc/swift/account-server/1.conf deleted file mode 100644 index 093be13a..00000000 --- a/saio/container/etc/swift/account-server/1.conf +++ /dev/null @@ -1,27 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.1 -bind_port = 8012 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon account-server - -[app:account-server] -use = egg:swift#account - -[filter:recon] -use = egg:swift#recon - -[account-replicator] -rsync_module = {replication_ip}::account{replication_port} - -[account-auditor] - -[account-reaper] diff --git a/saio/container/etc/swift/container-reconciler.conf b/saio/container/etc/swift/container-reconciler.conf deleted file mode 100644 index 30f81ca5..00000000 --- a/saio/container/etc/swift/container-reconciler.conf +++ /dev/null @@ -1,47 +0,0 @@ -[DEFAULT] -# swift_dir = /etc/swift -user = swift -# You can specify default log routing here if you want: -# log_name = swift -# log_facility = LOG_LOCAL0 -# log_level = INFO -# log_address = /dev/log -# -# comma separated list of functions to call to setup custom log handlers. -# functions get passed: conf, name, log_to_console, log_route, fmt, logger, -# adapted_logger -# log_custom_handlers = -# -# If set, log_udp_host will override log_address -# log_udp_host = -# log_udp_port = 514 -# -# You can enable StatsD logging here: -# log_statsd_host = -# log_statsd_port = 8125 -# log_statsd_default_sample_rate = 1.0 -# log_statsd_sample_rate_factor = 1.0 -# log_statsd_metric_prefix = - -[container-reconciler] -# reclaim_age = 604800 -# interval = 300 -# request_tries = 3 - -[pipeline:main] -pipeline = catch_errors proxy-logging cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy -# See proxy-server.conf-sample for options - -[filter:cache] -use = egg:swift#memcache -# See proxy-server.conf-sample for options - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:catch_errors] -use = egg:swift#catch_errors -# See proxy-server.conf-sample for options diff --git a/saio/container/etc/swift/container-server/1.conf b/saio/container/etc/swift/container-server/1.conf deleted file mode 100644 index e709e293..00000000 --- a/saio/container/etc/swift/container-server/1.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.1 -bind_port = 8011 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon container-server - -[app:container-server] -use = egg:swift#container - -[filter:recon] -use = egg:swift#recon - -[container-replicator] -rsync_module = {replication_ip}::container{replication_port} - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/saio/container/etc/swift/container-sync-realms.conf b/saio/container/etc/swift/container-sync-realms.conf deleted file mode 100644 index c84d3391..00000000 --- a/saio/container/etc/swift/container-sync-realms.conf +++ /dev/null @@ -1,4 +0,0 @@ -[saio] -key = changeme -key2 = changeme -cluster_saio_endpoint = http://127.0.0.1:8080/v1/ diff --git a/saio/container/etc/swift/object-expirer.conf b/saio/container/etc/swift/object-expirer.conf deleted file mode 100644 index ab8a4cba..00000000 --- a/saio/container/etc/swift/object-expirer.conf +++ /dev/null @@ -1,59 +0,0 @@ -[DEFAULT] -# swift_dir = /etc/swift -user = swift -# You can specify default log routing here if you want: -log_name = object-expirer -log_facility = LOG_LOCAL6 -log_level = INFO -#log_address = /dev/log -# -# comma separated list of functions to call to setup custom log handlers. -# functions get passed: conf, name, log_to_console, log_route, fmt, logger, -# adapted_logger -# log_custom_handlers = -# -# If set, log_udp_host will override log_address -# log_udp_host = -# log_udp_port = 514 -# -# You can enable StatsD logging here: -# log_statsd_host = -# log_statsd_port = 8125 -# log_statsd_default_sample_rate = 1.0 -# log_statsd_sample_rate_factor = 1.0 -# log_statsd_metric_prefix = - -[object-expirer] -interval = 300 -# auto_create_account_prefix = . -# report_interval = 300 -# concurrency is the level of concurrency o use to do the work, this value -# must be set to at least 1 -# concurrency = 1 -# processes is how many parts to divide the work into, one part per process -# that will be doing the work -# processes set 0 means that a single process will be doing all the work -# processes can also be specified on the command line and will override the -# config value -# processes = 0 -# process is which of the parts a particular process will work on -# process can also be specified on the command line and will override the config -# value -# process is "zero based", if you want to use 3 processes, you should run -# processes with process set to 0, 1, and 2 -# process = 0 - -[pipeline:main] -pipeline = catch_errors cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy -# See proxy-server.conf-sample for options - -[filter:cache] -use = egg:swift#memcache -# See proxy-server.conf-sample for options - -[filter:catch_errors] -use = egg:swift#catch_errors -# See proxy-server.conf-sample for options diff --git a/saio/container/etc/swift/object-server/1.conf b/saio/container/etc/swift/object-server/1.conf deleted file mode 100644 index 7bda5ef4..00000000 --- a/saio/container/etc/swift/object-server/1.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.1 -bind_port = 8010 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon object-server - -[app:object-server] -use = egg:swift#object - -[filter:recon] -use = egg:swift#recon - -[object-replicator] -rsync_module = {replication_ip}::object{replication_port} - -[object-reconstructor] - -[object-updater] - -[object-auditor] diff --git a/saio/container/etc/swift/proxy-server.conf b/saio/container/etc/swift/proxy-server.conf deleted file mode 100644 index b5413f5e..00000000 --- a/saio/container/etc/swift/proxy-server.conf +++ /dev/null @@ -1,93 +0,0 @@ -[DEFAULT] -bind_ip = 0.0.0.0 -bind_port = 8080 -workers = 1 -user = swift -log_facility = LOG_LOCAL1 -eventlet_debug = true - -[pipeline:main] -# Yes, proxy-logging appears twice. This is so that -# middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync s3api tempauth staticweb copy container-quotas account-quotas slo dlo pfs versioned_writes proxy-logging proxy-server - -[filter:catch_errors] -use = egg:swift#catch_errors - -[filter:gatekeeper] -use = egg:swift#gatekeeper - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:cache] -use = egg:swift#memcache -memcache_max_connections = 1024 - -[filter:bulk] -use = egg:swift#bulk - -[filter:tempurl] -use = egg:swift#tempurl - -[filter:ratelimit] -use = egg:swift#ratelimit - -[filter:crossdomain] -use = egg:swift#crossdomain - -[filter:container_sync] -use = egg:swift#container_sync -current = //saio/saio_endpoint - -[filter:s3api] -use = egg:swift#s3api -allow_multipart_uploads = yes -check_bucket_owner = no -dns_compliant_bucket_names = yes -force_swift_request_proxy_log = yes -s3_acl = no - -[filter:tempauth] -use = egg:swift#tempauth -user_admin_admin = admin .admin .reseller_admin -user_test_tester = testing .admin -user_test2_tester2 = testing2 .admin -user_test_tester3 = testing3 - -[filter:staticweb] -use = egg:swift#staticweb - -[filter:copy] -use = egg:swift#copy - -[filter:container-quotas] -use = egg:swift#container_quotas - -[filter:account-quotas] -use = egg:swift#account_quotas - -[filter:slo] -use = egg:swift#slo - -[filter:dlo] -use = egg:swift#dlo - -# For bypass_mode, one of off (default), read-only, or read-write -[filter:pfs] -use = egg:pfs_middleware#pfs -proxyfsd_host = 127.0.0.1 -proxyfsd_port = 12345 -bypass_mode = read-write - -[filter:versioned_writes] -use = egg:swift#versioned_writes -allow_versioned_writes = true - -[app:proxy-server] -use = egg:swift#proxy -allow_account_management = true -account_autocreate = true diff --git a/saio/container/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf b/saio/container/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf deleted file mode 100644 index 55bd95c8..00000000 --- a/saio/container/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf +++ /dev/null @@ -1,45 +0,0 @@ -[DEFAULT] -bind_ip = 127.0.0.1 -bind_port = 8090 -workers = 1 -user = swift -log_facility = LOG_LOCAL1 -eventlet_debug = true - -[pipeline:main] -# Yes, proxy-logging appears twice. This is so that -# middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache copy dlo meta versioned_writes proxy-logging proxy-server - -[filter:catch_errors] -use = egg:swift#catch_errors - -[filter:gatekeeper] -use = egg:swift#gatekeeper - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:cache] -use = egg:swift#memcache - -[filter:copy] -use = egg:swift#copy - -[filter:dlo] -use = egg:swift#dlo - -[filter:meta] -use = egg:meta_middleware#meta - -[filter:versioned_writes] -use = egg:swift#versioned_writes -allow_versioned_writes = true - -[app:proxy-server] -use = egg:swift#proxy -allow_account_management = true -account_autocreate = true diff --git a/saio/container/etc/swift/swift.conf b/saio/container/etc/swift/swift.conf deleted file mode 100644 index 3022ded1..00000000 --- a/saio/container/etc/swift/swift.conf +++ /dev/null @@ -1,17 +0,0 @@ -[swift-hash] -# random unique strings that can never change (DO NOT LOSE) -# Use only printable chars (python -c "import string; print(string.printable)") -swift_hash_path_prefix = changeme -swift_hash_path_suffix = changeme - -[storage-policy:0] -name = gold -policy_type = replication -default = no -deprecated = no - -[storage-policy:1] -name = silver -policy_type = replication -default = yes -deprecated = no diff --git a/saio/container/etc/swift/test.conf b/saio/container/etc/swift/test.conf deleted file mode 100644 index 1262b80d..00000000 --- a/saio/container/etc/swift/test.conf +++ /dev/null @@ -1,112 +0,0 @@ -[func_test] -# Sample config for Swift with tempauth -auth_host = 127.0.0.1 -auth_port = 8080 -auth_ssl = no -auth_prefix = /auth/ -# Sample config for Swift with Keystone v2 API. -# For keystone v2 change auth_version to 2 and auth_prefix to /v2.0/. -# And "allow_account_management" should not be set "true". -#auth_version = 3 -#auth_host = localhost -#auth_port = 5000 -#auth_ssl = no -#auth_prefix = /v3/ - -# Primary functional test account (needs admin access to the account) -account = test -username = tester -password = testing - -# User on a second account (needs admin access to the account) -account2 = test2 -username2 = tester2 -password2 = testing2 - -# User on same account as first, but without admin access -username3 = tester3 -password3 = testing3 - -# Fourth user is required for keystone v3 specific tests. -# Account must be in a non-default domain. -#account4 = test4 -#username4 = tester4 -#password4 = testing4 -#domain4 = test-domain - -# Fifth user is required for service token-specific tests. -# The account must be different from the primary test account. -# The user must not have a group (tempauth) or role (keystoneauth) on -# the primary test account. The user must have a group/role that is unique -# and not given to the primary tester and is specified in the options -# _require_group (tempauth) or _service_roles (keystoneauth). -#account5 = test5 -#username5 = tester5 -#password5 = testing5 - -# The service_prefix option is used for service token-specific tests. -# If service_prefix or username5 above is not supplied, the tests are skipped. -# To set the value and enable the service token tests, look at the -# reseller_prefix option in /etc/swift/proxy-server.conf. There must be at -# least two prefixes. If not, add a prefix as follows (where we add SERVICE): -# reseller_prefix = AUTH, SERVICE -# The service_prefix must match the used in _require_group -# (tempauth) or _service_roles (keystoneauth); for example: -# SERVICE_require_group = service -# SERVICE_service_roles = service -# Note: Do not enable service token tests if the first prefix in -# reseller_prefix is the empty prefix AND the primary functional test -# account contains an underscore. -#service_prefix = SERVICE - -# Sixth user is required for access control tests. -# Account must have a role for reseller_admin_role(keystoneauth). -#account6 = test -#username6 = tester6 -#password6 = testing6 - -collate = C - -# Only necessary if a pre-existing server uses self-signed certificate -insecure = no - -[unit_test] -fake_syslog = False - -[probe_test] -# check_server_timeout = 30 -# validate_rsync = false - -[swift-constraints] -# The functional test runner will try to use the constraint values provided in -# the swift-constraints section of test.conf. -# -# If a constraint value does not exist in that section, or because the -# swift-constraints section does not exist, the constraints values found in -# the /info API call (if successful) will be used. -# -# If a constraint value cannot be found in the /info results, either because -# the /info API call failed, or a value is not present, the constraint value -# used will fall back to those loaded by the constraints module at time of -# import (which will attempt to load /etc/swift/swift.conf, see the -# swift.common.constraints module for more information). -# -# Note that the cluster must have "sane" values for the test suite to pass -# (for some definition of sane). -# -#max_file_size = 5368709122 -#max_meta_name_length = 128 -#max_meta_value_length = 256 -#max_meta_count = 90 -#max_meta_overall_size = 4096 -#max_header_size = 8192 -#extra_header_count = 0 -#max_object_name_length = 1024 -#container_listing_limit = 10000 -#account_listing_limit = 10000 -#max_account_name_length = 256 -#max_container_name_length = 256 - -# Newer swift versions default to strict cors mode, but older ones were the -# opposite. -#strict_cors_mode = true diff --git a/saio/container/proxyfs.conf b/saio/container/proxyfs.conf deleted file mode 100644 index e2ed1596..00000000 --- a/saio/container/proxyfs.conf +++ /dev/null @@ -1,232 +0,0 @@ -# Single peer .conf file customized for ProxyFS in a container - -[Peer:Peer0] -PublicIPAddr: # To be set by docker_startup.sh -PrivateIPAddr: 0.0.0.0 # So that docker run -p can expose ports on PrivateIPAddr -ReadCacheQuotaFraction: 0.20 - -[Cluster] -WhoAmI: Peer0 -Peers: Peer0 -ServerGuid: 0bb51164-258f-4e04-a417-e16d736ca41c -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 - -[SwiftClient] -NoAuthIPAddr: 127.0.0.1 -NoAuthTCPPort: 8090 - -RetryDelay: 1s -RetryExpBackoff: 1.5 -RetryLimit: 1 - -RetryDelayObject: 1s -RetryExpBackoffObject: 1.95 -RetryLimitObject: 8 - -ChunkedConnectionPoolSize: 512 -NonChunkedConnectionPoolSize: 128 - -SwiftReconNoWriteThreshold: 80 -SwiftReconNoWriteErrno: ENOSPC -SwiftReconReadOnlyThreshold: 90 -SwiftReconReadOnlyErrno: EROFS -SwiftConfDir: /etc/swift -SwiftReconChecksPerConfCheck: 10 - -[PhysicalContainerLayout:CommonVolumePhysicalContainerLayoutReplicated3Way] -ContainerStoragePolicy: silver # bronze -ContainerNamePrefix: Replicated3Way_ # ErasureCoded_ -ContainersPerPeer: 10 -MaxObjectsPerContainer: 1000000 - -[SnapShotSchedule:MinutelySnapShotSchedule] -CronTab: * * * * * # Every minute -Keep: 59 - -[SnapShotSchedule:HourlySnapShotSchedule] -CronTab: 0 * * * * # At the top of every hour -Keep: 23 - -[SnapShotSchedule:DailySnapShotSchedule] -CronTab: 0 0 * * * # At midnight every day -Keep: 6 - -[SnapShotSchedule:WeeklySnapShotSchedule] -CronTab: 0 0 * * 0 # At midnight every Sunday -Keep: 8 - -[SnapShotSchedule:MonthlySnapShotSchedule] -CronTab: 0 0 1 * * # At midnight on the first of every month -Keep: 11 - -[SnapShotSchedule:YearlySnapShotSchedule] -CronTab: 0 0 1 1 * # At midnight on the January 1st of every year -Keep: 4 - -[SnapShotPolicy:CommonSnapShotPolicy] -ScheduleList: MinutelySnapShotSchedule,HourlySnapShotSchedule,DailySnapShotSchedule,WeeklySnapShotSchedule,MonthlySnapShotSchedule,YearlySnapShotSchedule -TimeZone: America/Los_Angeles - -[SnapShotSchedule:TestSnapShotSchedule] -CronTab: * * * * * # Every minute -Keep: 3 - -[SnapShotPolicy:TestSnapShotPolicy] -ScheduleList: TestSnapShotSchedule -TimeZone: America/Los_Angeles - -[Volume:CommonVolume] -FSID: 1 -FUSEMountPointName: CommonMountPoint -NFSExportClientMapList: CommonVolumeNFSClient0 -SMBShareName: CommonShare -AccountName: AUTH_test -AutoFormat: true -NonceValuesToReserve: 100 -MaxEntriesPerDirNode: 32 -MaxExtentsPerFileNode: 32 -MaxInodesPerMetadataNode: 32 -MaxLogSegmentsPerMetadataNode: 64 -MaxDirFileNodesPerMetadataNode: 16 -MaxCreatedDeletedObjectsPerMetadataNode: 256 -CheckpointEtcdKeyName: ProxyFS:Volume:CommonVolume:Checkpoint -CheckpointContainerName: .__checkpoint__ -CheckpointContainerStoragePolicy: gold -CheckpointInterval: 10s -#ReplayLogFileName: CommonVolume.rlog -DefaultPhysicalContainerLayout: CommonVolumePhysicalContainerLayoutReplicated3Way -MaxFlushSize: 10485760 -MaxFlushTime: 10s -FileDefragmentChunkSize: 10485760 -FileDefragmentChunkDelay: 10ms -ReportedBlockSize: 65536 -ReportedFragmentSize: 65536 -ReportedNumBlocks: 1677721600 -ReportedNumInodes: 107374182400 -SnapShotIDNumBits: 10 -MaxBytesInodeCache: 10485760 -InodeCacheEvictInterval: 1s -#SnapShotPolicy: CommonSnapShotPolicy # Optional -#SnapShotPolicy: TestSnapShotPolicy -SMBValidUserList: swift -SMBBrowseable: true -SMBStrictSync: yes -SMBAuditLogging: false -SMBEncryptionRequired: false -ActiveLeaseEvictLowLimit: 5000 -ActiveLeaseEvictHighLimit: 5010 - -[NFSClientMap:CommonVolumeNFSClient0] -ClientPattern: * -AccessMode: rw -RootSquash: no_root_squash -Secure: insecure - -[VolumeGroup:CommonVolumeGroup] -VolumeList: CommonVolume -VirtualIPAddr: -PrimaryPeer: Peer0 -ReadCacheLineSize: 1048576 -ReadCacheWeight: 100 -SMBWorkgroup: # If missing or blank, defaults to WORKGROUP -SMBActiveDirectoryEnabled: false # If true, all other SMBActiveDirectory* Key:Values are required (defaults to false) -SMBActiveDirectoryRealm: -SMBActiveDirectoryIDMapDefaultMin: -SMBActiveDirectoryIDMapDefaultMax: -SMBActiveDirectoryIDMapWorkgroupMin: -SMBActiveDirectoryIDMapWorkgroupMax: - -[FSGlobals] -VolumeGroupList: CommonVolumeGroup -CheckpointHeaderConsensusAttempts: 5 -MountRetryLimit: 6 -MountRetryDelay: 1s -MountRetryExpBackoff: 2 -LogCheckpointHeaderPosts: true -TryLockBackoffMin: 10ms -TryLockBackoffMax: 50ms -TryLockSerializationThreshhold: 5 -SymlinkMax: 32 -CoalesceElementChunkSize: 16 -InodeRecCacheEvictLowLimit: 10000 -InodeRecCacheEvictHighLimit: 10010 -LogSegmentRecCacheEvictLowLimit: 10000 -LogSegmentRecCacheEvictHighLimit: 10010 -BPlusTreeObjectCacheEvictLowLimit: 10000 -BPlusTreeObjectCacheEvictHighLimit: 10010 -CreatedDeletedObjectsCacheEvictLowLimit: 10000 -CreatedDeletedObjectsCacheEvictHighLimit: 10010 -DirEntryCacheEvictLowLimit: 10000 -DirEntryCacheEvictHighLimit: 10010 -FileExtentMapEvictLowLimit: 10000 -FileExtentMapEvictHighLimit: 10010 -EtcdEnabled: false -EtcdEndpoints: 127.0.0.1:2379 -EtcdAutoSyncInterval: 1m -EtcdCertDir: /etc/ssl/etcd/ssl/ -EtcdDialTimeout: 10s -EtcdOpTimeout: 20s -MetadataRecycleBin: false -SMBUserList: swift -SMBMapToGuest: # One of Never, Bad User, Bad Password, or Bad Uid (case insensitive)... defaults to Never -SMBNetBiosName: # Defaults to `hostname -s` (i.e. short host name) - -[SMBUsers] -swift: c3dpZnQ= # base64.standard_b64encode("swift") - -[JSONRPCServer] -TCPPort: 12345 -FastTCPPort: 32345 -DataPathLogging: false -Debug: false -RetryRPCPort: 32356 -RetryRPCTTLCompleted: 10m -RetryRPCAckTrim: 100ms -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s -RetryRPCCertFilePath: -RetryRPCKeyFilePath: -MinLeaseDuration: 250ms -LeaseInterruptInterval: 250ms -LeaseInterruptLimit: 20 - -[Logging] -LogFilePath: proxyfsd.log -TraceLevelLogging: none -DebugLevelLogging: none -LogToConsole: false - -[EventLog] -Enabled: false -BufferKey: 1234 -BufferLength: 1048576 # 1MiB -MinBackoff: 1us -MaxBackoff: 2us -DaemonPollDelay: 10ms -DaemonOutputPath: # If blank, os.Stdout is used - -[Stats] -UDPPort: 48125 # To use nc to test, use "TCPPort" rather than UDPPort. -BufferLength: 1000 -MaxLatency: 1s - -[StatsLogger] -Period: 10m -Verbose: true - -[HTTPServer] -TCPPort: 15346 -JobHistoryMaxSize: 5 - -[ProxyfsDebug] -ProfileType: None -DebugServerPort: 6060 diff --git a/saio/etc/exports b/saio/etc/exports deleted file mode 100644 index 42fe7f16..00000000 --- a/saio/etc/exports +++ /dev/null @@ -1 +0,0 @@ -/CommonMountPoint *(rw,sync,fsid=1000,no_subtree_check,no_root_squash,insecure) diff --git a/saio/etc/samba/smb.conf b/saio/etc/samba/smb.conf deleted file mode 100644 index f446ba92..00000000 --- a/saio/etc/samba/smb.conf +++ /dev/null @@ -1,25 +0,0 @@ -[global] - security = user - passdb backend = tdbsam - proxyfs:PrivateIPAddr = 127.0.0.1 - proxyfs:TCPPort = 12345 - proxyfs:FastTCPPort = 32345 - -[proxyfs] - comment = ProxyFS test volume - path = /mnt/CommonVolume - vfs objects = proxyfs - proxyfs:volume = CommonVolume - valid users = swift - writable = yes - printable = no - browseable = yes - read only = no - oplocks = False - level2 oplocks = False - aio read size = 1 - aio write size = 1 - case sensitive = yes - preserve case = yes - short preserve case = yes - strict sync = yes diff --git a/saio/etc/swift/account-server/1.conf b/saio/etc/swift/account-server/1.conf deleted file mode 100644 index 093be13a..00000000 --- a/saio/etc/swift/account-server/1.conf +++ /dev/null @@ -1,27 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.1 -bind_port = 8012 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon account-server - -[app:account-server] -use = egg:swift#account - -[filter:recon] -use = egg:swift#recon - -[account-replicator] -rsync_module = {replication_ip}::account{replication_port} - -[account-auditor] - -[account-reaper] diff --git a/saio/etc/swift/account-server/2.conf b/saio/etc/swift/account-server/2.conf deleted file mode 100644 index 8d3e1dfc..00000000 --- a/saio/etc/swift/account-server/2.conf +++ /dev/null @@ -1,27 +0,0 @@ -[DEFAULT] -devices = /srv/2/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.2 -bind_port = 8022 -workers = 1 -user = swift -log_facility = LOG_LOCAL3 -recon_cache_path = /var/cache/swift2 -eventlet_debug = true - -[pipeline:main] -pipeline = recon account-server - -[app:account-server] -use = egg:swift#account - -[filter:recon] -use = egg:swift#recon - -[account-replicator] -rsync_module = {replication_ip}::account{replication_port} - -[account-auditor] - -[account-reaper] diff --git a/saio/etc/swift/account-server/3.conf b/saio/etc/swift/account-server/3.conf deleted file mode 100644 index aca1c3f7..00000000 --- a/saio/etc/swift/account-server/3.conf +++ /dev/null @@ -1,27 +0,0 @@ -[DEFAULT] -devices = /srv/3/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.3 -bind_port = 8032 -workers = 1 -user = swift -log_facility = LOG_LOCAL4 -recon_cache_path = /var/cache/swift3 -eventlet_debug = true - -[pipeline:main] -pipeline = recon account-server - -[app:account-server] -use = egg:swift#account - -[filter:recon] -use = egg:swift#recon - -[account-replicator] -rsync_module = {replication_ip}::account{replication_port} - -[account-auditor] - -[account-reaper] diff --git a/saio/etc/swift/account-server/4.conf b/saio/etc/swift/account-server/4.conf deleted file mode 100644 index d478dde4..00000000 --- a/saio/etc/swift/account-server/4.conf +++ /dev/null @@ -1,27 +0,0 @@ -[DEFAULT] -devices = /srv/4/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.4 -bind_port = 8042 -workers = 1 -user = swift -log_facility = LOG_LOCAL5 -recon_cache_path = /var/cache/swift4 -eventlet_debug = true - -[pipeline:main] -pipeline = recon account-server - -[app:account-server] -use = egg:swift#account - -[filter:recon] -use = egg:swift#recon - -[account-replicator] -rsync_module = {replication_ip}::account{replication_port} - -[account-auditor] - -[account-reaper] diff --git a/saio/etc/swift/container-reconciler.conf b/saio/etc/swift/container-reconciler.conf deleted file mode 100644 index 30f81ca5..00000000 --- a/saio/etc/swift/container-reconciler.conf +++ /dev/null @@ -1,47 +0,0 @@ -[DEFAULT] -# swift_dir = /etc/swift -user = swift -# You can specify default log routing here if you want: -# log_name = swift -# log_facility = LOG_LOCAL0 -# log_level = INFO -# log_address = /dev/log -# -# comma separated list of functions to call to setup custom log handlers. -# functions get passed: conf, name, log_to_console, log_route, fmt, logger, -# adapted_logger -# log_custom_handlers = -# -# If set, log_udp_host will override log_address -# log_udp_host = -# log_udp_port = 514 -# -# You can enable StatsD logging here: -# log_statsd_host = -# log_statsd_port = 8125 -# log_statsd_default_sample_rate = 1.0 -# log_statsd_sample_rate_factor = 1.0 -# log_statsd_metric_prefix = - -[container-reconciler] -# reclaim_age = 604800 -# interval = 300 -# request_tries = 3 - -[pipeline:main] -pipeline = catch_errors proxy-logging cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy -# See proxy-server.conf-sample for options - -[filter:cache] -use = egg:swift#memcache -# See proxy-server.conf-sample for options - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:catch_errors] -use = egg:swift#catch_errors -# See proxy-server.conf-sample for options diff --git a/saio/etc/swift/container-server/1.conf b/saio/etc/swift/container-server/1.conf deleted file mode 100644 index e709e293..00000000 --- a/saio/etc/swift/container-server/1.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.1 -bind_port = 8011 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon container-server - -[app:container-server] -use = egg:swift#container - -[filter:recon] -use = egg:swift#recon - -[container-replicator] -rsync_module = {replication_ip}::container{replication_port} - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/saio/etc/swift/container-server/2.conf b/saio/etc/swift/container-server/2.conf deleted file mode 100644 index dd3c4623..00000000 --- a/saio/etc/swift/container-server/2.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/2/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.2 -bind_port = 8021 -workers = 1 -user = swift -log_facility = LOG_LOCAL3 -recon_cache_path = /var/cache/swift2 -eventlet_debug = true - -[pipeline:main] -pipeline = recon container-server - -[app:container-server] -use = egg:swift#container - -[filter:recon] -use = egg:swift#recon - -[container-replicator] -rsync_module = {replication_ip}::container{replication_port} - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/saio/etc/swift/container-server/3.conf b/saio/etc/swift/container-server/3.conf deleted file mode 100644 index 2ffc5e6c..00000000 --- a/saio/etc/swift/container-server/3.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/3/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.3 -bind_port = 8031 -workers = 1 -user = swift -log_facility = LOG_LOCAL4 -recon_cache_path = /var/cache/swift3 -eventlet_debug = true - -[pipeline:main] -pipeline = recon container-server - -[app:container-server] -use = egg:swift#container - -[filter:recon] -use = egg:swift#recon - -[container-replicator] -rsync_module = {replication_ip}::container{replication_port} - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/saio/etc/swift/container-server/4.conf b/saio/etc/swift/container-server/4.conf deleted file mode 100644 index 2a6539b7..00000000 --- a/saio/etc/swift/container-server/4.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/4/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.4 -bind_port = 8041 -workers = 1 -user = swift -log_facility = LOG_LOCAL5 -recon_cache_path = /var/cache/swift4 -eventlet_debug = true - -[pipeline:main] -pipeline = recon container-server - -[app:container-server] -use = egg:swift#container - -[filter:recon] -use = egg:swift#recon - -[container-replicator] -rsync_module = {replication_ip}::container{replication_port} - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/saio/etc/swift/container-sync-realms.conf b/saio/etc/swift/container-sync-realms.conf deleted file mode 100644 index c84d3391..00000000 --- a/saio/etc/swift/container-sync-realms.conf +++ /dev/null @@ -1,4 +0,0 @@ -[saio] -key = changeme -key2 = changeme -cluster_saio_endpoint = http://127.0.0.1:8080/v1/ diff --git a/saio/etc/swift/object-expirer.conf b/saio/etc/swift/object-expirer.conf deleted file mode 100644 index ab8a4cba..00000000 --- a/saio/etc/swift/object-expirer.conf +++ /dev/null @@ -1,59 +0,0 @@ -[DEFAULT] -# swift_dir = /etc/swift -user = swift -# You can specify default log routing here if you want: -log_name = object-expirer -log_facility = LOG_LOCAL6 -log_level = INFO -#log_address = /dev/log -# -# comma separated list of functions to call to setup custom log handlers. -# functions get passed: conf, name, log_to_console, log_route, fmt, logger, -# adapted_logger -# log_custom_handlers = -# -# If set, log_udp_host will override log_address -# log_udp_host = -# log_udp_port = 514 -# -# You can enable StatsD logging here: -# log_statsd_host = -# log_statsd_port = 8125 -# log_statsd_default_sample_rate = 1.0 -# log_statsd_sample_rate_factor = 1.0 -# log_statsd_metric_prefix = - -[object-expirer] -interval = 300 -# auto_create_account_prefix = . -# report_interval = 300 -# concurrency is the level of concurrency o use to do the work, this value -# must be set to at least 1 -# concurrency = 1 -# processes is how many parts to divide the work into, one part per process -# that will be doing the work -# processes set 0 means that a single process will be doing all the work -# processes can also be specified on the command line and will override the -# config value -# processes = 0 -# process is which of the parts a particular process will work on -# process can also be specified on the command line and will override the config -# value -# process is "zero based", if you want to use 3 processes, you should run -# processes with process set to 0, 1, and 2 -# process = 0 - -[pipeline:main] -pipeline = catch_errors cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy -# See proxy-server.conf-sample for options - -[filter:cache] -use = egg:swift#memcache -# See proxy-server.conf-sample for options - -[filter:catch_errors] -use = egg:swift#catch_errors -# See proxy-server.conf-sample for options diff --git a/saio/etc/swift/object-server/1.conf b/saio/etc/swift/object-server/1.conf deleted file mode 100644 index 7bda5ef4..00000000 --- a/saio/etc/swift/object-server/1.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.1 -bind_port = 8010 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon object-server - -[app:object-server] -use = egg:swift#object - -[filter:recon] -use = egg:swift#recon - -[object-replicator] -rsync_module = {replication_ip}::object{replication_port} - -[object-reconstructor] - -[object-updater] - -[object-auditor] diff --git a/saio/etc/swift/object-server/2.conf b/saio/etc/swift/object-server/2.conf deleted file mode 100644 index b9d41877..00000000 --- a/saio/etc/swift/object-server/2.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/2/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.2 -bind_port = 8020 -workers = 1 -user = swift -log_facility = LOG_LOCAL3 -recon_cache_path = /var/cache/swift2 -eventlet_debug = true - -[pipeline:main] -pipeline = recon object-server - -[app:object-server] -use = egg:swift#object - -[filter:recon] -use = egg:swift#recon - -[object-replicator] -rsync_module = {replication_ip}::object{replication_port} - -[object-reconstructor] - -[object-updater] - -[object-auditor] diff --git a/saio/etc/swift/object-server/3.conf b/saio/etc/swift/object-server/3.conf deleted file mode 100644 index 1f422cf7..00000000 --- a/saio/etc/swift/object-server/3.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/3/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.3 -bind_port = 8030 -workers = 1 -user = swift -log_facility = LOG_LOCAL4 -recon_cache_path = /var/cache/swift3 -eventlet_debug = true - -[pipeline:main] -pipeline = recon object-server - -[app:object-server] -use = egg:swift#object - -[filter:recon] -use = egg:swift#recon - -[object-replicator] -rsync_module = {replication_ip}::object{replication_port} - -[object-reconstructor] - -[object-updater] - -[object-auditor] diff --git a/saio/etc/swift/object-server/4.conf b/saio/etc/swift/object-server/4.conf deleted file mode 100644 index 8a71312c..00000000 --- a/saio/etc/swift/object-server/4.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/4/node -mount_check = false -disable_fallocate = true -bind_ip = 127.0.0.4 -bind_port = 8040 -workers = 1 -user = swift -log_facility = LOG_LOCAL5 -recon_cache_path = /var/cache/swift4 -eventlet_debug = true - -[pipeline:main] -pipeline = recon object-server - -[app:object-server] -use = egg:swift#object - -[filter:recon] -use = egg:swift#recon - -[object-replicator] -rsync_module = {replication_ip}::object{replication_port} - -[object-reconstructor] - -[object-updater] - -[object-auditor] diff --git a/saio/etc/swift/proxy-server.conf b/saio/etc/swift/proxy-server.conf deleted file mode 100644 index b5413f5e..00000000 --- a/saio/etc/swift/proxy-server.conf +++ /dev/null @@ -1,93 +0,0 @@ -[DEFAULT] -bind_ip = 0.0.0.0 -bind_port = 8080 -workers = 1 -user = swift -log_facility = LOG_LOCAL1 -eventlet_debug = true - -[pipeline:main] -# Yes, proxy-logging appears twice. This is so that -# middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync s3api tempauth staticweb copy container-quotas account-quotas slo dlo pfs versioned_writes proxy-logging proxy-server - -[filter:catch_errors] -use = egg:swift#catch_errors - -[filter:gatekeeper] -use = egg:swift#gatekeeper - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:cache] -use = egg:swift#memcache -memcache_max_connections = 1024 - -[filter:bulk] -use = egg:swift#bulk - -[filter:tempurl] -use = egg:swift#tempurl - -[filter:ratelimit] -use = egg:swift#ratelimit - -[filter:crossdomain] -use = egg:swift#crossdomain - -[filter:container_sync] -use = egg:swift#container_sync -current = //saio/saio_endpoint - -[filter:s3api] -use = egg:swift#s3api -allow_multipart_uploads = yes -check_bucket_owner = no -dns_compliant_bucket_names = yes -force_swift_request_proxy_log = yes -s3_acl = no - -[filter:tempauth] -use = egg:swift#tempauth -user_admin_admin = admin .admin .reseller_admin -user_test_tester = testing .admin -user_test2_tester2 = testing2 .admin -user_test_tester3 = testing3 - -[filter:staticweb] -use = egg:swift#staticweb - -[filter:copy] -use = egg:swift#copy - -[filter:container-quotas] -use = egg:swift#container_quotas - -[filter:account-quotas] -use = egg:swift#account_quotas - -[filter:slo] -use = egg:swift#slo - -[filter:dlo] -use = egg:swift#dlo - -# For bypass_mode, one of off (default), read-only, or read-write -[filter:pfs] -use = egg:pfs_middleware#pfs -proxyfsd_host = 127.0.0.1 -proxyfsd_port = 12345 -bypass_mode = read-write - -[filter:versioned_writes] -use = egg:swift#versioned_writes -allow_versioned_writes = true - -[app:proxy-server] -use = egg:swift#proxy -allow_account_management = true -account_autocreate = true diff --git a/saio/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf b/saio/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf deleted file mode 100644 index 7e7cf83e..00000000 --- a/saio/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf +++ /dev/null @@ -1,45 +0,0 @@ -[DEFAULT] -bind_ip = 0.0.0.0 -bind_port = 8090 -workers = 1 -user = swift -log_facility = LOG_LOCAL1 -eventlet_debug = true - -[pipeline:main] -# Yes, proxy-logging appears twice. This is so that -# middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache copy dlo meta versioned_writes proxy-logging proxy-server - -[filter:catch_errors] -use = egg:swift#catch_errors - -[filter:gatekeeper] -use = egg:swift#gatekeeper - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:cache] -use = egg:swift#memcache - -[filter:copy] -use = egg:swift#copy - -[filter:dlo] -use = egg:swift#dlo - -[filter:meta] -use = egg:meta_middleware#meta - -[filter:versioned_writes] -use = egg:swift#versioned_writes -allow_versioned_writes = true - -[app:proxy-server] -use = egg:swift#proxy -allow_account_management = true -account_autocreate = true diff --git a/saio/etc/swift/swift.conf b/saio/etc/swift/swift.conf deleted file mode 100644 index e5a8b1d9..00000000 --- a/saio/etc/swift/swift.conf +++ /dev/null @@ -1,29 +0,0 @@ -[swift-hash] -# random unique strings that can never change (DO NOT LOSE) -# Use only printable chars (python -c "import string; print(string.printable)") -swift_hash_path_prefix = changeme -swift_hash_path_suffix = changeme - -[storage-policy:0] -name = gold -policy_type = replication -default = no -deprecated = no - -[storage-policy:1] -name = silver -policy_type = replication -default = yes -deprecated = no - -[storage-policy:2] -name = bronze -policy_type = erasure_coding -# ec_type = isa_l_rs_cauchy -ec_type = liberasurecode_rs_vand -ec_num_data_fragments = 8 -ec_num_parity_fragments = 4 -ec_object_segment_size = 1048576 -# ec_duplication_factor = 1 -default = no -deprecated = no diff --git a/saio/etc/swift/test.conf b/saio/etc/swift/test.conf deleted file mode 100644 index 1262b80d..00000000 --- a/saio/etc/swift/test.conf +++ /dev/null @@ -1,112 +0,0 @@ -[func_test] -# Sample config for Swift with tempauth -auth_host = 127.0.0.1 -auth_port = 8080 -auth_ssl = no -auth_prefix = /auth/ -# Sample config for Swift with Keystone v2 API. -# For keystone v2 change auth_version to 2 and auth_prefix to /v2.0/. -# And "allow_account_management" should not be set "true". -#auth_version = 3 -#auth_host = localhost -#auth_port = 5000 -#auth_ssl = no -#auth_prefix = /v3/ - -# Primary functional test account (needs admin access to the account) -account = test -username = tester -password = testing - -# User on a second account (needs admin access to the account) -account2 = test2 -username2 = tester2 -password2 = testing2 - -# User on same account as first, but without admin access -username3 = tester3 -password3 = testing3 - -# Fourth user is required for keystone v3 specific tests. -# Account must be in a non-default domain. -#account4 = test4 -#username4 = tester4 -#password4 = testing4 -#domain4 = test-domain - -# Fifth user is required for service token-specific tests. -# The account must be different from the primary test account. -# The user must not have a group (tempauth) or role (keystoneauth) on -# the primary test account. The user must have a group/role that is unique -# and not given to the primary tester and is specified in the options -# _require_group (tempauth) or _service_roles (keystoneauth). -#account5 = test5 -#username5 = tester5 -#password5 = testing5 - -# The service_prefix option is used for service token-specific tests. -# If service_prefix or username5 above is not supplied, the tests are skipped. -# To set the value and enable the service token tests, look at the -# reseller_prefix option in /etc/swift/proxy-server.conf. There must be at -# least two prefixes. If not, add a prefix as follows (where we add SERVICE): -# reseller_prefix = AUTH, SERVICE -# The service_prefix must match the used in _require_group -# (tempauth) or _service_roles (keystoneauth); for example: -# SERVICE_require_group = service -# SERVICE_service_roles = service -# Note: Do not enable service token tests if the first prefix in -# reseller_prefix is the empty prefix AND the primary functional test -# account contains an underscore. -#service_prefix = SERVICE - -# Sixth user is required for access control tests. -# Account must have a role for reseller_admin_role(keystoneauth). -#account6 = test -#username6 = tester6 -#password6 = testing6 - -collate = C - -# Only necessary if a pre-existing server uses self-signed certificate -insecure = no - -[unit_test] -fake_syslog = False - -[probe_test] -# check_server_timeout = 30 -# validate_rsync = false - -[swift-constraints] -# The functional test runner will try to use the constraint values provided in -# the swift-constraints section of test.conf. -# -# If a constraint value does not exist in that section, or because the -# swift-constraints section does not exist, the constraints values found in -# the /info API call (if successful) will be used. -# -# If a constraint value cannot be found in the /info results, either because -# the /info API call failed, or a value is not present, the constraint value -# used will fall back to those loaded by the constraints module at time of -# import (which will attempt to load /etc/swift/swift.conf, see the -# swift.common.constraints module for more information). -# -# Note that the cluster must have "sane" values for the test suite to pass -# (for some definition of sane). -# -#max_file_size = 5368709122 -#max_meta_name_length = 128 -#max_meta_value_length = 256 -#max_meta_count = 90 -#max_meta_overall_size = 4096 -#max_header_size = 8192 -#extra_header_count = 0 -#max_object_name_length = 1024 -#container_listing_limit = 10000 -#account_listing_limit = 10000 -#max_account_name_length = 256 -#max_container_name_length = 256 - -# Newer swift versions default to strict cors mode, but older ones were the -# opposite. -#strict_cors_mode = true diff --git a/saio/home/swift/bin/remakerings b/saio/home/swift/bin/remakerings deleted file mode 100755 index b890e3a7..00000000 --- a/saio/home/swift/bin/remakerings +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -e - -cd /etc/swift - -rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz - -swift-ring-builder object.builder create 10 1 1 -swift-ring-builder object.builder add r1z1-127.0.0.1:8010/sdb1 1 -swift-ring-builder object.builder add r1z1-127.0.0.1:8010/sdb5 1 -swift-ring-builder object.builder add r1z1-127.0.0.1:8010/sdb9 1 -swift-ring-builder object.builder add r1z2-127.0.0.2:8020/sdb2 1 -swift-ring-builder object.builder add r1z2-127.0.0.2:8020/sdb6 1 -swift-ring-builder object.builder add r1z2-127.0.0.2:8020/sdbA 1 -swift-ring-builder object.builder add r1z3-127.0.0.3:8030/sdb3 1 -swift-ring-builder object.builder add r1z3-127.0.0.3:8030/sdb7 1 -swift-ring-builder object.builder add r1z3-127.0.0.3:8030/sdbB 1 -swift-ring-builder object.builder add r1z4-127.0.0.4:8040/sdb4 1 -swift-ring-builder object.builder add r1z4-127.0.0.4:8040/sdb8 1 -swift-ring-builder object.builder add r1z4-127.0.0.4:8040/sdbC 1 -swift-ring-builder object.builder rebalance -swift-ring-builder object-1.builder create 10 1 1 -swift-ring-builder object-1.builder add r1z1-127.0.0.1:8010/sdb1 1 -swift-ring-builder object-1.builder add r1z1-127.0.0.1:8010/sdb5 1 -swift-ring-builder object-1.builder add r1z1-127.0.0.1:8010/sdb9 1 -swift-ring-builder object-1.builder add r1z2-127.0.0.2:8020/sdb2 1 -swift-ring-builder object-1.builder add r1z2-127.0.0.2:8020/sdb6 1 -swift-ring-builder object-1.builder add r1z2-127.0.0.2:8020/sdbA 1 -swift-ring-builder object-1.builder add r1z3-127.0.0.3:8030/sdb3 1 -swift-ring-builder object-1.builder add r1z3-127.0.0.3:8030/sdb7 1 -swift-ring-builder object-1.builder add r1z3-127.0.0.3:8030/sdbB 1 -swift-ring-builder object-1.builder add r1z4-127.0.0.4:8040/sdb4 1 -swift-ring-builder object-1.builder add r1z4-127.0.0.4:8040/sdb8 1 -swift-ring-builder object-1.builder add r1z4-127.0.0.4:8040/sdbC 1 -swift-ring-builder object-1.builder rebalance -swift-ring-builder object-2.builder create 10 12 1 -swift-ring-builder object-2.builder add r1z1-127.0.0.1:8010/sdb1 1 -swift-ring-builder object-2.builder add r1z1-127.0.0.1:8010/sdb5 1 -swift-ring-builder object-2.builder add r1z1-127.0.0.1:8010/sdb9 1 -swift-ring-builder object-2.builder add r1z2-127.0.0.2:8020/sdb2 1 -swift-ring-builder object-2.builder add r1z2-127.0.0.2:8020/sdb6 1 -swift-ring-builder object-2.builder add r1z2-127.0.0.2:8020/sdbA 1 -swift-ring-builder object-2.builder add r1z3-127.0.0.3:8030/sdb3 1 -swift-ring-builder object-2.builder add r1z3-127.0.0.3:8030/sdb7 1 -swift-ring-builder object-2.builder add r1z3-127.0.0.3:8030/sdbB 1 -swift-ring-builder object-2.builder add r1z4-127.0.0.4:8040/sdb4 1 -swift-ring-builder object-2.builder add r1z4-127.0.0.4:8040/sdb8 1 -swift-ring-builder object-2.builder add r1z4-127.0.0.4:8040/sdbC 1 -swift-ring-builder object-2.builder rebalance -swift-ring-builder container.builder create 10 1 1 -swift-ring-builder container.builder add r1z1-127.0.0.1:8011/sdb1 1 -swift-ring-builder container.builder add r1z1-127.0.0.1:8011/sdb5 1 -swift-ring-builder container.builder add r1z1-127.0.0.1:8011/sdb9 1 -swift-ring-builder container.builder add r1z2-127.0.0.2:8021/sdb2 1 -swift-ring-builder container.builder add r1z2-127.0.0.2:8021/sdb6 1 -swift-ring-builder container.builder add r1z2-127.0.0.2:8021/sdbA 1 -swift-ring-builder container.builder add r1z3-127.0.0.3:8031/sdb3 1 -swift-ring-builder container.builder add r1z3-127.0.0.3:8031/sdb7 1 -swift-ring-builder container.builder add r1z3-127.0.0.3:8031/sdbB 1 -swift-ring-builder container.builder add r1z4-127.0.0.4:8041/sdb4 1 -swift-ring-builder container.builder add r1z4-127.0.0.4:8041/sdb8 1 -swift-ring-builder container.builder add r1z4-127.0.0.4:8041/sdbC 1 -swift-ring-builder container.builder rebalance -swift-ring-builder account.builder create 10 1 1 -swift-ring-builder account.builder add r1z1-127.0.0.1:8012/sdb1 1 -swift-ring-builder account.builder add r1z1-127.0.0.1:8012/sdb5 1 -swift-ring-builder account.builder add r1z1-127.0.0.1:8012/sdb9 1 -swift-ring-builder account.builder add r1z2-127.0.0.2:8022/sdb2 1 -swift-ring-builder account.builder add r1z2-127.0.0.2:8022/sdb6 1 -swift-ring-builder account.builder add r1z2-127.0.0.2:8022/sdbA 1 -swift-ring-builder account.builder add r1z3-127.0.0.3:8032/sdb3 1 -swift-ring-builder account.builder add r1z3-127.0.0.3:8032/sdb7 1 -swift-ring-builder account.builder add r1z3-127.0.0.3:8032/sdbB 1 -swift-ring-builder account.builder add r1z4-127.0.0.4:8042/sdb4 1 -swift-ring-builder account.builder add r1z4-127.0.0.4:8042/sdb8 1 -swift-ring-builder account.builder add r1z4-127.0.0.4:8042/sdbC 1 -swift-ring-builder account.builder rebalance diff --git a/saio/home/swift/bin/resetswift b/saio/home/swift/bin/resetswift deleted file mode 100755 index b39d1d2e..00000000 --- a/saio/home/swift/bin/resetswift +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -e - -sudo swift-init all kill - -for x in 11 22 33 44 15 26 37 48 19 2A 3B 4C -do - node=${x:0:1} - drive=${x:1:1} - if cut -d' ' -f2 /proc/mounts | grep -q /srv/$node/node/sdb$drive - then - sudo umount /srv/$node/node/sdb$drive - fi - sudo truncate -s 0 /srv/swift-disk-$drive - sudo truncate -s 1GB /srv/swift-disk-$drive - sudo mkfs.xfs -f /srv/swift-disk-$drive - sudo mount /srv/$node/node/sdb$drive - sudo chown swift:swift /srv/$node/node/sdb$drive -done - -sudo rm -f /var/log/debug /var/log/messages /var/log/rsyncd.log /var/log/syslog -find /var/cache/swift* -type f -name *.recon -exec rm -f {} \; -sudo systemctl restart memcached diff --git a/saio/proxyfs.conf b/saio/proxyfs.conf deleted file mode 100644 index 3a7edcd8..00000000 --- a/saio/proxyfs.conf +++ /dev/null @@ -1,38 +0,0 @@ -# Single peer .conf file customized for SAIO for ProxyFS VM (also includes ramswift info) - -[Peer:Peer0] -PublicIPAddr: 0.0.0.0 -PrivateIPAddr: 0.0.0.0 -ReadCacheQuotaFraction: 0.20 - -[Cluster] -WhoAmI: Peer0 -Peers: Peer0 -ServerGuid: 0bb51164-258f-4e04-a417-e16d736ca41c -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 - -.include ../proxyfsd/swift_client.conf -.include ../proxyfsd/file_server.conf -.include ../proxyfsd/rpc_server.conf -.include ../proxyfsd/saio_logging.conf -.include ../proxyfsd/stats.conf -.include ../proxyfsd/statslogger.conf -.include ../proxyfsd/httpserver.conf -.include ../proxyfsd/debug.conf - -# put it here to have the last word (override previous) -#[Logging] -#TraceLevelLogging: inode fs headhunter - -.include ../ramswift/chaos_settings.conf - -.include ../ramswift/swift_info.conf diff --git a/saio/usr/lib/systemd/system/pfsagentd.service b/saio/usr/lib/systemd/system/pfsagentd.service deleted file mode 100644 index c0db0533..00000000 --- a/saio/usr/lib/systemd/system/pfsagentd.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=PFSAgent service -After=proxyfsd.service - -[Service] -Environment=PATH=/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin:/vagrant/bin -Environment=GOTRACEBACK=crash -LimitCORE=infinity -ExecStart=/vagrant/bin/pfsagentd /vagrant/src/github.com/NVIDIA/proxyfs/pfsagentd/pfsagent.conf -ExecReload=/usr/bin/kill -HUP $MAINPID - -Restart=always -# Send stopping SIGTERM (and later SIGKILL if necessary) to the main process -# only. This prevents systemd from interfering with backends processing -# requests after the parent process has been sent a SIGHUP by the ExecReload. -KillMode=process - - -[Install] -WantedBy=multi-user.target diff --git a/saio/usr/lib/systemd/system/proxyfsd.service b/saio/usr/lib/systemd/system/proxyfsd.service deleted file mode 100644 index 0ae0258f..00000000 --- a/saio/usr/lib/systemd/system/proxyfsd.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Swift ProxyFS service - -[Service] -Environment=GOTRACEBACK=crash -LimitCORE=infinity -ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/NVIDIA/proxyfs/saio/proxyfs.conf -ExecReload=/usr/bin/kill -HUP $MAINPID -Restart=no -KillMode=process diff --git a/saio/usr/local/go/src/runtime/runtime-gdb.py b/saio/usr/local/go/src/runtime/runtime-gdb.py deleted file mode 100644 index d889e3d7..00000000 --- a/saio/usr/local/go/src/runtime/runtime-gdb.py +++ /dev/null @@ -1,541 +0,0 @@ -# Copyright 2010 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -"""GDB Pretty printers and convenience functions for Go's runtime structures. - -This script is loaded by GDB when it finds a .debug_gdb_scripts -section in the compiled binary. The [68]l linkers emit this with a -path to this file based on the path to the runtime package. -""" - -# Known issues: -# - pretty printing only works for the 'native' strings. E.g. 'type -# foo string' will make foo a plain struct in the eyes of gdb, -# circumventing the pretty print triggering. - - -from __future__ import print_function -import re -import sys - -print("Loading Go Runtime support.", file=sys.stderr) -#http://python3porting.com/differences.html -if sys.version > '3': - xrange = range -# allow to manually reload while developing -goobjfile = gdb.current_objfile() or gdb.objfiles()[0] -goobjfile.pretty_printers = [] - -# -# Value wrappers -# - -class SliceValue: - "Wrapper for slice values." - - def __init__(self, val): - self.val = val - - @property - def len(self): - return int(self.val['len']) - - @property - def cap(self): - return int(self.val['cap']) - - def __getitem__(self, i): - if i < 0 or i >= self.len: - raise IndexError(i) - ptr = self.val["array"] - return (ptr + i).dereference() - - -# -# Pretty Printers -# - - -class StringTypePrinter: - "Pretty print Go strings." - - pattern = re.compile(r'^struct string( \*)?$') - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'string' - - def to_string(self): - l = int(self.val['len']) - return self.val['str'].string("utf-8", "ignore", l) - - -class SliceTypePrinter: - "Pretty print slices." - - pattern = re.compile(r'^struct \[\]') - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'array' - - def to_string(self): - return str(self.val.type)[6:] # skip 'struct ' - - def children(self): - sval = SliceValue(self.val) - if sval.len > sval.cap: - return - for idx, item in enumerate(sval): - yield ('[{0}]'.format(idx), item) - - -class MapTypePrinter: - """Pretty print map[K]V types. - - Map-typed go variables are really pointers. dereference them in gdb - to inspect their contents with this pretty printer. - """ - - pattern = re.compile(r'^map\[.*\].*$') - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'map' - - def to_string(self): - return str(self.val.type) - - def children(self): - B = self.val['B'] - buckets = self.val['buckets'] - oldbuckets = self.val['oldbuckets'] - flags = self.val['flags'] - inttype = self.val['hash0'].type - cnt = 0 - for bucket in xrange(2 ** int(B)): - bp = buckets + bucket - if oldbuckets: - oldbucket = bucket & (2 ** (B - 1) - 1) - oldbp = oldbuckets + oldbucket - oldb = oldbp.dereference() - if (oldb['overflow'].cast(inttype) & 1) == 0: # old bucket not evacuated yet - if bucket >= 2 ** (B - 1): - continue # already did old bucket - bp = oldbp - while bp: - b = bp.dereference() - for i in xrange(8): - if b['tophash'][i] != 0: - k = b['keys'][i] - v = b['values'][i] - if flags & 1: - k = k.dereference() - if flags & 2: - v = v.dereference() - yield str(cnt), k - yield str(cnt + 1), v - cnt += 2 - bp = b['overflow'] - - -class ChanTypePrinter: - """Pretty print chan[T] types. - - Chan-typed go variables are really pointers. dereference them in gdb - to inspect their contents with this pretty printer. - """ - - pattern = re.compile(r'^struct hchan<.*>$') - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'array' - - def to_string(self): - return str(self.val.type) - - def children(self): - # see chan.c chanbuf(). et is the type stolen from hchan::recvq->first->elem - et = [x.type for x in self.val['recvq']['first'].type.target().fields() if x.name == 'elem'][0] - ptr = (self.val.address + 1).cast(et.pointer()) - for i in range(self.val["qcount"]): - j = (self.val["recvx"] + i) % self.val["dataqsiz"] - yield ('[{0}]'.format(i), (ptr + j).dereference()) - - -# -# Register all the *Printer classes above. -# - -def makematcher(klass): - def matcher(val): - try: - if klass.pattern.match(str(val.type)): - return klass(val) - except Exception: - pass - return matcher - -goobjfile.pretty_printers.extend([makematcher(var) for var in vars().values() if hasattr(var, 'pattern')]) - -# -# For reference, this is what we're trying to do: -# eface: p *(*(struct 'runtime.rtype'*)'main.e'->type_->data)->string -# iface: p *(*(struct 'runtime.rtype'*)'main.s'->tab->Type->data)->string -# -# interface types can't be recognized by their name, instead we check -# if they have the expected fields. Unfortunately the mapping of -# fields to python attributes in gdb.py isn't complete: you can't test -# for presence other than by trapping. - - -def is_iface(val): - try: - return str(val['tab'].type) == "struct runtime.itab *" and str(val['data'].type) == "void *" - except gdb.error: - pass - - -def is_eface(val): - try: - return str(val['_type'].type) == "struct runtime._type *" and str(val['data'].type) == "void *" - except gdb.error: - pass - - -def lookup_type(name): - try: - return gdb.lookup_type(name) - except gdb.error: - pass - try: - return gdb.lookup_type('struct ' + name) - except gdb.error: - pass - try: - return gdb.lookup_type('struct ' + name[1:]).pointer() - except gdb.error: - pass - - -def iface_commontype(obj): - if is_iface(obj): - go_type_ptr = obj['tab']['_type'] - elif is_eface(obj): - go_type_ptr = obj['_type'] - else: - return - - return go_type_ptr.cast(gdb.lookup_type("struct reflect.rtype").pointer()).dereference() - - -def iface_dtype(obj): - "Decode type of the data field of an eface or iface struct." - # known issue: dtype_name decoded from runtime.rtype is "nested.Foo" - # but the dwarf table lists it as "full/path/to/nested.Foo" - - dynamic_go_type = iface_commontype(obj) - if dynamic_go_type is None: - return - dtype_name = dynamic_go_type['string'].dereference()['str'].string() - - dynamic_gdb_type = lookup_type(dtype_name) - if dynamic_gdb_type is None: - return - - type_size = int(dynamic_go_type['size']) - uintptr_size = int(dynamic_go_type['size'].type.sizeof) # size is itself an uintptr - if type_size > uintptr_size: - dynamic_gdb_type = dynamic_gdb_type.pointer() - - return dynamic_gdb_type - - -def iface_dtype_name(obj): - "Decode type name of the data field of an eface or iface struct." - - dynamic_go_type = iface_commontype(obj) - if dynamic_go_type is None: - return - return dynamic_go_type['string'].dereference()['str'].string() - - -class IfacePrinter: - """Pretty print interface values - - Casts the data field to the appropriate dynamic type.""" - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'string' - - def to_string(self): - if self.val['data'] == 0: - return 0x0 - try: - dtype = iface_dtype(self.val) - except Exception: - return "" - - if dtype is None: # trouble looking up, print something reasonable - return "({0}){0}".format(iface_dtype_name(self.val), self.val['data']) - - try: - return self.val['data'].cast(dtype).dereference() - except Exception: - pass - return self.val['data'].cast(dtype) - - -def ifacematcher(val): - if is_iface(val) or is_eface(val): - return IfacePrinter(val) - -goobjfile.pretty_printers.append(ifacematcher) - -# -# Convenience Functions -# - - -class GoLenFunc(gdb.Function): - "Length of strings, slices, maps or channels" - - how = ((StringTypePrinter, 'len'), (SliceTypePrinter, 'len'), (MapTypePrinter, 'count'), (ChanTypePrinter, 'qcount')) - - def __init__(self): - gdb.Function.__init__(self, "len") - - def invoke(self, obj): - typename = str(obj.type) - for klass, fld in self.how: - if klass.pattern.match(typename): - return obj[fld] - - -class GoCapFunc(gdb.Function): - "Capacity of slices or channels" - - how = ((SliceTypePrinter, 'cap'), (ChanTypePrinter, 'dataqsiz')) - - def __init__(self): - gdb.Function.__init__(self, "cap") - - def invoke(self, obj): - typename = str(obj.type) - for klass, fld in self.how: - if klass.pattern.match(typename): - return obj[fld] - - -class DTypeFunc(gdb.Function): - """Cast Interface values to their dynamic type. - - For non-interface types this behaves as the identity operation. - """ - - def __init__(self): - gdb.Function.__init__(self, "dtype") - - def invoke(self, obj): - try: - return obj['data'].cast(iface_dtype(obj)) - except gdb.error: - pass - return obj - -# -# Commands -# - -sts = ('idle', 'runnable', 'running', 'syscall', 'waiting', 'moribund', 'dead', 'recovery') - - -def linked_list(ptr, linkfield): - while ptr: - yield ptr - ptr = ptr[linkfield] - - -class GoroutinesCmd(gdb.Command): - "List all goroutines." - - def __init__(self): - gdb.Command.__init__(self, "info goroutines", gdb.COMMAND_STACK, gdb.COMPLETE_NONE) - - def invoke(self, _arg, _from_tty): - # args = gdb.string_to_argv(arg) - vp = gdb.lookup_type('void').pointer() - for ptr in SliceValue(gdb.parse_and_eval("'runtime.allgs'")): - if ptr['atomicstatus'] == 6: # 'gdead' - continue - s = ' ' - if ptr['m']: - s = '*' - pc = ptr['sched']['pc'].cast(vp) - # python2 will not cast pc (type void*) to an int cleanly - # instead python2 and python3 work with the hex string representation - # of the void pointer which we can parse back into an int. - # int(pc) will not work. - try: - #python3 / newer versions of gdb - pc = int(pc) - except gdb.error: - # str(pc) can return things like - # "0x429d6c ", so - # chop at first space. - pc = int(str(pc).split(None, 1)[0], 16) - blk = gdb.block_for_pc(pc) - print(s, ptr['goid'], "{0:8s}".format(sts[int(ptr['atomicstatus'])]), blk.function) - - -def find_goroutine(goid): - """ - find_goroutine attempts to find the goroutine identified by goid - and returns a pointer to the goroutine info. - - @param int goid - - @return ptr - """ - vp = gdb.lookup_type('void').pointer() - for ptr in SliceValue(gdb.parse_and_eval("'runtime.allgs'")): - if ptr['atomicstatus'] == 6: # 'gdead' - continue - if ptr['goid'] == goid: - return ptr - return None - -def goroutine_info(ptr): - ''' - Given a pointer to goroutine info clean it up a bit - and return the interesting info in a dict. - ''' - gorinfo = {} - gorinfo['goid'] = ptr['goid'] - gorinfo['atomicstatus'] = sts[int(ptr['atomicstatus'])] - if gorinfo['atomicstatus'] == 'gdead': - return gorinfo - - vp = gdb.lookup_type('void').pointer() - gorinfo['pc_as_str'] = str(ptr['sched']['pc'].cast(vp)) - gorinfo['sp_as_str'] = str(ptr['sched']['sp'].cast(vp)) - - # str(pc) can return things like - # "0x429d6c ", so - # chop at first space. - gorinfo['pc_as_int'] = int(gorinfo['pc_as_str'].split(None, 1)[0], 16) - gorinfo['sp_as_int'] = int(gorinfo['sp_as_str'], 16) - - return gorinfo - -class GoroutineCmd(gdb.Command): - """Execute a gdb command in the context of goroutine . - - Switch PC and SP to the ones in the goroutine's G structure, - execute an arbitrary gdb command, and restore PC and SP. - - Usage: (gdb) goroutine - - Use goid 0 to invoke the command on all go routines. - - Note that it is ill-defined to modify state in the context of a goroutine. - Restrict yourself to inspecting values. - """ - - def __init__(self): - gdb.Command.__init__(self, "goroutine", gdb.COMMAND_STACK, gdb.COMPLETE_NONE) - - def invoke(self, arg, _from_tty): - goid, cmd = arg.split(None, 1) - goid = gdb.parse_and_eval(goid) - - if goid == 0: - goptr_list = SliceValue(gdb.parse_and_eval("'runtime.allgs'")) - else: - ptr = find_goroutine(goid) - if ptr is None: - print("No such goroutine: ", goid) - return - goptr_list = [ ptr ] - - for ptr in goptr_list: - gor = goroutine_info(ptr) - if gor['atomicstatus'] == 'gdead': - continue - - print("\ngoroutine %d:" % (gor['goid'])) - if gor['sp_as_int'] == 0: - print("#0 %s -- stack trace unavailable (goroutine status: %s)" % - (gor['pc_as_str'], gor['atomicstatus'])) - if gor['atomicstatus'] == 'running': - print("Try checking per thread stacks, i.e. 'thread apply all backtrace'") - continue - - save_frame = gdb.selected_frame() - gdb.parse_and_eval('$save_sp = $sp') - gdb.parse_and_eval('$save_pc = $pc') - gdb.parse_and_eval('$sp = {0}'.format(str(gor['sp_as_int']))) - gdb.parse_and_eval('$pc = {0}'.format(str(gor['pc_as_int']))) - try: - gdb.execute(cmd) - finally: - gdb.parse_and_eval('$sp = $save_sp') - gdb.parse_and_eval('$pc = $save_pc') - save_frame.select() - - -class GoIfaceCmd(gdb.Command): - "Print Static and dynamic interface types" - - def __init__(self): - gdb.Command.__init__(self, "iface", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL) - - def invoke(self, arg, _from_tty): - for obj in gdb.string_to_argv(arg): - try: - #TODO fix quoting for qualified variable names - obj = gdb.parse_and_eval(str(obj)) - except Exception as e: - print("Can't parse ", obj, ": ", e) - continue - - if obj['data'] == 0: - dtype = "nil" - else: - dtype = iface_dtype(obj) - - if dtype is None: - print("Not an interface: ", obj.type) - continue - - print("{0}: {1}".format(obj.type, dtype)) - -# TODO: print interface's methods and dynamic type's func pointers thereof. -#rsc: "to find the number of entries in the itab's Fn field look at -# itab.inter->numMethods -# i am sure i have the names wrong but look at the interface type -# and its method count" -# so Itype will start with a commontype which has kind = interface - -# -# Register all convenience functions and CLI commands -# -GoLenFunc() -GoCapFunc() -DTypeFunc() -GoroutinesCmd() -GoroutineCmd() -GoIfaceCmd() diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh deleted file mode 100644 index 07599baf..00000000 --- a/saio/vagrant_provision.sh +++ /dev/null @@ -1,411 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -# Note: This script assumes it is being run as root - -set -e -set -x - -# Enable core dumps -# -# Core files will be placed in /var/lib/systemd/coredump/ -# Core files will be compressed with xz... use unxz to uncompress them -# -# To install the delve debugger, you will need to `go get -u github.com/go-delve/delve/cmd/dlv` -# - An alias for the above, `gogetdlv`, should be issued if and when needed inside this VM - -sed -i '/DefaultLimitCORE=/c\DefaultLimitCORE=infinity' /etc/systemd/system.conf - -echo "kernel.core_pattern=| /usr/lib/systemd/systemd-coredump %p %u %g %s %t %c %e" > /etc/sysctl.d/90-override.conf -sysctl kernel.core_pattern='| /usr/lib/systemd/systemd-coredump %p %u %g %s %t %c %e' - -echo "GOTRACEBACK=crash" >> /etc/environment - -# Install yum-utils to deal with yum repos - -yum -y install yum-utils - -# Disable generic CentOS 7 repos - -yum-config-manager --disable CentOS-Base -yum-config-manager --disable CentOS-CR -yum-config-manager --disable CentOS-Debuginfo -yum-config-manager --disable CentOS-fasttrack -yum-config-manager --disable CentOS-Media -yum-config-manager --disable CentOS-Sources -yum-config-manager --disable CentOS-Vault - -rm -rf /etc/yum.repos.d/CentOS-Base.repo -rm -rf /etc/yum.repos.d/CentOS-CR.repo -rm -rf /etc/yum.repos.d/CentOS-Debuginfo.repo -rm -rf /etc/yum.repos.d/CentOS-fasttrack.repo -rm -rf /etc/yum.repos.d/CentOS-Media.repo -rm -rf /etc/yum.repos.d/CentOS-Sources.repo -rm -rf /etc/yum.repos.d/CentOS-Vault.repo - -# Add and enable CentOS 7.4 repos - -yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/os/x86_64/ -yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/updates/x86_64/ -yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/extras/x86_64/ -yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/centosplus/x86_64/ -yum-config-manager --enable vault.centos.org_centos_7.4.1708_os_x86_64_ -yum-config-manager --enable vault.centos.org_centos_7.4.1708_updates_x86_64_ -yum-config-manager --enable vault.centos.org_centos_7.4.1708_extras_x86_64_ -yum-config-manager --enable vault.centos.org_centos_7.4.1708_centosplus_x86_64_ - -yum clean all - -# Install tools needed above what's in a minimal base box - -yum -y install wget git nfs-utils vim lsof - -# Install Golang - -yum -y --disableexcludes=all install glibc-commmon gcc -cd /tmp -TARFILE_NAME=go1.16.6.linux-amd64.tar.gz -wget -q https://dl.google.com/go/$TARFILE_NAME -tar -C /usr/local -xf $TARFILE_NAME -rm $TARFILE_NAME -echo "export PATH=\$PATH:/usr/local/go/bin" >> ~vagrant/.bash_profile - -# Patch Golang's GDB runtime plug-in - -mv /usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/runtime-gdb.py_ORIGINAL -cp /vagrant/src/github.com/NVIDIA/proxyfs/saio/usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/. - -# Install GDB and enable above Golang GDB runtime plug-in as well as other niceties - -yum -y install gdb -echo "add-auto-load-safe-path /usr/local/go/src/runtime/runtime-gdb.py" > /home/vagrant/.gdbinit -echo "set print thread-events off" >> /home/vagrant/.gdbinit -echo "set print pretty on" >> /home/vagrant/.gdbinit -echo "set print object on" >> /home/vagrant/.gdbinit -echo "set pagination off" >> /home/vagrant/.gdbinit -chown vagrant:vagrant /home/vagrant/.gdbinit -chmod 644 /home/vagrant/.gdbinit -cp /home/vagrant/.gdbinit /root/. - -# Install Python 3.6 - -yum -y install centos-release-scl -yum -y install rh-python36 -ln -s /opt/rh/rh-python36/root/bin/python3.6 /bin/python3.6 -ln -s /bin/python3.6 /bin/python3 -ln -s /opt/rh/rh-python36/root/usr/include /opt/rh/rh-python36/root/include - -# Install Python pip - -yum -y install epel-release -yum -y install python-pip -pip install --upgrade 'pip<21.0' - -# Setup ProxyFS build environment - -pip install requests -yum -y install json-c-devel -yum -y install fuse -echo "alias cdpfs=\"cd /vagrant/src/github.com/NVIDIA/proxyfs\"" >> ~vagrant/.bash_profile -echo "alias goclean=\"go clean;go clean --cache;go clean --testcache\"" >> ~vagrant/.bash_profile -echo "alias gogetdlv=\"go get -u github.com/go-delve/delve/cmd/dlv\"" >> ~vagrant/.bash_profile -echo "user_allow_other" >> /etc/fuse.conf - -# Install Python tox - -pip install tox==3.5.3 - -# Setup Swift -# -# Guided by https://docs.openstack.org/swift/latest/development_saio.html - -# [Setup Swift] Create the swift:swift user - -useradd --user-group --groups wheel swift -chmod 755 ~swift - -# Using loopback devices for storage - -mkdir -p /srv - -for x in 11 22 33 44 15 26 37 48 19 2A 3B 4C -do - node=${x:0:1} - drive=${x:1:1} - truncate -s 0 /srv/swift-disk-$drive - truncate -s 1GB /srv/swift-disk-$drive - mkfs.xfs -f /srv/swift-disk-$drive - mkdir -p /srv/$node/node/sdb$drive - echo "/srv/swift-disk-$drive /srv/$node/node/sdb$drive xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0" >> /etc/fstab - mount /srv/$node/node/sdb$drive - chown swift:swift /srv/$node/node/sdb$drive -done - -# [Setup Swift] Common Post-Device Setup (Add /var boot-time provisioning to /etc/rc.d/rc.local) - -echo "mkdir -p /var/cache/swift /var/cache/swift2 /var/cache/swift3 /var/cache/swift4" >> /etc/rc.d/rc.local -echo "chown swift:swift /var/cache/swift*" >> /etc/rc.d/rc.local -echo "mkdir -p /var/run/swift" >> /etc/rc.d/rc.local -echo "chown swift:swift /var/run/swift" >> /etc/rc.d/rc.local -chmod +x /etc/rc.d/rc.local - -# [Setup Swift] Do boot-time provisioning now... as if we just booted - -mkdir -p /var/cache/swift /var/cache/swift2 /var/cache/swift3 /var/cache/swift4 -chown swift:swift /var/cache/swift* -mkdir -p /var/run/swift -chown swift:swift /var/run/swift - -# [Setup Swift] Getting the code - -yum -y install \ - memcached \ - sqlite \ - xfsprogs \ - libffi-devel \ - xinetd \ - openssl-devel \ - python-setuptools \ - python-coverage \ - python-devel \ - python-nose \ - pyxattr \ - python-eventlet \ - python-greenlet \ - python-paste-deploy \ - python-netifaces \ - python-pip \ - python-dns \ - python-mock - -pip install --upgrade setuptools - -# Build liberasure.so from source - -cd ~swift -git clone https://github.com/openstack/liberasurecode.git -cd liberasurecode -yum install -y gcc make autoconf automake libtool -./autogen.sh -./configure -make -make install - -# Install it where Python/PyECLib will see it - -echo "/usr/local/lib" > /etc/ld.so.conf.d/liberasurecode.conf -ldconfig -# Alternatively, we could simply have done -# ln -s /usr/local/lib/liberasurecode.so.1 /lib64/liberasurecode.so.1 - -# Install PyECLib from source - -cd ~swift -git clone https://github.com/openstack/pyeclib.git -cd pyeclib -pip install -e . -pip install -r test-requirements.txt - -# Install python-swiftclient from source & setup ENVs for its use - -cd ~swift -git clone -b master --single-branch --depth 1 https://github.com/openstack/python-swiftclient.git -cd python-swiftclient -python setup.py develop - -echo "export ST_AUTH=http://localhost:8080/auth/v1.0" >> ~vagrant/.bash_profile -echo "export ST_USER=test:tester" >> ~vagrant/.bash_profile -echo "export ST_KEY=testing" >> ~vagrant/.bash_profile - -# Now we can actually install Swift from source - -cd ~swift -git clone https://github.com/NVIDIA/swift.git -cd swift -export SWIFT_TAG="$( git describe --tags --abbrev=0 $(git rev-list --tags --max-count=1) )" -git checkout $SWIFT_TAG -pip install wheel -python setup.py bdist_wheel -yum remove -y python-greenlet -pip install --constraint py2-constraints.txt -r requirements.txt -python setup.py develop -# The following avoid dependency on pip-installed pyOpenSSL being newer than required -pip install python-openstackclient==3.12.0 python-glanceclient==2.7.0 -# This is a temporary fix while bandit gets added to py2-constraints.txt -pip install bandit==1.6.2 -pip install --constraint py2-constraints.txt -r test-requirements.txt - -# [Setup Swift] Setting up rsync - -cd /etc -cp ~swift/swift/doc/saio/rsyncd.conf . -sed -i "s//swift/" rsyncd.conf - -cd /etc/xinetd.d -echo "disable = no" >> rsync - -systemctl restart xinetd.service -systemctl enable rsyncd.service -systemctl start rsyncd.service - -rsync rsync://pub@localhost/ - -# [Setup Swift] Setting up memcached - -systemctl enable memcached.service -systemctl start memcached.service - -# [Setup Swift] Configuring each node - -rm -rf /etc/swift -cp -R /vagrant/src/github.com/NVIDIA/proxyfs/saio/etc/swift /etc/swift -chown -R swift:swift /etc/swift - -# [Setup Swift] Setting up scripts for running Swift - -mkdir -p ~swift/bin - -cd ~swift/bin -cp /vagrant/src/github.com/NVIDIA/proxyfs/saio/home/swift/bin/* . -echo "export PATH=\$PATH:~swift/bin" >> ~vagrant/.bash_profile - -~swift/bin/remakerings - -# Install ProxyFS's pfs_middleware into the "normal" Swift Proxy pipeline - -cd /vagrant/src/github.com/NVIDIA/proxyfs/pfs_middleware -python setup.py develop - -# Install ProxyFS's meta_middleware into the "NoAuth" Swift Proxy pipeline - -cd /vagrant/src/github.com/NVIDIA/proxyfs/meta_middleware -python setup.py develop - -# Setup AWS access for local vagrant user - -pip install awscli-plugin-endpoint -mkdir -p ~vagrant/.aws -cat > ~vagrant/.aws/config << EOF -[plugins] -endpoint = awscli_plugin_endpoint - -[default] -s3 = - endpoint_url = http://127.0.0.1:8080 - multipart_threshold = 64MB - multipart_chunksize = 16MB -s3api = - endpoint_url = http://127.0.0.1:8080 - multipart_threshold = 64MB - multipart_chunksize = 16MB -EOF -cat > ~vagrant/.aws/credentials << EOF -[default] -aws_access_key_id = test:tester -aws_secret_access_key = testing -EOF -chown -R vagrant:vagrant ~vagrant/.aws - -# Ensure proxyfsd logging will work - -rm -rf /var/log/proxyfsd -mkdir -p /var/log/proxyfsd -touch /var/log/proxyfsd/proxyfsd.log -chmod 777 /var -chmod 777 /var/log -chmod 777 /var/log/proxyfsd -chmod 666 /var/log/proxyfsd/proxyfsd.log - -# Create Mount Points for ProxyFS (embedded FUSE) - -rm -rf /CommonMountPoint -mkdir /CommonMountPoint -chmod 777 /CommonMountPoint - -# Install systemd .service files for ProxyFS - -cp /vagrant/src/github.com/NVIDIA/proxyfs/saio/usr/lib/systemd/system/proxyfsd.service /usr/lib/systemd/system/. -cp /vagrant/src/github.com/NVIDIA/proxyfs/saio/usr/lib/systemd/system/pfsagentd.service /usr/lib/systemd/system/. - -# Place symlink in root's $PATH to locate pfsagentd-swift-auth-plugin referenced without a path - -ln -s /vagrant/bin/pfsagentd-swift-auth-plugin /usr/bin/pfsagentd-swift-auth-plugin - -# Enable Samba service in an SELinux environment - -yum -y install policycoreutils-python -semanage port -a -t smbd_port_t -p tcp 12345 -semanage port -a -t smbd_port_t -p tcp 32345 - -# Enable start/stop tools - -echo "export PATH=\$PATH:/vagrant/src/github.com/NVIDIA/proxyfs/saio/bin" >> ~vagrant/.bash_profile - -# Install wireshark - -yum -y install wireshark-gnome \ - xorg-x11-fonts-Type1 \ - xorg-x11-xauth \ - xeyes -echo "X11Forwarding yes" >> /etc/sysconfig/sshd -systemctl restart sshd -usermod -aG wireshark vagrant - -# Install benchmark support tools - -yum -y install atop-2.3.0-8.el7 bc fio gawk - -# Install ssh helper - -yum -y install sshpass-1.06-2.el7 - -# Install dstat - -yum -y install dstat - -# Install tree - -yum -y install tree - -# Install jq... a very handy JSON parser - -yum -y install jq - -# Install and configure a localhost-only one-node etcd cluster - -ETCD_VERSION=3.4.7 -wget https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -tar xzf etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -rm -rf etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -install -C -m 755 etcd-v${ETCD_VERSION}-linux-amd64/etcd /usr/local/bin/ -install -C -m 755 etcd-v${ETCD_VERSION}-linux-amd64/etcdctl /usr/local/bin/ -rm -rf etcd-v${ETCD_VERSION}-linux-amd64 - -mkdir /etcd - -cat > /etc/systemd/system/proxyfs-etcd.service << EOF -[Unit] -Description=ProxyFS etcd instance -After=network.target -StartLimitIntervalSec=0 - -[Service] -Type=simple -Restart=always -RestartSec=1 -User=root -ExecStart=/usr/local/bin/etcd --name proxyfs --data-dir /etcd/proxyfs.etcd --initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 --listen-client-urls http://localhost:2379 --advertise-client-urls http://localhost:2379 --initial-cluster-token etcd-cluster --initial-cluster default=http://localhost:2380 --initial-cluster-state new - -[Install] -WantedBy=multi-user.target -EOF - -# Inform systemd that we've updated .service files - -systemctl daemon-reload - -# All done - -echo "SAIO for ProxyFS provisioned" diff --git a/sait/Vagrantfile b/sait/Vagrantfile deleted file mode 100644 index 1b8b3eb9..00000000 --- a/sait/Vagrantfile +++ /dev/null @@ -1,82 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Notes: -# -# 1) vboxnet0 is assumed to be a host-only network @ address 172.28.128.1 (DHCP disabled) -# 2) Though not required, GOPATH is assumed to be the ../../../../../ directory -# 3) The directory on the VM Host will be /vagrant on the VM and be the path in GOPATH -# 4) ProxyFS repo git clone'd to $GOPATH/src/github.com/NVIDIA/ -# 5) samba repo automatically staged in $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ -# 6) Swift repos et. al. git clone'd to $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ -# 7) ../Makefile will be ready to be executed after `cdpfs` inside the VM -# 8) As GOPATH is effectively shared between Host and VM, builds in the two environments -# will collide in the contents of the $GOPATH/bin (only executables, not libraries) -# 9) As of now, only ProxyFS is supported (no Samba, no ramswift), so, at the most, -# users shoult only do `make version fmt pre-generate generate install` -# 10) Only the following tools in bin/ are currently supported: -# provision_middleware -# start_proxyfs_and_swift -# start_swift_only -# unmount_and_stop_pfs -# 11) bin/start_proxyfs_and_swift requires an argument of either 1, 2, or 3 -# 12) home/swift/bin/resetswift requires an argument of either 1, 2, or 3 - -Vagrant.configure(2) do |config| - config.vm.define "sait1" do |sconfig| - sconfig.vm.box = "centos-74-minimal-20180727" - sconfig.vm.box_url = "https://o.swiftstack.org/v1/AUTH_misc/vagrant_boxes/centos-74-minimal-20180727.box" - sconfig.vm.define "sait1pfs" do |sait1pfs| - end - sconfig.vm.provider :virtualbox do |vb| - vb.name = "SAIT 1 for ProxyFS" - vb.cpus = Integer(ENV['VAGRANT_CPUS'] || 1) - vb.memory = Integer(ENV['VAGRANT_RAM'] || 2048) - vb.customize ["modifyvm", :id, "--audio", "none"] - end - sconfig.vm.synced_folder "../../../../../", "/vagrant", type: "virtualbox" - sconfig.vm.network "private_network", ip: "172.28.128.3", :name => 'vboxnet0', :adapter => 2 - sconfig.vm.network "forwarded_port", guest: 15346, host: 15347 - sconfig.vm.network "forwarded_port", guest: 9090, host: 9092 - sconfig.vm.network "private_network", ip: "192.168.22.114", :name => 'vboxnet1', :adapter => 3 - sconfig.vm.provision "shell", path: "vagrant_provision.sh", args: "1" - end - - config.vm.define "sait2" do |sconfig| - sconfig.vm.box = "centos-74-minimal-20180727" - sconfig.vm.box_url = "https://o.swiftstack.org/v1/AUTH_misc/vagrant_boxes/centos-74-minimal-20180727.box" - sconfig.vm.define "sait2pfs" do |sait2pfs| - end - sconfig.vm.provider :virtualbox do |vb| - vb.name = "SAIT 2 for ProxyFS" - vb.cpus = Integer(ENV['VAGRANT_CPUS'] || 1) - vb.memory = Integer(ENV['VAGRANT_RAM'] || 2048) - vb.customize ["modifyvm", :id, "--audio", "none"] - end - sconfig.vm.synced_folder "../../../../../", "/vagrant", type: "virtualbox" - sconfig.vm.network "private_network", ip: "172.28.128.4", :name => 'vboxnet0', :adapter => 2 - sconfig.vm.network "forwarded_port", guest: 15346, host: 15348 - sconfig.vm.network "forwarded_port", guest: 9090, host: 9093 - sconfig.vm.network "private_network", ip: "192.168.22.115", :name => 'vboxnet1', :adapter => 3 - sconfig.vm.provision "shell", path: "vagrant_provision.sh", args: "2" - end - - config.vm.define "sait3" do |sconfig| - sconfig.vm.box = "centos-74-minimal-20180727" - sconfig.vm.box_url = "https://o.swiftstack.org/v1/AUTH_misc/vagrant_boxes/centos-74-minimal-20180727.box" - sconfig.vm.define "sait1pfs" do |sait3pfs| - end - sconfig.vm.provider :virtualbox do |vb| - vb.name = "SAIT 3 for ProxyFS" - vb.cpus = Integer(ENV['VAGRANT_CPUS'] || 1) - vb.memory = Integer(ENV['VAGRANT_RAM'] || 2048) - vb.customize ["modifyvm", :id, "--audio", "none"] - end - sconfig.vm.synced_folder "../../../../../", "/vagrant", type: "virtualbox" - sconfig.vm.network "private_network", ip: "172.28.128.5", :name => 'vboxnet0', :adapter => 2 - sconfig.vm.network "forwarded_port", guest: 15346, host: 15349 - sconfig.vm.network "forwarded_port", guest: 9090, host: 9094 - sconfig.vm.network "private_network", ip: "192.168.22.116", :name => 'vboxnet1', :adapter => 3 - sconfig.vm.provision "shell", path: "vagrant_provision.sh", args: "3" - end -end diff --git a/sait/bin/provision_middleware b/sait/bin/provision_middleware deleted file mode 100755 index 1c1559e9..00000000 --- a/sait/bin/provision_middleware +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -cd $GOPATH/src/github.com/NVIDIA/proxyfs/pfs_middleware -sudo python setup.py develop -cd $GOPATH/src/github.com/NVIDIA/proxyfs/meta_middleware -sudo python setup.py develop diff --git a/sait/bin/start_proxyfs_and_swift b/sait/bin/start_proxyfs_and_swift deleted file mode 100755 index 28e2ba68..00000000 --- a/sait/bin/start_proxyfs_and_swift +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -if [ "$1" == "1" ] -then - PRIVATE_IP_ADDR="192.168.22.114" - VOLUME_TO_SERVE="CommonVolume" -elif [ "$1" == "2" ] -then - PRIVATE_IP_ADDR="192.168.22.115" - VOLUME_TO_SERVE="" -elif [ "$1" == "3" ] -then - PRIVATE_IP_ADDR="192.168.22.116" - VOLUME_TO_SERVE="" -else - echo Arg1 is unexpected: $1 - exit 1 -fi - -SAIT_DIR=sait$1 - -function await_proxyfsd_startup { - while true - do - /usr/bin/systemctl -q is-active proxyfsd - if [ $? -ne 0 ] - then - echo "ProxyFS failed to start. Exiting..." - exit 1 - fi - curl http://$PRIVATE_IP_ADDR:15346/ 2>/dev/null >/dev/null - if [ $? -eq 0 ] - then - break - fi - sleep 1 - done -} - -function await_swift_startup { - while true - do - curl http://127.0.0.1:8090/info 2>/dev/null >/dev/null - if [ $? -eq 0 ] - then - break - fi - echo "Waiting for Swift to be started..." - sleep 1 - done -} - -function format_volume_if_necessary { - if [ "" != "$1" ] - then - sudo /vagrant/bin/mkproxyfs -I $1 /vagrant/src/github.com/NVIDIA/proxyfs/sait/$SAIT_DIR/proxyfs.conf SwiftClient.RetryLimit=1 - if [ $? -ne 0 ] - then - echo "Could not pre-format $1" - exit 1 - fi - fi -} - -sudo mount -a - -echo "Shutting down services and mount points..." -/vagrant/src/github.com/NVIDIA/proxyfs/sait/bin/unmount_and_stop_pfs -echo -echo "Bringing up services..." -if [ -f /usr/bin/systemctl ]; then - # Centos - sudo /usr/bin/systemctl start memcached - sudo /usr/bin/swift-init main start - await_swift_startup - format_volume_if_necessary $VOLUME_TO_SERVE - sudo /usr/bin/systemctl start proxyfsd - await_proxyfsd_startup -else - # Ubuntu (not tested!) - sudo /usr/sbin/service memcached start - sudo /usr/bin/swift-init main start - await_swift_startup - format_volume_if_necessary $VOLUME_TO_SERVE - sudo /usr/sbin/service proxyfsd start - await_proxyfsd_startup -fi diff --git a/sait/bin/start_swift_only b/sait/bin/start_swift_only deleted file mode 100755 index 3269977b..00000000 --- a/sait/bin/start_swift_only +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -sudo mount -a - -echo "Shutting down services and mount points..." -/vagrant/src/github.com/NVIDIA/proxyfs/sait/bin/unmount_and_stop_pfs -echo -echo "Bringing up services..." -if [ -f /usr/bin/systemctl ]; then - # Centos - sudo /usr/bin/systemctl start memcached - sudo /usr/bin/swift-init main start -else - # Ubuntu (not tested!) - sudo /usr/sbin/service memcached start - sudo /usr/bin/swift-init main start -fi diff --git a/sait/bin/unmount_and_stop_pfs b/sait/bin/unmount_and_stop_pfs deleted file mode 100755 index 294b0f85..00000000 --- a/sait/bin/unmount_and_stop_pfs +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -function await_proxyfsd_shutdown { - while true - do - pidof proxyfsd > /dev/null - if [ $? -ne 0 ] - then - break - fi - echo "Waiting for ProxyFS to be stopped..." - sleep 1 - done -} - -if [ -f /usr/bin/systemctl ]; then - # Centos - sudo /usr/bin/systemctl stop proxyfsd - await_proxyfsd_shutdown - sudo /usr/bin/swift-init main stop - sudo /usr/bin/systemctl stop memcached -else - # Ubuntu (not tested!) - # Here we should stop pfsagentd, but we don't support Ubuntu - sudo /usr/sbin/service proxyfsd stop - await_proxyfsd_shutdown - sudo /usr/bin/swift-init main stop - sudo /usr/sbin/service memcached stop -fi diff --git a/sait/etc/swift/container-reconciler.conf b/sait/etc/swift/container-reconciler.conf deleted file mode 100644 index 30f81ca5..00000000 --- a/sait/etc/swift/container-reconciler.conf +++ /dev/null @@ -1,47 +0,0 @@ -[DEFAULT] -# swift_dir = /etc/swift -user = swift -# You can specify default log routing here if you want: -# log_name = swift -# log_facility = LOG_LOCAL0 -# log_level = INFO -# log_address = /dev/log -# -# comma separated list of functions to call to setup custom log handlers. -# functions get passed: conf, name, log_to_console, log_route, fmt, logger, -# adapted_logger -# log_custom_handlers = -# -# If set, log_udp_host will override log_address -# log_udp_host = -# log_udp_port = 514 -# -# You can enable StatsD logging here: -# log_statsd_host = -# log_statsd_port = 8125 -# log_statsd_default_sample_rate = 1.0 -# log_statsd_sample_rate_factor = 1.0 -# log_statsd_metric_prefix = - -[container-reconciler] -# reclaim_age = 604800 -# interval = 300 -# request_tries = 3 - -[pipeline:main] -pipeline = catch_errors proxy-logging cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy -# See proxy-server.conf-sample for options - -[filter:cache] -use = egg:swift#memcache -# See proxy-server.conf-sample for options - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:catch_errors] -use = egg:swift#catch_errors -# See proxy-server.conf-sample for options diff --git a/sait/etc/swift/container-sync-realms.conf b/sait/etc/swift/container-sync-realms.conf deleted file mode 100644 index c84d3391..00000000 --- a/sait/etc/swift/container-sync-realms.conf +++ /dev/null @@ -1,4 +0,0 @@ -[saio] -key = changeme -key2 = changeme -cluster_saio_endpoint = http://127.0.0.1:8080/v1/ diff --git a/sait/etc/swift/object-expirer.conf b/sait/etc/swift/object-expirer.conf deleted file mode 100644 index ab8a4cba..00000000 --- a/sait/etc/swift/object-expirer.conf +++ /dev/null @@ -1,59 +0,0 @@ -[DEFAULT] -# swift_dir = /etc/swift -user = swift -# You can specify default log routing here if you want: -log_name = object-expirer -log_facility = LOG_LOCAL6 -log_level = INFO -#log_address = /dev/log -# -# comma separated list of functions to call to setup custom log handlers. -# functions get passed: conf, name, log_to_console, log_route, fmt, logger, -# adapted_logger -# log_custom_handlers = -# -# If set, log_udp_host will override log_address -# log_udp_host = -# log_udp_port = 514 -# -# You can enable StatsD logging here: -# log_statsd_host = -# log_statsd_port = 8125 -# log_statsd_default_sample_rate = 1.0 -# log_statsd_sample_rate_factor = 1.0 -# log_statsd_metric_prefix = - -[object-expirer] -interval = 300 -# auto_create_account_prefix = . -# report_interval = 300 -# concurrency is the level of concurrency o use to do the work, this value -# must be set to at least 1 -# concurrency = 1 -# processes is how many parts to divide the work into, one part per process -# that will be doing the work -# processes set 0 means that a single process will be doing all the work -# processes can also be specified on the command line and will override the -# config value -# processes = 0 -# process is which of the parts a particular process will work on -# process can also be specified on the command line and will override the config -# value -# process is "zero based", if you want to use 3 processes, you should run -# processes with process set to 0, 1, and 2 -# process = 0 - -[pipeline:main] -pipeline = catch_errors cache proxy-server - -[app:proxy-server] -use = egg:swift#proxy -# See proxy-server.conf-sample for options - -[filter:cache] -use = egg:swift#memcache -# See proxy-server.conf-sample for options - -[filter:catch_errors] -use = egg:swift#catch_errors -# See proxy-server.conf-sample for options diff --git a/sait/etc/swift/proxy-server.conf b/sait/etc/swift/proxy-server.conf deleted file mode 100644 index 652614ce..00000000 --- a/sait/etc/swift/proxy-server.conf +++ /dev/null @@ -1,93 +0,0 @@ -[DEFAULT] -bind_ip = 0.0.0.0 -bind_port = 8080 -workers = 1 -user = swift -log_facility = LOG_LOCAL1 -eventlet_debug = true - -[pipeline:main] -# Yes, proxy-logging appears twice. This is so that -# middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync s3api tempauth staticweb copy container-quotas account-quotas slo dlo pfs versioned_writes proxy-logging proxy-server - -[filter:catch_errors] -use = egg:swift#catch_errors - -[filter:gatekeeper] -use = egg:swift#gatekeeper - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:cache] -use = egg:swift#memcache -memcache_max_connections = 1024 - -[filter:bulk] -use = egg:swift#bulk - -[filter:tempurl] -use = egg:swift#tempurl - -[filter:ratelimit] -use = egg:swift#ratelimit - -[filter:crossdomain] -use = egg:swift#crossdomain - -[filter:container_sync] -use = egg:swift#container_sync -current = //saio/saio_endpoint - -[filter:s3api] -use = egg:swift#s3api -allow_multipart_uploads = yes -check_bucket_owner = no -dns_compliant_bucket_names = yes -force_swift_request_proxy_log = yes -s3_acl = no - -[filter:tempauth] -use = egg:swift#tempauth -user_admin_admin = admin .admin .reseller_admin -user_test_tester = testing .admin -user_test2_tester2 = testing2 .admin -user_test_tester3 = testing3 - -[filter:staticweb] -use = egg:swift#staticweb - -[filter:copy] -use = egg:swift#copy - -[filter:container-quotas] -use = egg:swift#container_quotas - -[filter:account-quotas] -use = egg:swift#account_quotas - -[filter:slo] -use = egg:swift#slo - -[filter:dlo] -use = egg:swift#dlo - -# For bypass_mode, one of off (default), read-only, or read-write -[filter:pfs] -use = egg:pfs_middleware#pfs -proxyfsd_host = 192.168.22.114 -proxyfsd_port = 12345 -bypass_mode = read-write - -[filter:versioned_writes] -use = egg:swift#versioned_writes -allow_versioned_writes = true - -[app:proxy-server] -use = egg:swift#proxy -allow_account_management = true -account_autocreate = true diff --git a/sait/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf b/sait/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf deleted file mode 100644 index 7e7cf83e..00000000 --- a/sait/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf +++ /dev/null @@ -1,45 +0,0 @@ -[DEFAULT] -bind_ip = 0.0.0.0 -bind_port = 8090 -workers = 1 -user = swift -log_facility = LOG_LOCAL1 -eventlet_debug = true - -[pipeline:main] -# Yes, proxy-logging appears twice. This is so that -# middleware-originated requests get logged too. -pipeline = catch_errors gatekeeper healthcheck proxy-logging cache copy dlo meta versioned_writes proxy-logging proxy-server - -[filter:catch_errors] -use = egg:swift#catch_errors - -[filter:gatekeeper] -use = egg:swift#gatekeeper - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:proxy-logging] -use = egg:swift#proxy_logging - -[filter:cache] -use = egg:swift#memcache - -[filter:copy] -use = egg:swift#copy - -[filter:dlo] -use = egg:swift#dlo - -[filter:meta] -use = egg:meta_middleware#meta - -[filter:versioned_writes] -use = egg:swift#versioned_writes -allow_versioned_writes = true - -[app:proxy-server] -use = egg:swift#proxy -allow_account_management = true -account_autocreate = true diff --git a/sait/etc/swift/swift.conf b/sait/etc/swift/swift.conf deleted file mode 100644 index 3022ded1..00000000 --- a/sait/etc/swift/swift.conf +++ /dev/null @@ -1,17 +0,0 @@ -[swift-hash] -# random unique strings that can never change (DO NOT LOSE) -# Use only printable chars (python -c "import string; print(string.printable)") -swift_hash_path_prefix = changeme -swift_hash_path_suffix = changeme - -[storage-policy:0] -name = gold -policy_type = replication -default = no -deprecated = no - -[storage-policy:1] -name = silver -policy_type = replication -default = yes -deprecated = no diff --git a/sait/home/swift/bin/remakerings b/sait/home/swift/bin/remakerings deleted file mode 100755 index ef3e6d34..00000000 --- a/sait/home/swift/bin/remakerings +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -e - -cd /etc/swift - -rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz - -swift-ring-builder object.builder create 10 1 1 -swift-ring-builder object.builder add r1z1-192.168.22.114:8010/sdb1 1 -swift-ring-builder object.builder add r1z1-192.168.22.115:8010/sdb2 1 -swift-ring-builder object.builder add r1z1-192.168.22.116:8010/sdb3 1 -swift-ring-builder object.builder rebalance -swift-ring-builder object-1.builder create 10 1 1 -swift-ring-builder object-1.builder add r1z1-192.168.22.114:8010/sdb1 1 -swift-ring-builder object-1.builder add r1z1-192.168.22.115:8010/sdb2 1 -swift-ring-builder object-1.builder add r1z1-192.168.22.116:8010/sdb3 1 -swift-ring-builder object-1.builder rebalance -swift-ring-builder container.builder create 10 1 1 -swift-ring-builder container.builder add r1z1-192.168.22.114:8011/sdb1 1 -swift-ring-builder container.builder add r1z1-192.168.22.115:8011/sdb2 1 -swift-ring-builder container.builder add r1z1-192.168.22.116:8011/sdb3 1 -swift-ring-builder container.builder rebalance -swift-ring-builder account.builder create 10 1 1 -swift-ring-builder account.builder add r1z1-192.168.22.114:8012/sdb1 1 -swift-ring-builder account.builder add r1z1-192.168.22.115:8012/sdb2 1 -swift-ring-builder account.builder add r1z1-192.168.22.116:8012/sdb3 1 -swift-ring-builder account.builder rebalance diff --git a/sait/home/swift/bin/resetswift b/sait/home/swift/bin/resetswift deleted file mode 100755 index b295deed..00000000 --- a/sait/home/swift/bin/resetswift +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -if [ "$1" != "1" ] && [ "$1" != "2" ] && [ "$1" != "3" ] -then - echo Arg1 is unexpected: $1 - exit 1 -fi - -set -e - -SAIT_INSTANCE=$1 - -sudo swift-init all kill - -if cut -d' ' -f2 /proc/mounts | grep -q /srv/$SAIT_INSTANCE/node/sdb$SAIT_INSTANCE -then - sudo umount /srv/$SAIT_INSTANCE/node/sdb$SAIT_INSTANCE -fi -sudo truncate -s 0 /srv/swift-disk -sudo truncate -s 1GB /srv/swift-disk -sudo mkfs.xfs -f /srv/swift-disk -sudo mount /srv/$SAIT_INSTANCE/node/sdb$SAIT_INSTANCE -sudo chown swift:swift /srv/$SAIT_INSTANCE/node/sdb$SAIT_INSTANCE - -sudo rm -f /var/log/debug /var/log/messages /var/log/rsyncd.log /var/log/syslog -find /var/cache/swift* -type f -name *.recon -exec rm -f {} \; -sudo systemctl restart memcached diff --git a/sait/proxyfs.conf b/sait/proxyfs.conf deleted file mode 100644 index b24b1c99..00000000 --- a/sait/proxyfs.conf +++ /dev/null @@ -1,43 +0,0 @@ -# Three peer .conf file customized for SAIT for ProxyFS VMs (included by per-sait?/proxyfs.conf) - -[Peer:Peer1] -PublicIPAddr: 172.28.128.3 -PrivateIPAddr: 192.168.22.114 -ReadCacheQuotaFraction: 0.20 - -[Peer:Peer2] -PublicIPAddr: 172.28.128.4 -PrivateIPAddr: 192.168.22.115 -ReadCacheQuotaFraction: 0.20 - -[Peer:Peer3] -PublicIPAddr: 172.28.128.5 -PrivateIPAddr: 192.168.22.116 -ReadCacheQuotaFraction: 0.20 - -[Cluster] -Peers: Peer1 Peer2 Peer3 -ServerGuid: 0bb51164-258f-4e04-a417-e16d736ca41c -PrivateClusterUDPPort: 8123 -UDPPacketSendSize: 1400 -UDPPacketRecvSize: 1500 -UDPPacketCapPerMessage: 5 -HeartBeatDuration: 1s -HeartBeatMissLimit: 3 -MessageQueueDepthPerPeer: 4 -MaxRequestDuration: 1s -LivenessCheckRedundancy: 2 -LogLevel: 0 - -.include ../proxyfsd/file_server.conf - -[VolumeGroup:CommonVolumeGroup] -PrimaryPeer: Peer1 - -.include ../proxyfsd/swift_client.conf -.include ../proxyfsd/rpc_server.conf -.include ../proxyfsd/saio_logging.conf -.include ../proxyfsd/stats.conf -.include ../proxyfsd/statslogger.conf -.include ../proxyfsd/httpserver.conf -.include ../proxyfsd/debug.conf diff --git a/sait/sait1/etc/swift/account-server/1.conf b/sait/sait1/etc/swift/account-server/1.conf deleted file mode 100644 index 54afdfcb..00000000 --- a/sait/sait1/etc/swift/account-server/1.conf +++ /dev/null @@ -1,27 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.114 -bind_port = 8012 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon account-server - -[app:account-server] -use = egg:swift#account - -[filter:recon] -use = egg:swift#recon - -[account-replicator] -rsync_module = {replication_ip}::account{replication_port} - -[account-auditor] - -[account-reaper] diff --git a/sait/sait1/etc/swift/container-server/1.conf b/sait/sait1/etc/swift/container-server/1.conf deleted file mode 100644 index 85a99ed1..00000000 --- a/sait/sait1/etc/swift/container-server/1.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.114 -bind_port = 8011 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon container-server - -[app:container-server] -use = egg:swift#container - -[filter:recon] -use = egg:swift#recon - -[container-replicator] -rsync_module = {replication_ip}::container{replication_port} - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/sait/sait1/etc/swift/object-server/1.conf b/sait/sait1/etc/swift/object-server/1.conf deleted file mode 100644 index ae4f29ca..00000000 --- a/sait/sait1/etc/swift/object-server/1.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/1/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.114 -bind_port = 8010 -workers = 1 -user = swift -log_facility = LOG_LOCAL2 -recon_cache_path = /var/cache/swift -eventlet_debug = true - -[pipeline:main] -pipeline = recon object-server - -[app:object-server] -use = egg:swift#object - -[filter:recon] -use = egg:swift#recon - -[object-replicator] -rsync_module = {replication_ip}::object{replication_port} - -[object-reconstructor] - -[object-updater] - -[object-auditor] diff --git a/sait/sait1/proxyfs.conf b/sait/sait1/proxyfs.conf deleted file mode 100644 index e941c7b8..00000000 --- a/sait/sait1/proxyfs.conf +++ /dev/null @@ -1,4 +0,0 @@ -[Cluster] -WhoAmI: Peer1 - -.include ../proxyfs.conf diff --git a/sait/sait1/usr/lib/systemd/system/proxyfsd.service b/sait/sait1/usr/lib/systemd/system/proxyfsd.service deleted file mode 100644 index abfc477b..00000000 --- a/sait/sait1/usr/lib/systemd/system/proxyfsd.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Swift ProxyFS service - -[Service] -Environment=GOTRACEBACK=crash -LimitCORE=infinity -ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait1/proxyfs.conf -ExecReload=/usr/bin/kill -HUP $MAINPID -Restart=no -KillMode=process diff --git a/sait/sait2/etc/swift/account-server/2.conf b/sait/sait2/etc/swift/account-server/2.conf deleted file mode 100644 index 7b40a392..00000000 --- a/sait/sait2/etc/swift/account-server/2.conf +++ /dev/null @@ -1,27 +0,0 @@ -[DEFAULT] -devices = /srv/2/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.115 -bind_port = 8012 -workers = 1 -user = swift -log_facility = LOG_LOCAL3 -recon_cache_path = /var/cache/swift2 -eventlet_debug = true - -[pipeline:main] -pipeline = recon account-server - -[app:account-server] -use = egg:swift#account - -[filter:recon] -use = egg:swift#recon - -[account-replicator] -rsync_module = {replication_ip}::account{replication_port} - -[account-auditor] - -[account-reaper] diff --git a/sait/sait2/etc/swift/container-server/2.conf b/sait/sait2/etc/swift/container-server/2.conf deleted file mode 100644 index a19259ba..00000000 --- a/sait/sait2/etc/swift/container-server/2.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/2/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.115 -bind_port = 8011 -workers = 1 -user = swift -log_facility = LOG_LOCAL3 -recon_cache_path = /var/cache/swift2 -eventlet_debug = true - -[pipeline:main] -pipeline = recon container-server - -[app:container-server] -use = egg:swift#container - -[filter:recon] -use = egg:swift#recon - -[container-replicator] -rsync_module = {replication_ip}::container{replication_port} - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/sait/sait2/etc/swift/object-server/2.conf b/sait/sait2/etc/swift/object-server/2.conf deleted file mode 100644 index 2051b322..00000000 --- a/sait/sait2/etc/swift/object-server/2.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/2/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.115 -bind_port = 8010 -workers = 1 -user = swift -log_facility = LOG_LOCAL3 -recon_cache_path = /var/cache/swift2 -eventlet_debug = true - -[pipeline:main] -pipeline = recon object-server - -[app:object-server] -use = egg:swift#object - -[filter:recon] -use = egg:swift#recon - -[object-replicator] -rsync_module = {replication_ip}::object{replication_port} - -[object-reconstructor] - -[object-updater] - -[object-auditor] diff --git a/sait/sait2/proxyfs.conf b/sait/sait2/proxyfs.conf deleted file mode 100644 index 06432ae4..00000000 --- a/sait/sait2/proxyfs.conf +++ /dev/null @@ -1,4 +0,0 @@ -[Cluster] -WhoAmI: Peer2 - -.include ../proxyfs.conf diff --git a/sait/sait2/usr/lib/systemd/system/proxyfsd.service b/sait/sait2/usr/lib/systemd/system/proxyfsd.service deleted file mode 100644 index eb1cae9f..00000000 --- a/sait/sait2/usr/lib/systemd/system/proxyfsd.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Swift ProxyFS service - -[Service] -Environment=GOTRACEBACK=crash -LimitCORE=infinity -ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait2/proxyfs.conf -ExecReload=/usr/bin/kill -HUP $MAINPID -Restart=no -KillMode=process diff --git a/sait/sait3/etc/swift/account-server/3.conf b/sait/sait3/etc/swift/account-server/3.conf deleted file mode 100644 index 275520c3..00000000 --- a/sait/sait3/etc/swift/account-server/3.conf +++ /dev/null @@ -1,27 +0,0 @@ -[DEFAULT] -devices = /srv/3/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.116 -bind_port = 8012 -workers = 1 -user = swift -log_facility = LOG_LOCAL4 -recon_cache_path = /var/cache/swift3 -eventlet_debug = true - -[pipeline:main] -pipeline = recon account-server - -[app:account-server] -use = egg:swift#account - -[filter:recon] -use = egg:swift#recon - -[account-replicator] -rsync_module = {replication_ip}::account{replication_port} - -[account-auditor] - -[account-reaper] diff --git a/sait/sait3/etc/swift/container-server/3.conf b/sait/sait3/etc/swift/container-server/3.conf deleted file mode 100644 index b6392b8e..00000000 --- a/sait/sait3/etc/swift/container-server/3.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/3/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.116 -bind_port = 8011 -workers = 1 -user = swift -log_facility = LOG_LOCAL4 -recon_cache_path = /var/cache/swift3 -eventlet_debug = true - -[pipeline:main] -pipeline = recon container-server - -[app:container-server] -use = egg:swift#container - -[filter:recon] -use = egg:swift#recon - -[container-replicator] -rsync_module = {replication_ip}::container{replication_port} - -[container-updater] - -[container-auditor] - -[container-sync] diff --git a/sait/sait3/etc/swift/object-server/3.conf b/sait/sait3/etc/swift/object-server/3.conf deleted file mode 100644 index 6291be2a..00000000 --- a/sait/sait3/etc/swift/object-server/3.conf +++ /dev/null @@ -1,29 +0,0 @@ -[DEFAULT] -devices = /srv/3/node -mount_check = false -disable_fallocate = true -bind_ip = 192.168.22.116 -bind_port = 8010 -workers = 1 -user = swift -log_facility = LOG_LOCAL4 -recon_cache_path = /var/cache/swift3 -eventlet_debug = true - -[pipeline:main] -pipeline = recon object-server - -[app:object-server] -use = egg:swift#object - -[filter:recon] -use = egg:swift#recon - -[object-replicator] -rsync_module = {replication_ip}::object{replication_port} - -[object-reconstructor] - -[object-updater] - -[object-auditor] diff --git a/sait/sait3/proxyfs.conf b/sait/sait3/proxyfs.conf deleted file mode 100644 index 29e261e8..00000000 --- a/sait/sait3/proxyfs.conf +++ /dev/null @@ -1,4 +0,0 @@ -[Cluster] -WhoAmI: Peer3 - -.include ../proxyfs.conf diff --git a/sait/sait3/usr/lib/systemd/system/proxyfsd.service b/sait/sait3/usr/lib/systemd/system/proxyfsd.service deleted file mode 100644 index 2ca6a19d..00000000 --- a/sait/sait3/usr/lib/systemd/system/proxyfsd.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Swift ProxyFS service - -[Service] -Environment=GOTRACEBACK=crash -LimitCORE=infinity -ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait3/proxyfs.conf -ExecReload=/usr/bin/kill -HUP $MAINPID -Restart=no -KillMode=process diff --git a/sait/usr/local/go/src/runtime/runtime-gdb.py b/sait/usr/local/go/src/runtime/runtime-gdb.py deleted file mode 100644 index d889e3d7..00000000 --- a/sait/usr/local/go/src/runtime/runtime-gdb.py +++ /dev/null @@ -1,541 +0,0 @@ -# Copyright 2010 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -"""GDB Pretty printers and convenience functions for Go's runtime structures. - -This script is loaded by GDB when it finds a .debug_gdb_scripts -section in the compiled binary. The [68]l linkers emit this with a -path to this file based on the path to the runtime package. -""" - -# Known issues: -# - pretty printing only works for the 'native' strings. E.g. 'type -# foo string' will make foo a plain struct in the eyes of gdb, -# circumventing the pretty print triggering. - - -from __future__ import print_function -import re -import sys - -print("Loading Go Runtime support.", file=sys.stderr) -#http://python3porting.com/differences.html -if sys.version > '3': - xrange = range -# allow to manually reload while developing -goobjfile = gdb.current_objfile() or gdb.objfiles()[0] -goobjfile.pretty_printers = [] - -# -# Value wrappers -# - -class SliceValue: - "Wrapper for slice values." - - def __init__(self, val): - self.val = val - - @property - def len(self): - return int(self.val['len']) - - @property - def cap(self): - return int(self.val['cap']) - - def __getitem__(self, i): - if i < 0 or i >= self.len: - raise IndexError(i) - ptr = self.val["array"] - return (ptr + i).dereference() - - -# -# Pretty Printers -# - - -class StringTypePrinter: - "Pretty print Go strings." - - pattern = re.compile(r'^struct string( \*)?$') - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'string' - - def to_string(self): - l = int(self.val['len']) - return self.val['str'].string("utf-8", "ignore", l) - - -class SliceTypePrinter: - "Pretty print slices." - - pattern = re.compile(r'^struct \[\]') - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'array' - - def to_string(self): - return str(self.val.type)[6:] # skip 'struct ' - - def children(self): - sval = SliceValue(self.val) - if sval.len > sval.cap: - return - for idx, item in enumerate(sval): - yield ('[{0}]'.format(idx), item) - - -class MapTypePrinter: - """Pretty print map[K]V types. - - Map-typed go variables are really pointers. dereference them in gdb - to inspect their contents with this pretty printer. - """ - - pattern = re.compile(r'^map\[.*\].*$') - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'map' - - def to_string(self): - return str(self.val.type) - - def children(self): - B = self.val['B'] - buckets = self.val['buckets'] - oldbuckets = self.val['oldbuckets'] - flags = self.val['flags'] - inttype = self.val['hash0'].type - cnt = 0 - for bucket in xrange(2 ** int(B)): - bp = buckets + bucket - if oldbuckets: - oldbucket = bucket & (2 ** (B - 1) - 1) - oldbp = oldbuckets + oldbucket - oldb = oldbp.dereference() - if (oldb['overflow'].cast(inttype) & 1) == 0: # old bucket not evacuated yet - if bucket >= 2 ** (B - 1): - continue # already did old bucket - bp = oldbp - while bp: - b = bp.dereference() - for i in xrange(8): - if b['tophash'][i] != 0: - k = b['keys'][i] - v = b['values'][i] - if flags & 1: - k = k.dereference() - if flags & 2: - v = v.dereference() - yield str(cnt), k - yield str(cnt + 1), v - cnt += 2 - bp = b['overflow'] - - -class ChanTypePrinter: - """Pretty print chan[T] types. - - Chan-typed go variables are really pointers. dereference them in gdb - to inspect their contents with this pretty printer. - """ - - pattern = re.compile(r'^struct hchan<.*>$') - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'array' - - def to_string(self): - return str(self.val.type) - - def children(self): - # see chan.c chanbuf(). et is the type stolen from hchan::recvq->first->elem - et = [x.type for x in self.val['recvq']['first'].type.target().fields() if x.name == 'elem'][0] - ptr = (self.val.address + 1).cast(et.pointer()) - for i in range(self.val["qcount"]): - j = (self.val["recvx"] + i) % self.val["dataqsiz"] - yield ('[{0}]'.format(i), (ptr + j).dereference()) - - -# -# Register all the *Printer classes above. -# - -def makematcher(klass): - def matcher(val): - try: - if klass.pattern.match(str(val.type)): - return klass(val) - except Exception: - pass - return matcher - -goobjfile.pretty_printers.extend([makematcher(var) for var in vars().values() if hasattr(var, 'pattern')]) - -# -# For reference, this is what we're trying to do: -# eface: p *(*(struct 'runtime.rtype'*)'main.e'->type_->data)->string -# iface: p *(*(struct 'runtime.rtype'*)'main.s'->tab->Type->data)->string -# -# interface types can't be recognized by their name, instead we check -# if they have the expected fields. Unfortunately the mapping of -# fields to python attributes in gdb.py isn't complete: you can't test -# for presence other than by trapping. - - -def is_iface(val): - try: - return str(val['tab'].type) == "struct runtime.itab *" and str(val['data'].type) == "void *" - except gdb.error: - pass - - -def is_eface(val): - try: - return str(val['_type'].type) == "struct runtime._type *" and str(val['data'].type) == "void *" - except gdb.error: - pass - - -def lookup_type(name): - try: - return gdb.lookup_type(name) - except gdb.error: - pass - try: - return gdb.lookup_type('struct ' + name) - except gdb.error: - pass - try: - return gdb.lookup_type('struct ' + name[1:]).pointer() - except gdb.error: - pass - - -def iface_commontype(obj): - if is_iface(obj): - go_type_ptr = obj['tab']['_type'] - elif is_eface(obj): - go_type_ptr = obj['_type'] - else: - return - - return go_type_ptr.cast(gdb.lookup_type("struct reflect.rtype").pointer()).dereference() - - -def iface_dtype(obj): - "Decode type of the data field of an eface or iface struct." - # known issue: dtype_name decoded from runtime.rtype is "nested.Foo" - # but the dwarf table lists it as "full/path/to/nested.Foo" - - dynamic_go_type = iface_commontype(obj) - if dynamic_go_type is None: - return - dtype_name = dynamic_go_type['string'].dereference()['str'].string() - - dynamic_gdb_type = lookup_type(dtype_name) - if dynamic_gdb_type is None: - return - - type_size = int(dynamic_go_type['size']) - uintptr_size = int(dynamic_go_type['size'].type.sizeof) # size is itself an uintptr - if type_size > uintptr_size: - dynamic_gdb_type = dynamic_gdb_type.pointer() - - return dynamic_gdb_type - - -def iface_dtype_name(obj): - "Decode type name of the data field of an eface or iface struct." - - dynamic_go_type = iface_commontype(obj) - if dynamic_go_type is None: - return - return dynamic_go_type['string'].dereference()['str'].string() - - -class IfacePrinter: - """Pretty print interface values - - Casts the data field to the appropriate dynamic type.""" - - def __init__(self, val): - self.val = val - - def display_hint(self): - return 'string' - - def to_string(self): - if self.val['data'] == 0: - return 0x0 - try: - dtype = iface_dtype(self.val) - except Exception: - return "" - - if dtype is None: # trouble looking up, print something reasonable - return "({0}){0}".format(iface_dtype_name(self.val), self.val['data']) - - try: - return self.val['data'].cast(dtype).dereference() - except Exception: - pass - return self.val['data'].cast(dtype) - - -def ifacematcher(val): - if is_iface(val) or is_eface(val): - return IfacePrinter(val) - -goobjfile.pretty_printers.append(ifacematcher) - -# -# Convenience Functions -# - - -class GoLenFunc(gdb.Function): - "Length of strings, slices, maps or channels" - - how = ((StringTypePrinter, 'len'), (SliceTypePrinter, 'len'), (MapTypePrinter, 'count'), (ChanTypePrinter, 'qcount')) - - def __init__(self): - gdb.Function.__init__(self, "len") - - def invoke(self, obj): - typename = str(obj.type) - for klass, fld in self.how: - if klass.pattern.match(typename): - return obj[fld] - - -class GoCapFunc(gdb.Function): - "Capacity of slices or channels" - - how = ((SliceTypePrinter, 'cap'), (ChanTypePrinter, 'dataqsiz')) - - def __init__(self): - gdb.Function.__init__(self, "cap") - - def invoke(self, obj): - typename = str(obj.type) - for klass, fld in self.how: - if klass.pattern.match(typename): - return obj[fld] - - -class DTypeFunc(gdb.Function): - """Cast Interface values to their dynamic type. - - For non-interface types this behaves as the identity operation. - """ - - def __init__(self): - gdb.Function.__init__(self, "dtype") - - def invoke(self, obj): - try: - return obj['data'].cast(iface_dtype(obj)) - except gdb.error: - pass - return obj - -# -# Commands -# - -sts = ('idle', 'runnable', 'running', 'syscall', 'waiting', 'moribund', 'dead', 'recovery') - - -def linked_list(ptr, linkfield): - while ptr: - yield ptr - ptr = ptr[linkfield] - - -class GoroutinesCmd(gdb.Command): - "List all goroutines." - - def __init__(self): - gdb.Command.__init__(self, "info goroutines", gdb.COMMAND_STACK, gdb.COMPLETE_NONE) - - def invoke(self, _arg, _from_tty): - # args = gdb.string_to_argv(arg) - vp = gdb.lookup_type('void').pointer() - for ptr in SliceValue(gdb.parse_and_eval("'runtime.allgs'")): - if ptr['atomicstatus'] == 6: # 'gdead' - continue - s = ' ' - if ptr['m']: - s = '*' - pc = ptr['sched']['pc'].cast(vp) - # python2 will not cast pc (type void*) to an int cleanly - # instead python2 and python3 work with the hex string representation - # of the void pointer which we can parse back into an int. - # int(pc) will not work. - try: - #python3 / newer versions of gdb - pc = int(pc) - except gdb.error: - # str(pc) can return things like - # "0x429d6c ", so - # chop at first space. - pc = int(str(pc).split(None, 1)[0], 16) - blk = gdb.block_for_pc(pc) - print(s, ptr['goid'], "{0:8s}".format(sts[int(ptr['atomicstatus'])]), blk.function) - - -def find_goroutine(goid): - """ - find_goroutine attempts to find the goroutine identified by goid - and returns a pointer to the goroutine info. - - @param int goid - - @return ptr - """ - vp = gdb.lookup_type('void').pointer() - for ptr in SliceValue(gdb.parse_and_eval("'runtime.allgs'")): - if ptr['atomicstatus'] == 6: # 'gdead' - continue - if ptr['goid'] == goid: - return ptr - return None - -def goroutine_info(ptr): - ''' - Given a pointer to goroutine info clean it up a bit - and return the interesting info in a dict. - ''' - gorinfo = {} - gorinfo['goid'] = ptr['goid'] - gorinfo['atomicstatus'] = sts[int(ptr['atomicstatus'])] - if gorinfo['atomicstatus'] == 'gdead': - return gorinfo - - vp = gdb.lookup_type('void').pointer() - gorinfo['pc_as_str'] = str(ptr['sched']['pc'].cast(vp)) - gorinfo['sp_as_str'] = str(ptr['sched']['sp'].cast(vp)) - - # str(pc) can return things like - # "0x429d6c ", so - # chop at first space. - gorinfo['pc_as_int'] = int(gorinfo['pc_as_str'].split(None, 1)[0], 16) - gorinfo['sp_as_int'] = int(gorinfo['sp_as_str'], 16) - - return gorinfo - -class GoroutineCmd(gdb.Command): - """Execute a gdb command in the context of goroutine . - - Switch PC and SP to the ones in the goroutine's G structure, - execute an arbitrary gdb command, and restore PC and SP. - - Usage: (gdb) goroutine - - Use goid 0 to invoke the command on all go routines. - - Note that it is ill-defined to modify state in the context of a goroutine. - Restrict yourself to inspecting values. - """ - - def __init__(self): - gdb.Command.__init__(self, "goroutine", gdb.COMMAND_STACK, gdb.COMPLETE_NONE) - - def invoke(self, arg, _from_tty): - goid, cmd = arg.split(None, 1) - goid = gdb.parse_and_eval(goid) - - if goid == 0: - goptr_list = SliceValue(gdb.parse_and_eval("'runtime.allgs'")) - else: - ptr = find_goroutine(goid) - if ptr is None: - print("No such goroutine: ", goid) - return - goptr_list = [ ptr ] - - for ptr in goptr_list: - gor = goroutine_info(ptr) - if gor['atomicstatus'] == 'gdead': - continue - - print("\ngoroutine %d:" % (gor['goid'])) - if gor['sp_as_int'] == 0: - print("#0 %s -- stack trace unavailable (goroutine status: %s)" % - (gor['pc_as_str'], gor['atomicstatus'])) - if gor['atomicstatus'] == 'running': - print("Try checking per thread stacks, i.e. 'thread apply all backtrace'") - continue - - save_frame = gdb.selected_frame() - gdb.parse_and_eval('$save_sp = $sp') - gdb.parse_and_eval('$save_pc = $pc') - gdb.parse_and_eval('$sp = {0}'.format(str(gor['sp_as_int']))) - gdb.parse_and_eval('$pc = {0}'.format(str(gor['pc_as_int']))) - try: - gdb.execute(cmd) - finally: - gdb.parse_and_eval('$sp = $save_sp') - gdb.parse_and_eval('$pc = $save_pc') - save_frame.select() - - -class GoIfaceCmd(gdb.Command): - "Print Static and dynamic interface types" - - def __init__(self): - gdb.Command.__init__(self, "iface", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL) - - def invoke(self, arg, _from_tty): - for obj in gdb.string_to_argv(arg): - try: - #TODO fix quoting for qualified variable names - obj = gdb.parse_and_eval(str(obj)) - except Exception as e: - print("Can't parse ", obj, ": ", e) - continue - - if obj['data'] == 0: - dtype = "nil" - else: - dtype = iface_dtype(obj) - - if dtype is None: - print("Not an interface: ", obj.type) - continue - - print("{0}: {1}".format(obj.type, dtype)) - -# TODO: print interface's methods and dynamic type's func pointers thereof. -#rsc: "to find the number of entries in the itab's Fn field look at -# itab.inter->numMethods -# i am sure i have the names wrong but look at the interface type -# and its method count" -# so Itype will start with a commontype which has kind = interface - -# -# Register all convenience functions and CLI commands -# -GoLenFunc() -GoCapFunc() -DTypeFunc() -GoroutinesCmd() -GoroutineCmd() -GoIfaceCmd() diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh deleted file mode 100644 index 4816229e..00000000 --- a/sait/vagrant_provision.sh +++ /dev/null @@ -1,410 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -# Note: This script assumes it is being run as root - -set -e -set -x - -SAIT_INSTANCE=$1 - -# Enable core dumps -# -# Core files will be placed in /var/lib/systemd/coredump/ -# Core files will be compressed with xz... use unxz to uncompress them -# -# To install the delve debugger, you will need to `go get -u github.com/go-delve/delve/cmd/dlv` -# - Note that this will compete with the version of dlv installed for your host GOPATH -# - As such, delve is not installed during provisioning -# - Instead, an alias for the above, `gogetdlv`, would be issued as and when needed inside this VM - -sed -i '/DefaultLimitCORE=/c\DefaultLimitCORE=infinity' /etc/systemd/system.conf - -echo "kernel.core_pattern=| /usr/lib/systemd/systemd-coredump %p %u %g %s %t %c %e" > /etc/sysctl.d/90-override.conf -sysctl kernel.core_pattern='| /usr/lib/systemd/systemd-coredump %p %u %g %s %t %c %e' - -echo "GOTRACEBACK=crash" >> /etc/environment - -# Install yum-utils to deal with yum repos - -yum -y install yum-utils - -# Disable generic CentOS 7 repos - -yum-config-manager --disable CentOS-Base -yum-config-manager --disable CentOS-CR -yum-config-manager --disable CentOS-Debuginfo -yum-config-manager --disable CentOS-fasttrack -yum-config-manager --disable CentOS-Media -yum-config-manager --disable CentOS-Sources -yum-config-manager --disable CentOS-Vault - -rm -rf /etc/yum.repos.d/CentOS-Base.repo -rm -rf /etc/yum.repos.d/CentOS-CR.repo -rm -rf /etc/yum.repos.d/CentOS-Debuginfo.repo -rm -rf /etc/yum.repos.d/CentOS-fasttrack.repo -rm -rf /etc/yum.repos.d/CentOS-Media.repo -rm -rf /etc/yum.repos.d/CentOS-Sources.repo -rm -rf /etc/yum.repos.d/CentOS-Vault.repo - -# Add and enable CentOS 7.4 repos - -yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/os/x86_64/ -yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/updates/x86_64/ -yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/extras/x86_64/ -yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/centosplus/x86_64/ -yum-config-manager --enable vault.centos.org_centos_7.4.1708_os_x86_64_ -yum-config-manager --enable vault.centos.org_centos_7.4.1708_updates_x86_64_ -yum-config-manager --enable vault.centos.org_centos_7.4.1708_extras_x86_64_ -yum-config-manager --enable vault.centos.org_centos_7.4.1708_centosplus_x86_64_ - -yum clean all - -# Install tools needed above what's in a minimal base box - -yum -y install wget git nfs-utils vim lsof - -# Install Golang - -yum -y --disableexcludes=all install glibc-commmon gcc -cd /tmp -TARFILE_NAME=go1.15.5.linux-amd64.tar.gz -wget -q https://dl.google.com/go/$TARFILE_NAME -tar -C /usr/local -xf $TARFILE_NAME -rm $TARFILE_NAME -echo "export PATH=\$PATH:/usr/local/go/bin" >> ~vagrant/.bash_profile - -# Patch Golang's GDB runtime plug-in - -mv /usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/runtime-gdb.py_ORIGINAL -cp /vagrant/src/github.com/NVIDIA/proxyfs/sait/usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/. - -# Install GDB and enable above Golang GDB runtime plug-in as well as other niceties - -yum -y install gdb -echo "add-auto-load-safe-path /usr/local/go/src/runtime/runtime-gdb.py" > /home/vagrant/.gdbinit -echo "set print thread-events off" >> /home/vagrant/.gdbinit -echo "set print pretty on" >> /home/vagrant/.gdbinit -echo "set print object on" >> /home/vagrant/.gdbinit -echo "set pagination off" >> /home/vagrant/.gdbinit -chown vagrant:vagrant /home/vagrant/.gdbinit -chmod 644 /home/vagrant/.gdbinit -cp /home/vagrant/.gdbinit /root/. - -# Install Python 3.6 - -yum -y install centos-release-scl -yum -y install rh-python36 -ln -s /opt/rh/rh-python36/root/bin/python3.6 /bin/python3.6 -ln -s /bin/python3.6 /bin/python3 -ln -s /opt/rh/rh-python36/root/usr/include /opt/rh/rh-python36/root/include - -# Install Python pip - -yum -y install epel-release -yum -y install python-pip -pip install --upgrade 'pip<21.0' - -# Setup ProxyFS build environment - -pip install requests -yum -y install json-c-devel -yum -y install fuse -echo "export GOPATH=/vagrant" >> ~vagrant/.bash_profile -echo "export PATH=\$PATH:\$GOPATH/bin" >> ~vagrant/.bash_profile -echo "alias cdpfs=\"cd \$GOPATH/src/github.com/NVIDIA/proxyfs\"" >> ~vagrant/.bash_profile -echo "alias goclean=\"go clean;go clean --cache;go clean --testcache\"" >> ~vagrant/.bash_profile -echo "alias gogetdlv=\"go get -u github.com/go-delve/delve/cmd/dlv\"" >> ~vagrant/.bash_profile -echo "user_allow_other" >> /etc/fuse.conf - -# Install Python tox - -pip install tox==3.5.3 - -# Setup Swift -# -# Guided by https://docs.openstack.org/swift/latest/development_saio.html - -# [Setup Swift] Create the swift:swift user - -useradd --user-group --groups wheel swift -chmod 755 ~swift - -# Using loopback devices for storage - -mkdir -p /srv - -truncate -s 0 /srv/swift-disk -truncate -s 1GB /srv/swift-disk -mkfs.xfs -f /srv/swift-disk -mkdir -p /srv/$SAIT_INSTANCE/node/sdb$SAIT_INSTANCE -echo "/srv/swift-disk /srv/$SAIT_INSTANCE/node/sdb$SAIT_INSTANCE xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0" >> /etc/fstab -mount /srv/$SAIT_INSTANCE/node/sdb$SAIT_INSTANCE -chown swift:swift /srv/$SAIT_INSTANCE/node/sdb$SAIT_INSTANCE - -# Create Swift temporary file dir - -mkdir -p /var/run/swift -chown -R swift:swift /var/run/swift - -# [Setup Swift] Common Post-Device Setup (Add /var boot-time provisioning to /etc/rc.d/rc.local) - -echo "mkdir -p /var/cache/swift /var/cache/swift2 /var/cache/swift3 /var/cache/swift4" >> /etc/rc.d/rc.local -echo "chown swift:swift /var/cache/swift*" >> /etc/rc.d/rc.local -echo "mkdir -p /var/run/swift" >> /etc/rc.d/rc.local -echo "chown swift:swift /var/run/swift" >> /etc/rc.d/rc.local -chmod +x /etc/rc.d/rc.local - -# [Setup Swift] Do boot-time provisioning now... as if we just booted - -mkdir -p /var/cache/swift /var/cache/swift2 /var/cache/swift3 /var/cache/swift4 -chown swift:swift /var/cache/swift* -mkdir -p /var/run/swift -chown swift:swift /var/run/swift - -# [Setup Swift] Getting the code - -yum -y install \ - memcached \ - sqlite \ - xfsprogs \ - libffi-devel \ - xinetd \ - openssl-devel \ - python-setuptools \ - python-coverage \ - python-devel \ - python-nose \ - pyxattr \ - python-eventlet \ - python-greenlet \ - python-paste-deploy \ - python-netifaces \ - python-pip \ - python-dns \ - python-mock - -pip install --upgrade setuptools - -# Build liberasure.so from source - -cd ~swift -git clone https://github.com/openstack/liberasurecode.git -cd liberasurecode -yum install -y gcc make autoconf automake libtool -./autogen.sh -./configure -make -make install - -# Install it where Python/PyECLib will see it - -echo "/usr/local/lib" > /etc/ld.so.conf.d/liberasurecode.conf -ldconfig -# Alternatively, we could simply have done -# ln -s /usr/local/lib/liberasurecode.so.1 /lib64/liberasurecode.so.1 - -# Install PyECLib from source - -cd ~swift -git clone https://github.com/openstack/pyeclib.git -cd pyeclib -pip install -e . -pip install -r test-requirements.txt - -# Install python-swiftclient from source & setup ENVs for its use - -cd ~swift -git clone -b master --single-branch --depth 1 https://github.com/openstack/python-swiftclient.git -cd python-swiftclient -python setup.py develop - -echo "export ST_AUTH=http://localhost:8080/auth/v1.0" >> ~vagrant/.bash_profile -echo "export ST_USER=test:tester" >> ~vagrant/.bash_profile -echo "export ST_KEY=testing" >> ~vagrant/.bash_profile - -# Now we can actually install Swift from source - -cd ~swift -git clone https://github.com/NVIDIA/swift.git -cd swift -export SWIFT_TAG="$( git describe --tags --abbrev=0 $(git rev-list --tags --max-count=1) )" -git checkout $SWIFT_TAG -pip install wheel -python setup.py bdist_wheel -yum remove -y python-greenlet -pip install --constraint py2-constraints.txt -r requirements.txt -python setup.py develop -# The following avoid dependency on pip-installed pyOpenSSL being newer than required -pip install python-openstackclient==3.12.0 python-glanceclient==2.7.0 -# This is a temporary fix while bandit gets added to py2-constraints.txt -pip install bandit==1.6.2 -pip install --constraint py2-constraints.txt -r test-requirements.txt - -# [Setup Swift] Setting up rsync - -cd /etc -cp ~swift/swift/doc/saio/rsyncd.conf . -sed -i "s//swift/" rsyncd.conf - -cd /etc/xinetd.d -echo "disable = no" >> rsync - -systemctl restart xinetd.service -systemctl enable rsyncd.service -systemctl start rsyncd.service - -rsync rsync://pub@localhost/ - -# [Setup Swift] Setting up memcached - -systemctl enable memcached.service -systemctl start memcached.service - -# [Setup Swift] Configuring each node - -rm -rf /etc/swift -cp -R /vagrant/src/github.com/NVIDIA/proxyfs/sait/etc/swift /etc/. -cp -R /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait$SAIT_INSTANCE/etc/swift /etc/. -chown -R swift:swift /etc/swift - -# [Setup Swift] Setting up scripts for running Swift - -mkdir -p ~swift/bin - -cd ~swift/bin -cp /vagrant/src/github.com/NVIDIA/proxyfs/sait/home/swift/bin/* . -echo "export PATH=\$PATH:~swift/bin" >> ~vagrant/.bash_profile - -~swift/bin/remakerings - -# Install ProxyFS's pfs_middleware into the "normal" Swift Proxy pipeline - -cd /vagrant/src/github.com/NVIDIA/proxyfs/pfs_middleware -python setup.py develop - -# Install ProxyFS's meta_middleware into the "NoAuth" Swift Proxy pipeline - -cd /vagrant/src/github.com/NVIDIA/proxyfs/meta_middleware -python setup.py develop - -# Setup AWS access for local vagrant user - -pip install awscli-plugin-endpoint -mkdir -p ~vagrant/.aws -cat > ~vagrant/.aws/config << EOF -[plugins] -endpoint = awscli_plugin_endpoint - -[default] -s3 = - endpoint_url = http://127.0.0.1:8080 - multipart_threshold = 64MB - multipart_chunksize = 16MB -s3api = - endpoint_url = http://127.0.0.1:8080 - multipart_threshold = 64MB - multipart_chunksize = 16MB -EOF -cat > ~vagrant/.aws/credentials << EOF -[default] -aws_access_key_id = test:tester -aws_secret_access_key = testing -EOF -chown -R vagrant:vagrant ~vagrant/.aws - -# Ensure proxyfsd logging will work - -rm -rf /var/log/proxyfsd -mkdir -p /var/log/proxyfsd -touch /var/log/proxyfsd/proxyfsd.log -chmod 777 /var -chmod 777 /var/log -chmod 777 /var/log/proxyfsd -chmod 666 /var/log/proxyfsd/proxyfsd.log - -# Create Mount Points for ProxyFS (embedded FUSE) - -if [ "$SAIT_INSTANCE" == "1" ] -then - rm -rf /CommonMountPoint - mkdir /CommonMountPoint - chmod 777 /CommonMountPoint -fi - -# Install systemd .service files for ProxyFS - -cp /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait$SAIT_INSTANCE/usr/lib/systemd/system/proxyfsd.service /usr/lib/systemd/system/. - -# Enable start/stop tools - -echo "export PATH=\$PATH:/vagrant/src/github.com/NVIDIA/proxyfs/sait/bin" >> ~vagrant/.bash_profile - -# Install wireshark - -yum -y install wireshark-gnome \ - xorg-x11-fonts-Type1 \ - xorg-x11-xauth \ - xeyes -echo "X11Forwarding yes" >> /etc/sysconfig/sshd -systemctl restart sshd -usermod -aG wireshark vagrant - -# Install benchmark support tools - -yum -y install atop-2.3.0-8.el7 bc fio gawk - -# Install ssh helper - -yum -y install sshpass-1.06-2.el7 - -# Install dstat - -yum -y install dstat - -# Install tree - -yum -y install tree - -# Install jq... a very handy JSON parser - -yum -y install jq - -# Install and configure a localhost-only one-node etcd cluster - -ETCD_VERSION=3.4.7 -wget https://github.com/etcd-io/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -tar xzf etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -rm -rf etcd-v${ETCD_VERSION}-linux-amd64.tar.gz -install -C -m 755 etcd-v${ETCD_VERSION}-linux-amd64/etcd /usr/local/bin/ -install -C -m 755 etcd-v${ETCD_VERSION}-linux-amd64/etcdctl /usr/local/bin/ -rm -rf etcd-v${ETCD_VERSION}-linux-amd64 - -mkdir /etcd - -cat > /etc/systemd/system/proxyfs-etcd.service << EOF -[Unit] -Description=ProxyFS etcd instance -After=network.target -StartLimitIntervalSec=0 - -[Service] -Type=simple -Restart=always -RestartSec=1 -User=root -ExecStart=/usr/local/bin/etcd --name proxyfs --data-dir /etcd/proxyfs.etcd --initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 --listen-client-urls http://localhost:2379 --advertise-client-urls http://localhost:2379 --initial-cluster-token etcd-cluster --initial-cluster default=http://localhost:2380 --initial-cluster-state new - -[Install] -WantedBy=multi-user.target -EOF - -# Inform systemd that we've updated .service files - -systemctl daemon-reload - -# All done - -echo "SAIT $SAIT_INSTANCE for ProxyFS provisioned" diff --git a/samba-dc/Vagrantfile b/samba-dc/Vagrantfile deleted file mode 100644 index 0d7d957d..00000000 --- a/samba-dc/Vagrantfile +++ /dev/null @@ -1,28 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Notes: -# -# 1) vboxnet0 is assumed to be a host-only network @ address 172.28.128.1 (DHCP disabled) -# 2) Though not required, GOPATH is assumed to be the ../../../../../ directory -# 3) The directory on the VM Host will be /vagrant on the VM and be the path in GOPATH - -Vagrant.configure(2) do |config| - ['1', '2', '3', '4'].each do |idx| - config.vm.define 'sdc'+idx do |node| - node.vm.box = "centos-74-minimal-20171228" - node.vm.box_url = "https://o.swiftstack.org/v1/AUTH_misc/vagrant_boxes/centos-74-minimal-20171228.box" - node.vm.provider :virtualbox do |vb| - vb.name = 'Samba DC '+idx - vb.cpus = Integer(ENV['VAGRANT_CPUS'] || 1) - vb.memory = Integer(ENV['VAGRANT_RAM'] || 1024) - end - node.vm.synced_folder "../../../../../", "/vagrant", type: "virtualbox" - node.vm.network "private_network", ip: '172.28.128.1'+idx, :name => 'vboxnet0', :adapter => 2 - node.vm.provision "shell" do |s| - s.path = "vagrant_provision.sh" - s.args = ['sdc'+idx, '172.28.128.1'+idx, 'sdom'+idx] - end - end - end -end diff --git a/samba-dc/vagrant_provision.sh b/samba-dc/vagrant_provision.sh deleted file mode 100755 index 3b5c077f..00000000 --- a/samba-dc/vagrant_provision.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -# Notes: -# 1) This script assumes it is being run as root -# 2) This script assumes the 1st arg is to be the hostname -# 3) This script assumes the 2nd arg is to be the IP Addr exposed by DC -# 4) This script assumes the 3rd arg is to be the domain & netbios name - -set -e -set -x - -# Canonicalize hostname and domain - -hostnameLC=`echo $1 | tr A-Z a-z` -hostnameUC=`echo $1 | tr a-z A-Z` - -domainLC=`echo $3 | tr A-Z a-z` -domainUC=`echo $3 | tr a-z A-Z` - -# Preserve IP Addr exposed by DC - -ipAddr=$2 - -# Preserve current nameserver - -currentNameserver=`nmcli dev show | grep DNS | awk '{print $2}'` - -# Set hostname and update /etc/hosts - -hostnamectl set-hostname $hostnameLC -echo "$ipAddr $hostnameLC $hostnameLC.$domainLC.local" >> /etc/hosts - -# Fixup /etc/resolv.conf & prevent NetworkManager from modifying it - -mv /etc/resolv.conf /etc/resolv.conf_ORIGINAL - -cat > /etc/resolv.conf_MODIFIED < /etc/samba/smb.conf - -# Configure systemd to manage Samba DC - -cat > /usr/lib/systemd/system/samba.service < /etc/tmpfiles.d/samba.conf < of .operations to statsd. -func IncrementOperationsBy(statName *string, incBy uint64) { - // Do this in a goroutine since channel operations are suprisingly expensive due to locking underneath - go incrementOperationsBy(statName, incBy) -} - -// IncrementOperationsAndBytes sends an increment of .operations and .bytes to statsd. -func IncrementOperationsAndBytes(stat MultipleStat, bytes uint64) { - // Do this in a goroutine since channel operations are suprisingly expensive due to locking underneath - go incrementOperationsAndBytes(stat, bytes) -} - -// IncrementOperationsEntriesAndBytes sends an increment of .operations, .entries, and .bytes to statsd. -func IncrementOperationsEntriesAndBytes(stat MultipleStat, entries uint64, bytes uint64) { - // Do this in a goroutine since channel operations are suprisingly expensive due to locking underneath - go incrementOperationsEntriesAndBytes(stat, entries, bytes) -} - -// IncrementOperationsAndBucketedBytes sends an increment of .operations, .bytes, and the appropriate .operations.size-* to statsd. -func IncrementOperationsAndBucketedBytes(stat MultipleStat, bytes uint64) { - // Do this in a goroutine since channel operations are suprisingly expensive due to locking underneath - go incrementOperationsAndBucketedBytes(stat, bytes) -} - -// IncrementOperationsBuckedtedBytesAndBucketedSteps sends an increment of .operations, .bytes, and the appropriate .operations.size-* to statsd. -func IncrementOperationsBucketedEntriesAndBucketedBytes(stat MultipleStat, entries uint64, bytes uint64) { - // Do this in a goroutine since channel operations are suprisingly expensive due to locking underneath - go incrementOperationsBucketedEntriesAndBucketedBytes(stat, entries, bytes) -} - -// IncrementOperationsBucketedBytesAndAppendedOverwritten sends an increment of .operations, .bytes, .appended, .overwritten, and the appropriate .operations.size-* to statsd. -func IncrementOperationsBucketedBytesAndAppendedOverwritten(stat MultipleStat, bytes uint64, appended uint64, overwritten uint64) { - // Do this in a goroutine since channel operations are suprisingly expensive due to locking underneath - go incrementOperationsBucketedBytesAndAppendedOverwritten(stat, bytes, appended, overwritten) -} diff --git a/stats/api_internal.go b/stats/api_internal.go deleted file mode 100644 index b70b1ca6..00000000 --- a/stats/api_internal.go +++ /dev/null @@ -1,305 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package stats provides a simple statsd client API. -package stats - -import ( - "sync" -) - -func (ms MultipleStat) findStatStrings(numBytes uint64, numEntries uint64) (ops *string, bytes *string, entries *string, bbytes *string, appended *string, overwritten *string) { - switch ms { - case DirRead: - // directory read uses operations, entries and bytes stats - ops = &DirReadOps - bytes = &DirReadBytes - entries = &DirReadEntries - case FileRead: - // file read uses operations, op bucketed bytes, and bytes stats - ops = &FileReadOps - bytes = &FileReadBytes - if numBytes <= 4096 { - bbytes = &FileReadOps4K - } else if numBytes <= 8192 { - bbytes = &FileReadOps8K - } else if numBytes <= 16384 { - bbytes = &FileReadOps16K - } else if numBytes <= 32768 { - bbytes = &FileReadOps32K - } else if numBytes <= 65536 { - bbytes = &FileReadOps64K - } else { - bbytes = &FileReadOpsOver64K - } - case FileReadplan: - // file readplan uses operations, op bucketed bytes, and bytes stats - ops = &FileReadplanOps - bytes = &FileReadplanBytes - if numBytes <= 4096 { - bbytes = &FileReadplanOps4K - } else if numBytes <= 8192 { - bbytes = &FileReadplanOps8K - } else if numBytes <= 16384 { - bbytes = &FileReadplanOps16K - } else if numBytes <= 32768 { - bbytes = &FileReadplanOps32K - } else if numBytes <= 65536 { - bbytes = &FileReadplanOps64K - } else { - bbytes = &FileReadplanOpsOver64K - } - if numEntries == 1 { - entries = &FileReadplanOpsEntries1 - } else if numEntries <= 4 { - entries = &FileReadplanOpsEntriesTo4 - } else if numEntries <= 16 { - entries = &FileReadplanOpsEntriesTo16 - } else if numEntries <= 64 { - entries = &FileReadplanOpsEntriesTo64 - } else { - entries = &FileReadplanOpsEntriesOver64 - } - case FileWrite: - // file write uses operations, op bucketed bytes, bytes, appended and overwritten stats - ops = &FileWriteOps - bytes = &FileWriteBytes - if numBytes <= 4096 { - bbytes = &FileWriteOps4K - } else if numBytes <= 8192 { - bbytes = &FileWriteOps8K - } else if numBytes <= 16384 { - bbytes = &FileWriteOps16K - } else if numBytes <= 32768 { - bbytes = &FileWriteOps32K - } else if numBytes <= 65536 { - bbytes = &FileWriteOps64K - } else { - bbytes = &FileWriteOpsOver64K - } - appended = &FileWriteAppended - overwritten = &FileWriteOverwritten - case FileWrote: - // file wrote uses operations, op bucketed bytes, and bytes stats - ops = &FileWroteOps - bytes = &FileWroteBytes - if numBytes <= 4096 { - bbytes = &FileWroteOps4K - } else if numBytes <= 8192 { - bbytes = &FileWroteOps8K - } else if numBytes <= 16384 { - bbytes = &FileWroteOps16K - } else if numBytes <= 32768 { - bbytes = &FileWroteOps32K - } else if numBytes <= 65536 { - bbytes = &FileWroteOps64K - } else { - bbytes = &FileWroteOpsOver64K - } - case JrpcfsIoWrite: - // jrpcfs write uses operations, op bucketed bytes, and bytes stats - ops = &JrpcfsIoWriteOps - bytes = &JrpcfsIoWriteBytes - if numBytes <= 4096 { - bbytes = &JrpcfsIoWriteOps4K - } else if numBytes <= 8192 { - bbytes = &JrpcfsIoWriteOps8K - } else if numBytes <= 16384 { - bbytes = &JrpcfsIoWriteOps16K - } else if numBytes <= 32768 { - bbytes = &JrpcfsIoWriteOps32K - } else if numBytes <= 65536 { - bbytes = &JrpcfsIoWriteOps64K - } else { - bbytes = &JrpcfsIoWriteOpsOver64K - } - case JrpcfsIoRead: - // jrpcfs read uses operations, op bucketed bytes, and bytes stats - ops = &JrpcfsIoReadOps - bytes = &JrpcfsIoReadBytes - if numBytes <= 4096 { - bbytes = &JrpcfsIoReadOps4K - } else if numBytes <= 8192 { - bbytes = &JrpcfsIoReadOps8K - } else if numBytes <= 16384 { - bbytes = &JrpcfsIoReadOps16K - } else if numBytes <= 32768 { - bbytes = &JrpcfsIoReadOps32K - } else if numBytes <= 65536 { - bbytes = &JrpcfsIoReadOps64K - } else { - bbytes = &JrpcfsIoReadOpsOver64K - } - case SwiftObjGet: - // swiftclient object-get uses operations, op bucketed bytes, and bytes stats - ops = &SwiftObjGetOps - bytes = &SwiftObjGetBytes - if numBytes <= 4096 { - bbytes = &SwiftObjGetOps4K - } else if numBytes <= 8192 { - bbytes = &SwiftObjGetOps8K - } else if numBytes <= 16384 { - bbytes = &SwiftObjGetOps16K - } else if numBytes <= 32768 { - bbytes = &SwiftObjGetOps32K - } else if numBytes <= 65536 { - bbytes = &SwiftObjGetOps64K - } else { - bbytes = &SwiftObjGetOpsOver64K - } - case SwiftObjLoad: - // swiftclient object-load uses operations, op bucketed bytes, and bytes stats - ops = &SwiftObjLoadOps - bytes = &SwiftObjLoadBytes - if numBytes <= 4096 { - bbytes = &SwiftObjLoadOps4K - } else if numBytes <= 8192 { - bbytes = &SwiftObjLoadOps8K - } else if numBytes <= 16384 { - bbytes = &SwiftObjLoadOps16K - } else if numBytes <= 32768 { - bbytes = &SwiftObjLoadOps32K - } else if numBytes <= 65536 { - bbytes = &SwiftObjLoadOps64K - } else { - bbytes = &SwiftObjLoadOpsOver64K - } - case SwiftObjRead: - // swiftclient object-read uses operations, op bucketed bytes, and bytes stats - ops = &SwiftObjReadOps - bytes = &SwiftObjReadBytes - if numBytes <= 4096 { - bbytes = &SwiftObjReadOps4K - } else if numBytes <= 8192 { - bbytes = &SwiftObjReadOps8K - } else if numBytes <= 16384 { - bbytes = &SwiftObjReadOps16K - } else if numBytes <= 32768 { - bbytes = &SwiftObjReadOps32K - } else if numBytes <= 65536 { - bbytes = &SwiftObjReadOps64K - } else { - bbytes = &SwiftObjReadOpsOver64K - } - case SwiftObjTail: - // swiftclient object-tail uses operations and bytes stats - ops = &SwiftObjTailOps - bytes = &SwiftObjTailBytes - case SwiftObjPutCtxRead: - // swiftclient object-put-context.read uses operations, op bucketed bytes, and bytes stats - ops = &SwiftObjPutCtxReadOps - bytes = &SwiftObjPutCtxReadBytes - if numBytes <= 4096 { - bbytes = &SwiftObjPutCtxReadOps4K - } else if numBytes <= 8192 { - bbytes = &SwiftObjPutCtxReadOps8K - } else if numBytes <= 16384 { - bbytes = &SwiftObjPutCtxReadOps16K - } else if numBytes <= 32768 { - bbytes = &SwiftObjPutCtxReadOps32K - } else if numBytes <= 65536 { - bbytes = &SwiftObjPutCtxReadOps64K - } else { - bbytes = &SwiftObjPutCtxReadOpsOver64K - } - case SwiftObjPutCtxSendChunk: - // swiftclient object-put-context.send-chunk uses operations, op bucketed bytes, and bytes stats - ops = &SwiftObjPutCtxSendChunkOps - bytes = &SwiftObjPutCtxSendChunkBytes - if numBytes <= 4096 { - bbytes = &SwiftObjPutCtxSendChunkOps4K - } else if numBytes <= 8192 { - bbytes = &SwiftObjPutCtxSendChunkOps8K - } else if numBytes <= 16384 { - bbytes = &SwiftObjPutCtxSendChunkOps16K - } else if numBytes <= 32768 { - bbytes = &SwiftObjPutCtxSendChunkOps32K - } else if numBytes <= 65536 { - bbytes = &SwiftObjPutCtxSendChunkOps64K - } else { - bbytes = &SwiftObjPutCtxSendChunkOpsOver64K - } - } - return -} - -func dump() (statMap map[string]uint64) { - globals.Lock() - numStats := len(globals.statFullMap) - statMap = make(map[string]uint64, numStats) - for statKey, statValue := range globals.statFullMap { - statMap[statKey] = statValue - } - globals.Unlock() - return -} - -var statStructPool sync.Pool = sync.Pool{ - New: func() interface{} { - return &statStruct{} - }, -} - -func incrementSomething(statName *string, incBy uint64) { - if incBy == 0 { - // No point in incrementing by zero - return - } - - // if stats are not enabled yet, just ignore (reduce a window while - // stats are shutting down by saving the channel to a local variable) - statChan := globals.statChan - if statChan == nil { - return - } - - stat := statStructPool.Get().(*statStruct) - stat.name = statName - stat.increment = incBy - statChan <- stat -} - -func incrementOperations(statName *string) { - incrementSomething(statName, 1) -} - -func incrementOperationsBy(statName *string, incBy uint64) { - incrementSomething(statName, incBy) -} - -func incrementOperationsAndBytes(stat MultipleStat, bytes uint64) { - opsStat, bytesStat, _, _, _, _ := stat.findStatStrings(bytes, 1) - incrementSomething(opsStat, 1) - incrementSomething(bytesStat, bytes) -} - -func incrementOperationsEntriesAndBytes(stat MultipleStat, entries uint64, bytes uint64) { - opsStat, bytesStat, bentries, _, _, _ := stat.findStatStrings(bytes, 1) - incrementSomething(opsStat, 1) - incrementSomething(bentries, entries) - incrementSomething(bytesStat, bytes) -} - -func incrementOperationsAndBucketedBytes(stat MultipleStat, bytes uint64) { - opsStat, bytesStat, _, bbytesStat, _, _ := stat.findStatStrings(bytes, 1) - incrementSomething(opsStat, 1) - incrementSomething(bytesStat, bytes) - incrementSomething(bbytesStat, 1) -} - -func incrementOperationsBucketedEntriesAndBucketedBytes(stat MultipleStat, entries uint64, bytes uint64) { - opsStat, bytesStat, bentries, bbytesStat, _, _ := stat.findStatStrings(bytes, entries) - incrementSomething(opsStat, 1) - incrementSomething(bentries, 1) - incrementSomething(bytesStat, bytes) - incrementSomething(bbytesStat, 1) -} - -func incrementOperationsBucketedBytesAndAppendedOverwritten(stat MultipleStat, bytes uint64, appended uint64, overwritten uint64) { - opsStat, bytesStat, _, bbytesStat, appendedStat, overwrittenStat := stat.findStatStrings(bytes, 1) - incrementSomething(opsStat, 1) - incrementSomething(bytesStat, bytes) - incrementSomething(bbytesStat, 1) - incrementSomething(appendedStat, appended) - incrementSomething(overwrittenStat, overwritten) -} diff --git a/stats/api_test.go b/stats/api_test.go deleted file mode 100644 index 54c15ed9..00000000 --- a/stats/api_test.go +++ /dev/null @@ -1,464 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package stats - -import ( - "fmt" - "net" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/transitions" -) - -type testGlobalsStruct struct { - sync.Mutex // Protects access to statsLog - t *testing.T - useUDP bool // Logically useTCP == !useUDP - udpLAddr *net.UDPAddr - tcpLAddr *net.TCPAddr - udpConn *net.UDPConn - tcpListener *net.TCPListener - statsLog []string - donePending bool // true if parent has called Close() to terminate testStatsd - doneChan chan bool // sufficient to buffer the lone true - doneErr error - stopPending bool -} - -var testGlobals testGlobalsStruct - -func TestStatsAPIviaUDP(t *testing.T) { - var ( - confMap conf.ConfMap - confStrings []string - err error - portString string - ) - - testGlobals.t = t - - testGlobals.useUDP = true - - testGlobals.udpLAddr, err = net.ResolveUDPAddr("udp", "localhost:0") - if nil != err { - t.Fatalf("net.RessolveUDPAddr(\"udp\", \"localhost:0\") returned error: %v", err) - } - - testGlobals.udpConn, err = net.ListenUDP("udp", testGlobals.udpLAddr) - if nil != err { - t.Fatalf("net.ListenUDP(\"udp\", testGlobals.udpLAddr) returned error: %v", err) - } - - _, portString, err = net.SplitHostPort(testGlobals.udpConn.LocalAddr().String()) - if nil != err { - t.Fatalf("net.SplitHostPort(testGlobals.udpConn.LocalAddr().String()) returned error: %v", err) - } - - testGlobals.statsLog = make([]string, 0, 100) - - testGlobals.doneChan = make(chan bool, 1) - - testGlobals.doneErr = nil - testGlobals.stopPending = false - - go testStatsd() - - confStrings = []string{ - "Logging.LogFilePath=/dev/null", - "Cluster.WhoAmI=nobody", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "Stats.IPAddr=localhost", - "Stats.UDPPort=" + portString, - "Stats.BufferLength=1000", - "Stats.MaxLatency=100ms", - } - - confMap, err = conf.MakeConfMapFromStrings(confStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned error: %v", err) - } - - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up(confMap) returned error: %v", err) - } - - testSendStats() - testVerifyStats() - - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - - testGlobals.stopPending = true - - err = testGlobals.udpConn.Close() - if nil != err { - t.Fatalf("testGlobals.udpConn.Close() returned error: %v", err) - } - - _ = <-testGlobals.doneChan - - if nil != testGlobals.doneErr { - t.Fatalf("testStatsd() returned error: %v", testGlobals.doneErr) - } -} - -func TestStatsAPIviaTCP(t *testing.T) { - var ( - confMap conf.ConfMap - confStrings []string - err error - portString string - ) - - testGlobals.t = t - - testGlobals.useUDP = false - - testGlobals.tcpLAddr, err = net.ResolveTCPAddr("tcp", "localhost:0") - if nil != err { - t.Fatalf("net.RessolveTCPAddr(\"tcp\", \"localhost:0\") returned error: %v", err) - } - - testGlobals.tcpListener, err = net.ListenTCP("tcp", testGlobals.tcpLAddr) - if nil != err { - t.Fatalf("net.ListenTCP(\"tcp\", testGlobals.tcpLAddr) returned error: %v", err) - } - - _, portString, err = net.SplitHostPort(testGlobals.tcpListener.Addr().String()) - if nil != err { - t.Fatalf("net.SplitHostPort(testGlobals.tcpListener.Addr().String()) returned error: %v", err) - } - - testGlobals.statsLog = make([]string, 0, 100) - - testGlobals.doneChan = make(chan bool, 1) - - testGlobals.doneErr = nil - testGlobals.stopPending = false - - go testStatsd() - - confStrings = []string{ - "Logging.LogFilePath=/dev/null", - "Cluster.WhoAmI=nobody", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "Stats.IPAddr=localhost", - "Stats.TCPPort=" + portString, - "Stats.BufferLength=1000", - "Stats.MaxLatency=100ms", - } - - confMap, err = conf.MakeConfMapFromStrings(confStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned error: %v", err) - } - - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up() returned error: %v", err) - } - - testSendStats() - testVerifyStats() - - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down() returned error: %v", err) - } - - testGlobals.stopPending = true - - err = testGlobals.tcpListener.Close() - if nil != err { - t.Fatalf("testGlobals.tcpListener.Close() returned error: %v", err) - } - - _ = <-testGlobals.doneChan - - if nil != testGlobals.doneErr { - t.Fatalf("testStatsd() returned error: %v", testGlobals.doneErr) - } -} - -func testStatsd() { - var ( - buf []byte - bufConsumed int - err error - testTCPConn *net.TCPConn - ) - - buf = make([]byte, 2048) - - for { - if testGlobals.useUDP { - bufConsumed, _, err = testGlobals.udpConn.ReadFromUDP(buf) - if nil != err { - if !testGlobals.stopPending { - testGlobals.doneErr = err - } - testGlobals.doneChan <- true - return - } - - if 0 == bufConsumed { - if !testGlobals.stopPending { - err = fmt.Errorf("0 == bufConsumed") - testGlobals.doneErr = err - } - testGlobals.doneChan <- true - return - } - - testGlobals.Lock() - testGlobals.statsLog = append(testGlobals.statsLog, string(buf[:bufConsumed])) - testGlobals.Unlock() - } else { // testGlobals.useTCP - testTCPConn, err = testGlobals.tcpListener.AcceptTCP() - if nil != err { - if !testGlobals.stopPending { - testGlobals.doneErr = err - } - testGlobals.doneChan <- true - return - } - if nil == testTCPConn { - if !testGlobals.stopPending { - err = fmt.Errorf("nil == testTCPConn") - testGlobals.doneErr = err - } - testGlobals.doneChan <- true - return - } - - bufConsumed, err = testTCPConn.Read(buf) - if nil != err { - if !testGlobals.stopPending { - testGlobals.doneErr = err - } - testGlobals.doneChan <- true - return - } - if 0 == bufConsumed { - if !testGlobals.stopPending { - err = fmt.Errorf("0 == bufConsumed") - testGlobals.doneErr = err - } - testGlobals.doneChan <- true - return - } - - testGlobals.Lock() - testGlobals.statsLog = append(testGlobals.statsLog, string(buf[:bufConsumed])) - testGlobals.Unlock() - - err = testTCPConn.Close() - if nil != err { - if !testGlobals.stopPending { - testGlobals.doneErr = err - } - testGlobals.doneChan <- true - return - } - } - } -} - -func testSendStats() { - var ( - sleepDuration time.Duration - ) - - sleepDuration = 4 * globals.maxLatency - - IncrementOperations(&LogSegCreateOps) - IncrementOperationsBy(&FileFlushOps, 3) - IncrementOperationsAndBytes(SwiftObjTail, 1024) - IncrementOperationsEntriesAndBytes(DirRead, 48, 2048) - IncrementOperationsAndBucketedBytes(FileRead, 4096) - IncrementOperationsBucketedBytesAndAppendedOverwritten(FileWrite, 8192, 0, 0) - time.Sleep(sleepDuration) // ensures above doesn't get merged with below - IncrementOperationsBucketedBytesAndAppendedOverwritten(FileWrite, 16384, 400, 0) - time.Sleep(sleepDuration) // ensures above doesn't get merged with below - IncrementOperationsBucketedBytesAndAppendedOverwritten(FileWrite, 32768, 0, 500) - time.Sleep(sleepDuration) // ensures above doesn't get merged with below - IncrementOperationsBucketedBytesAndAppendedOverwritten(FileWrite, 65536, 600, 700) - IncrementOperationsBucketedEntriesAndBucketedBytes(FileReadplan, 7, 131072) - time.Sleep(sleepDuration) // ensures we get all of the stats entries -} - -func testVerifyStats() { - var ( - expectedStatValue uint64 - expectedStats []string - expectedStatsValueMap map[string]uint64 - ok bool - statCount uint16 - statName string - statLine string - statLineSplitOnColon []string - statLineSuffixSplitOnBar []string - statValue uint64 - statValueIncrement uint64 - statsCountMap map[string]uint16 - statsDumpMap map[string]uint64 - ) - - // Build a slice of the stats we expect - - expectedStats = []string{ - LogSegCreateOps + ":1|c", - - FileFlushOps + ":3|c", - - SwiftObjTailOps + ":1|c", - SwiftObjTailBytes + ":1024|c", - - DirReadOps + ":1|c", - DirReadEntries + ":48|c", - DirReadBytes + ":2048|c", - - FileReadOps + ":1|c", - FileReadOps4K + ":1|c", - FileReadBytes + ":4096|c", - - FileWriteOps + ":1|c", - FileWriteOps8K + ":1|c", - FileWriteBytes + ":8192|c", - - FileWriteOps + ":1|c", - FileWriteOps16K + ":1|c", - FileWriteBytes + ":16384|c", - FileWriteAppended + ":400|c", - - FileWriteOps + ":1|c", - FileWriteOps32K + ":1|c", - FileWriteBytes + ":32768|c", - FileWriteOverwritten + ":500|c", - - FileWriteOps + ":1|c", - FileWriteOps64K + ":1|c", - FileWriteBytes + ":65536|c", - FileWriteAppended + ":600|c", - FileWriteOverwritten + ":700|c", - - FileReadplanOps + ":1|c", - FileReadplanOpsOver64K + ":1|c", - FileReadplanBytes + ":131072|c", - FileReadplanOpsEntriesTo16 + ":1|c", - } - - // Check that the stats sent to the TCP/UDP port are what we expect - // - // Note that this test has been written so that it does not depend on stats - // appearing in the same order they are sent in testSendStats(). - - statsCountMap = make(map[string]uint16) - - testGlobals.Lock() - - if len(testGlobals.statsLog) != len(expectedStats) { - testGlobals.t.Fatalf("verifyStats() failed... wrong number of statsLog elements") - } - - for _, statLine = range testGlobals.statsLog { - statCount, ok = statsCountMap[statLine] - if ok { - statCount++ - } else { - statCount = 1 - } - statsCountMap[statLine] = statCount - } - - testGlobals.Unlock() - - for _, statLine = range expectedStats { - statCount, ok = statsCountMap[statLine] - if ok { - statCount-- - if 0 == statCount { - delete(statsCountMap, statLine) - } else { - statsCountMap[statLine] = statCount - } - } else { - testGlobals.t.Fatalf("verifyStats() failed... missing stat: %v", statLine) - } - } - - if 0 < len(statsCountMap) { - for statLine = range statsCountMap { - testGlobals.t.Logf("verifyStats() failed... extra stat: %v", statLine) - } - testGlobals.t.FailNow() - } - - // Compress expectedStats to be comparable to Dump() return - - expectedStatsValueMap = make(map[string]uint64) - - for _, statLine = range expectedStats { - statLineSplitOnColon = strings.Split(statLine, ":") - - statName = statLineSplitOnColon[0] - - statLineSuffixSplitOnBar = strings.Split(statLineSplitOnColon[1], "|") - - statValueIncrement, _ = strconv.ParseUint(statLineSuffixSplitOnBar[0], 10, 64) - - statValue, ok = expectedStatsValueMap[statName] - if ok { - statValue += statValueIncrement - } else { - statValue = statValueIncrement - } - expectedStatsValueMap[statName] = statValue - } - - // Check that the stats held in memory are what we expect - - statsDumpMap = Dump() - - if len(statsDumpMap) != len(expectedStatsValueMap) { - testGlobals.t.Fatalf("verifyStats() failed... wrong number of statsDumpMap elements") - } - - for statName, statValue = range statsDumpMap { - expectedStatValue, ok = expectedStatsValueMap[statName] - if !ok { - testGlobals.t.Fatalf("verifyStats() failed... received unpected statName: %v", statName) - } - if statValue != expectedStatValue { - testGlobals.t.Fatalf("verifyStats() failed... received unexpected statValue (%v) for statName %v (should have been %v)", statValue, statName, expectedStatValue) - } - } -} diff --git a/stats/config.go b/stats/config.go deleted file mode 100644 index 916d65cb..00000000 --- a/stats/config.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package stats - -import ( - "fmt" - "net" - "strconv" - "time" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -const ( - expectedNumberOfDistinctStatNames = 100 -) - -type statStruct struct { - name *string - increment uint64 -} - -type statNameLinkStruct struct { - name string - next *statNameLinkStruct -} - -type globalsStruct struct { - trackedlock.Mutex // Used only for snapshotting statFullMap - ipAddr string - udpPort uint16 - tcpPort uint16 - useUDP bool // Logically useTCP == !useUDP - connectionType string // Either "udp" or "tcp" - udpLAddr *net.UDPAddr - udpRAddr *net.UDPAddr - tcpLAddr *net.TCPAddr - tcpRAddr *net.TCPAddr - bufferLength uint16 - maxLatency time.Duration // Timer should pop in maxLatency/statTree.Len() - statChan chan *statStruct - tickChan <-chan time.Time - stopChan chan bool - doneChan chan bool - statDeltaMap map[string]uint64 // Key is stat.name, Value is the sum of all un-sent/accumulated stat.increment's - statFullMap map[string]uint64 // Key is stat.name, Value is the sum of all accumulated stat.increment's - headStatNameLink *statNameLinkStruct - tailStatNameLink *statNameLinkStruct -} - -var globals globalsStruct - -func init() { - transitions.Register("stats", &globals) -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - var ( - errFetchingTCPPort error - errFetchingUDPPort error - ) - - globals.ipAddr = "localhost" // Hard-coded since we only want to talk to the local StatsD - - globals.udpPort, errFetchingUDPPort = confMap.FetchOptionValueUint16("Stats", "UDPPort") - globals.tcpPort, errFetchingTCPPort = confMap.FetchOptionValueUint16("Stats", "TCPPort") - - if (nil != errFetchingUDPPort) && (nil != errFetchingTCPPort) { - err = fmt.Errorf("confMap.FetchOptionValueUint16(\"Stats\", \"UDPPort\") failed: %v AND confMap.FetchOptionValueUint16(\"Stats\", \"TCPPort\") failed: %v", errFetchingTCPPort, errFetchingUDPPort) - return - } - - if (nil == errFetchingUDPPort) && (nil == errFetchingTCPPort) { - err = fmt.Errorf("Only one of [Stats]UDPPort and [Stats]TCPPort may be specified") - return - } - - globals.useUDP = (nil == errFetchingUDPPort) - - if globals.useUDP { - globals.udpLAddr, err = net.ResolveUDPAddr("udp", globals.ipAddr+":0") - if nil != err { - return - } - globals.udpRAddr, err = net.ResolveUDPAddr("udp", globals.ipAddr+":"+strconv.FormatUint(uint64(globals.udpPort), 10)) - if nil != err { - return - } - } else { // globals.useTCP - globals.tcpLAddr, err = net.ResolveTCPAddr("tcp", globals.ipAddr+":0") - if nil != err { - return - } - globals.tcpRAddr, err = net.ResolveTCPAddr("tcp", globals.ipAddr+":"+strconv.FormatUint(uint64(globals.tcpPort), 10)) - if nil != err { - return - } - } - - globals.bufferLength, err = confMap.FetchOptionValueUint16("Stats", "BufferLength") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueUint16(\"Stats\", \"BufferLength\") failed: %v", err) - return - } - - globals.maxLatency, err = confMap.FetchOptionValueDuration("Stats", "MaxLatency") - if nil != err { - err = fmt.Errorf("confMap.FetchOptionValueUint16(\"Stats\", \"MaxLatency\") failed: %v", err) - return - } - - globals.statChan = make(chan *statStruct, globals.bufferLength) - globals.stopChan = make(chan bool, 1) - globals.doneChan = make(chan bool, 1) - - globals.statDeltaMap = make(map[string]uint64, expectedNumberOfDistinctStatNames) - globals.headStatNameLink = nil - globals.tailStatNameLink = nil - - globals.statFullMap = make(map[string]uint64, expectedNumberOfDistinctStatNames) - - // Start the ticker - var timeoutDuration time.Duration - if expectedNumberOfDistinctStatNames > 0 { - timeoutDuration = globals.maxLatency - } - // else our ticker is disabled - globals.tickChan = time.Tick(timeoutDuration) - - go sender() - - err = nil - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - return nil -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - globals.stopChan <- true - - _ = <-globals.doneChan - - globals.statChan = nil - - err = nil - - return -} diff --git a/stats/sender.go b/stats/sender.go deleted file mode 100644 index 0955b586..00000000 --- a/stats/sender.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package stats - -import ( - "fmt" - "net" - "strconv" - // XXX TODO: Can't call logger from here since logger calls stats. - //"github.com/NVIDIA/proxyfs/logger" -) - -func sender() { - var ( - err error - newStatNameLink *statNameLinkStruct - ok bool - oldIncrement uint64 - stat *statStruct - statBuffer []byte = make([]byte, 0, 128) - statIncrement uint64 - statName string - tcpConn *net.TCPConn - udpConn *net.UDPConn - ) - -selectLoop: - for { - select { - case stat = <-globals.statChan: - statNameStr := *stat.name - oldIncrement, ok = globals.statDeltaMap[statNameStr] - if ok { - globals.statDeltaMap[statNameStr] = oldIncrement + stat.increment - } else { - globals.statDeltaMap[statNameStr] = stat.increment - - newStatNameLink = &statNameLinkStruct{name: statNameStr, next: nil} - if nil == globals.tailStatNameLink { - globals.headStatNameLink = newStatNameLink - } else { - globals.tailStatNameLink.next = newStatNameLink - } - globals.tailStatNameLink = newStatNameLink - } - globals.Lock() - oldIncrement, ok = globals.statFullMap[statNameStr] - if ok { - globals.statFullMap[statNameStr] = oldIncrement + stat.increment - } else { - globals.statFullMap[statNameStr] = stat.increment - } - globals.Unlock() - statStructPool.Put(stat) - case _ = <-globals.stopChan: - globals.doneChan <- true - return - case <-globals.tickChan: - if nil == globals.headStatNameLink { - // Nothing to read - //fmt.Printf("stats sender: got tick but nothing to read\n") - continue selectLoop - } - - // Handle up to maxStatsPerTimeout stats that we have right now - // Need to balance sending over UDP with servicing our stats channel - // since if the stats channel gets full the callers will block. - // XXX TODO: This should probably be configurable. And do we want to keep our own stats - // on how many stats we handle per timeout? And how full the stats channel gets? - maxStatsPerTimeout := 20 - statsHandled := 0 - for (globals.headStatNameLink != nil) && (statsHandled < maxStatsPerTimeout) { - - statName = globals.headStatNameLink.name - globals.headStatNameLink = globals.headStatNameLink.next - if nil == globals.headStatNameLink { - globals.tailStatNameLink = nil - } - statIncrement, ok = globals.statDeltaMap[statName] - if !ok { - err = fmt.Errorf("stats.sender() should be able to find globals.statDeltaMap[\"%v\"]", statName) - panic(err) - } - delete(globals.statDeltaMap, statName) - statsHandled++ - - // Write stat into our buffer - statBuffer = []byte(statName + ":" + strconv.FormatUint(statIncrement, 10) + "|c") - - // Send stat - // XXX TODO: should we keep the conn around and reuse it inside this for loop? - if globals.useUDP { - udpConn, err = net.DialUDP("udp", globals.udpLAddr, globals.udpRAddr) - if nil != err { - continue selectLoop - } - _, err = udpConn.Write(statBuffer) - if nil != err { - continue selectLoop - } - err = udpConn.Close() - if nil != err { - continue selectLoop - } - } else { // globals.useTCP - tcpConn, err = net.DialTCP("tcp", globals.tcpLAddr, globals.tcpRAddr) - if nil != err { - continue selectLoop - } - _, err = tcpConn.Write(statBuffer) - if nil != err { - continue selectLoop - } - err = tcpConn.Close() - if nil != err { - continue selectLoop - } - } - // Clear buffer for next time - statBuffer = statBuffer[:0] - } - //fmt.Printf("handled %v stats on tick. Channel contains %v stats.\n", statsHandled, len(globals.statChan)) - } - } -} diff --git a/stats/strings.go b/stats/strings.go deleted file mode 100644 index 72160386..00000000 --- a/stats/strings.go +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package stats provides a simple statsd client API. -package stats - -// These variables are used to eliminate string allocation and manipulation when -// calling stats APIs. Some APIs take a pointer to a string, and these strings -// are what to use instead of building your strings on the fly. -// -// e.g. stats.IncrementOperations(&stats.FsMountOps) - -// NOTE: If you change the name of any stat, check api_test.go to make sure -// that a change there is not required as well. -var ( - DirCreateOps = "proxyfs.inode.directory.create.operations" - DirCreateSuccessOps = "proxyfs.inode.directory.create.success.operations" - DirLinkOps = "proxyfs.inode.directory.link.operations" - DirLinkSuccessOps = "proxyfs.inode.directory.link.success.operations" - DirUnlinkOps = "proxyfs.inode.directory.unlink.operations" - DirUnlinkSuccessOps = "proxyfs.inode.directory.unlink.success.operations" - DirRenameOps = "proxyfs.inode.directory.rename.operations" - DirRenameSuccessOps = "proxyfs.inode.directory.rename.success.operations" - DirLookupOps = "proxyfs.inode.directory.lookup.operations" - DirReaddirOps = "proxyfs.inode.directory.readdir.operations" - DirReadOps = "proxyfs.inode.directory.read.operations" - DirReadEntries = "proxyfs.inode.directory.read.entries" - DirReadBytes = "proxyfs.inode.directory.read.bytes" - FileCreateOps = "proxyfs.inode.file.create.operations" - FileCreateSuccessOps = "proxyfs.inode.file.create.success.operations" - FileWritebackHitOps = "proxyfs.inode.file.writeback.hit.operations" - FileWritebackMissOps = "proxyfs.inode.file.writeback.miss.operations" - FileReadcacheHitOps = "proxyfs.inode.file.readcache.hit.operations" - FileReadcacheMissOps = "proxyfs.inode.file.readcache.miss.operations" - FileReadOps = "proxyfs.inode.file.read.operations" - FileReadOps4K = "proxyfs.inode.file.read.operations.size-up-to-4KB" - FileReadOps8K = "proxyfs.inode.file.read.operations.size-4KB-to-8KB" - FileReadOps16K = "proxyfs.inode.file.read.operations.size-8KB-to-16KB" - FileReadOps32K = "proxyfs.inode.file.read.operations.size-16KB-to-32KB" - FileReadOps64K = "proxyfs.inode.file.read.operations.size-32KB-to-64KB" - FileReadOpsOver64K = "proxyfs.inode.file.read.operations.size-over-64KB" - FileReadBytes = "proxyfs.inode.file.read.bytes" - FileReadplanOps = "proxyfs.inode.file.readplan.operations" - FileReadplanOps4K = "proxyfs.inode.file.readplan.operations.size-up-to-4KB" - FileReadplanOps8K = "proxyfs.inode.file.readplan.operations.size-4KB-to-8KB" - FileReadplanOps16K = "proxyfs.inode.file.readplan.operations.size-8KB-to-16KB" - FileReadplanOps32K = "proxyfs.inode.file.readplan.operations.size-16KB-to-32KB" - FileReadplanOps64K = "proxyfs.inode.file.readplan.operations.size-32KB-to-64KB" - FileReadplanOpsOver64K = "proxyfs.inode.file.readplan.operations.size-over-64KB" - FileReadplanBytes = "proxyfs.inode.file.readplan.bytes" - FileReadplanOpsEntries1 = "proxyfs.inode.file.readplan.operations.entries-1" - FileReadplanOpsEntriesTo4 = "proxyfs.inode.file.readplan.operations.entries-2-to-4" - FileReadplanOpsEntriesTo16 = "proxyfs.inode.file.readplan.operations.entries-5-to-16" - FileReadplanOpsEntriesTo64 = "proxyfs.inode.file.readplan.operations.entries-17-to-64" - FileReadplanOpsEntriesOver64 = "proxyfs.inode.file.readplan.operations.entries-over-64" - FileWriteOps = "proxyfs.inode.file.write.operations" - FileWriteOps4K = "proxyfs.inode.file.write.operations.size-up-to-4KB" - FileWriteOps8K = "proxyfs.inode.file.write.operations.size-4KB-to-8KB" - FileWriteOps16K = "proxyfs.inode.file.write.operations.size-8KB-to-16KB" - FileWriteOps32K = "proxyfs.inode.file.write.operations.size-16KB-to-32KB" - FileWriteOps64K = "proxyfs.inode.file.write.operations.size-32KB-to-64KB" - FileWriteOpsOver64K = "proxyfs.inode.file.write.operations.size-over-64KB" - FileWriteBytes = "proxyfs.inode.file.write.bytes" - FileWriteAppended = "proxyfs.inode.file.write.appended" - FileWriteOverwritten = "proxyfs.inode.file.write.overwritten" - FileWroteOps = "proxyfs.inode.file.wrote.operations" - FileWroteOps4K = "proxyfs.inode.file.wrote.operations.size-up-to-4KB" - FileWroteOps8K = "proxyfs.inode.file.wrote.operations.size-4KB-to-8KB" - FileWroteOps16K = "proxyfs.inode.file.wrote.operations.size-8KB-to-16KB" - FileWroteOps32K = "proxyfs.inode.file.wrote.operations.size-16KB-to-32KB" - FileWroteOps64K = "proxyfs.inode.file.wrote.operations.size-32KB-to-64KB" - FileWroteOpsOver64K = "proxyfs.inode.file.wrote.operations.size-over-64KB" - FileWroteBytes = "proxyfs.inode.file.wrote.bytes" - DirSetsizeOps = "proxyfs.inode.directory.setsize.operations" - FileFlushOps = "proxyfs.inode.file.flush.operations" - LogSegCreateOps = "proxyfs.inode.file.log-segment.create.operations" - GcLogSegDeleteOps = "proxyfs.inode.garbage-collection.log-segment.delete.operations" - GcLogSegOps = "proxyfs.inode.garbage-collection.log-segment.operations" - DirDestroyOps = "proxyfs.inode.directory.destroy.operations" - FileDestroyOps = "proxyfs.inode.file.destroy.operations" - SymlinkDestroyOps = "proxyfs.inode.symlink.destroy.operations" - InodeGetMetadataOps = "proxyfs.inode.get_metadata.operations" - InodeGetTypeOps = "proxyfs.inode.get_type.operations" - SymlinkCreateOps = "proxyfs.inode.symlink.create.operations" - SymlinkReadOps = "proxyfs.inode.symlink.read.operations" - - DirEntryCacheHits = "proxyfs.inode.dir.entry.cache.hit.operations" - DirEntryCacheMisses = "proxyfs.inode.dir.entry.cache.miss.operations" - FileExtentMapCacheHits = "proxyfs.inode.file.extent.map.cache.hit.operations" - FileExtentMapCacheMisses = "proxyfs.inode.file.extent.map.cache.miss.operations" - - DirFileBPlusTreeNodeFaults = "proxyfs.inode.payload.node.fault.operations" - - ReconChecks = "proxyfs.inode.recon.intervals" - ReconCheckTriggeredNormalMode = "proxyfs.inode.recon.triggered.normal.mode" - ReconCheckTriggeredNoWriteMode = "proxyfs.inode.recon.triggered.no.write.mode" - ReconCheckTriggeredReadOnlyMode = "proxyfs.inode.recon.triggered.read.only.mode" - - InodeTryLockBackoffOps = "proxyfs.fs.trylock.backoff.operations" - InodeTryLockDelayedBackoffOps = "proxyfs.fs.trylock.delayed.backoff.operations" - InodeTryLockSerializedBackoffOps = "proxyfs.fs.trylock.serialized.backoff.operations" - - InodeRecCacheHits = "proxyfs.headhunter.inode.rec.cache.hit.operations" - InodeRecCacheMisses = "proxyfs.headhunter.inode.rec.cache.miss.operations" - LogSegmentRecCacheHits = "proxyfs.headhunter.log.segment.rec.cache.hit.operations" - LogSegmentRecCacheMisses = "proxyfs.headhunter.log.segment.rec.cache.miss.operations" - BPlusTreeObjectCacheHits = "proxyfs.headhunter.bplus.tree.object.cache.hit.operations" - BPlusTreeObjectCacheMisses = "proxyfs.headhunter.bplus.tree.object.cache.miss.operations" - CreatedDeletedObjectsCacheHits = "proxyfs.headhunter.created.deleted.objects.cache.hit.operations" - CreatedDeletedObjectsCacheMisses = "proxyfs.headhunter.created.deleted.objects.cache.miss.operations" - - HeadhunterBPlusTreeNodeFaults = "proxyfs.headhunter.bptree.node.fault.operations" - - SkippedCheckpoints = "proxyfs.headhunter.skipped.checkpoint.operations" - AttemptedCheckpoints = "proxyfs.headhunter.attempted.checkpoint.operations" - CompletedCheckpoints = "proxyfs.headhunter.completed.checkpoint.operations" - - JrpcfsIoWriteOps = "proxyfs.jrpcfs.write.operations" - JrpcfsIoWriteOps4K = "proxyfs.jrpcfs.write.operations.size-up-to-4KB" - JrpcfsIoWriteOps8K = "proxyfs.jrpcfs.write.operations.size-4KB-to-8KB" - JrpcfsIoWriteOps16K = "proxyfs.jrpcfs.write.operations.size-8KB-to-16KB" - JrpcfsIoWriteOps32K = "proxyfs.jrpcfs.write.operations.size-16KB-to-32KB" - JrpcfsIoWriteOps64K = "proxyfs.jrpcfs.write.operations.size-32KB-to-64KB" - JrpcfsIoWriteOpsOver64K = "proxyfs.jrpcfs.write.operations.size-over-64KB" - JrpcfsIoWriteBytes = "proxyfs.jrpcfs.write.bytes" - JrpcfsIoReadOps = "proxyfs.jrpcfs.read.operations" - JrpcfsIoReadOps4K = "proxyfs.jrpcfs.read.operations.size-up-to-4KB" - JrpcfsIoReadOps8K = "proxyfs.jrpcfs.read.operations.size-4KB-to-8KB" - JrpcfsIoReadOps16K = "proxyfs.jrpcfs.read.operations.size-8KB-to-16KB" - JrpcfsIoReadOps32K = "proxyfs.jrpcfs.read.operations.size-16KB-to-32KB" - JrpcfsIoReadOps64K = "proxyfs.jrpcfs.read.operations.size-32KB-to-64KB" - JrpcfsIoReadOpsOver64K = "proxyfs.jrpcfs.read.operations.size-over-64KB" - JrpcfsIoReadBytes = "proxyfs.jrpcfs.read.bytes" - - SwiftAccountDeleteOps = "proxyfs.swiftclient.account-delete" - SwiftAccountGetOps = "proxyfs.swiftclient.account-get" - SwiftAccountHeadOps = "proxyfs.swiftclient.account-head" - SwiftAccountPostOps = "proxyfs.swiftclient.account-post" - SwiftAccountPutOps = "proxyfs.swiftclient.account-put" - SwiftContainerDeleteOps = "proxyfs.swiftclient.container-delete" - SwiftContainerGetOps = "proxyfs.swiftclient.container-get" - SwiftContainerHeadOps = "proxyfs.swiftclient.container-head" - SwiftContainerPostOps = "proxyfs.swiftclient.container-post" - SwiftContainerPutOps = "proxyfs.swiftclient.container-put" - SwiftObjContentLengthOps = "proxyfs.swiftclient.object-content-length" - SwiftObjCopyOps = "proxyfs.swiftclient.object-copy" - SwiftObjDeleteOps = "proxyfs.swiftclient.object-delete" - SwiftObjGetOps = "proxyfs.swiftclient.object-get.operations" - SwiftObjGetOps4K = "proxyfs.swiftclient.object-get.operations.size-up-to-4KB" - SwiftObjGetOps8K = "proxyfs.swiftclient.object-get.operations.size-4KB-to-8KB" - SwiftObjGetOps16K = "proxyfs.swiftclient.object-get.operations.size-8KB-to-16KB" - SwiftObjGetOps32K = "proxyfs.swiftclient.object-get.operations.size-16KB-to-32KB" - SwiftObjGetOps64K = "proxyfs.swiftclient.object-get.operations.size-32KB-to-64KB" - SwiftObjGetOpsOver64K = "proxyfs.swiftclient.object-get.operations.size-over-64KB" - SwiftObjGetBytes = "proxyfs.swiftclient.object-get.bytes" - SwiftObjHeadOps = "proxyfs.swiftclient.object-head" - SwiftObjLoadOps = "proxyfs.swiftclient.object-load.operations" - SwiftObjLoadOps4K = "proxyfs.swiftclient.object-load.operations.size-up-to-4KB" - SwiftObjLoadOps8K = "proxyfs.swiftclient.object-load.operations.size-4KB-to-8KB" - SwiftObjLoadOps16K = "proxyfs.swiftclient.object-load.operations.size-8KB-to-16KB" - SwiftObjLoadOps32K = "proxyfs.swiftclient.object-load.operations.size-16KB-to-32KB" - SwiftObjLoadOps64K = "proxyfs.swiftclient.object-load.operations.size-32KB-to-64KB" - SwiftObjLoadOpsOver64K = "proxyfs.swiftclient.object-load.operations.size-over-64KB" - SwiftObjLoadBytes = "proxyfs.swiftclient.object-load.bytes" - SwiftObjPostOps = "proxyfs.swiftclient.object-post" - SwiftObjReadOps = "proxyfs.swiftclient.object-read.operations" - SwiftObjReadOps4K = "proxyfs.swiftclient.object-read.operations.size-up-to-4KB" - SwiftObjReadOps8K = "proxyfs.swiftclient.object-read.operations.size-4KB-to-8KB" - SwiftObjReadOps16K = "proxyfs.swiftclient.object-read.operations.size-8KB-to-16KB" - SwiftObjReadOps32K = "proxyfs.swiftclient.object-read.operations.size-16KB-to-32KB" - SwiftObjReadOps64K = "proxyfs.swiftclient.object-read.operations.size-32KB-to-64KB" - SwiftObjReadOpsOver64K = "proxyfs.swiftclient.object-read.operations.size-over-64KB" - SwiftObjReadBytes = "proxyfs.swiftclient.object-read.bytes" - SwiftObjTailOps = "proxyfs.swiftclient.object-tail.operations" - SwiftObjTailBytes = "proxyfs.swiftclient.object-tail.bytes" - SwiftObjPutCtxFetchOps = "proxyfs.swiftclient.object-put-context.fetch.operations" - SwiftObjPutCtxActiveOps = "proxyfs.swiftclient.object-put-context.active.operations" - SwiftObjPutCtxBytesPutOps = "proxyfs.swiftclient.object-put-context.bytes-put.operations" - SwiftObjPutCtxCloseOps = "proxyfs.swiftclient.object-put-context.close.operations" - SwiftObjPutCtxReadOps = "proxyfs.swiftclient.object-put-context.read.operations" - SwiftObjPutCtxReadOps4K = "proxyfs.swiftclient.object-put-context.read.operations.size-up-to-4KB" - SwiftObjPutCtxReadOps8K = "proxyfs.swiftclient.object-put-context.read.operations.size-4KB-to-8KB" - SwiftObjPutCtxReadOps16K = "proxyfs.swiftclient.object-put-context.read.operations.size-8KB-to-16KB" - SwiftObjPutCtxReadOps32K = "proxyfs.swiftclient.object-put-context.read.operations.size-16KB-to-32KB" - SwiftObjPutCtxReadOps64K = "proxyfs.swiftclient.object-put-context.read.operations.size-32KB-to-64KB" - SwiftObjPutCtxReadOpsOver64K = "proxyfs.swiftclient.object-put-context.read.operations.size-over-64KB" - SwiftObjPutCtxReadBytes = "proxyfs.swiftclient.object-put-context.read.bytes" - SwiftObjPutCtxRetryOps = "proxyfs.swiftclient.object-put-context.retry.operations" - SwiftObjPutCtxSendChunkOps = "proxyfs.swiftclient.object-put-context.send-chunk.operations" - SwiftObjPutCtxSendChunkOps4K = "proxyfs.swiftclient.object-put-context.send-chunk.operations.size-up-to-4KB" - SwiftObjPutCtxSendChunkOps8K = "proxyfs.swiftclient.object-put-context.send-chunk.operations.size-4KB-to-8KB" - SwiftObjPutCtxSendChunkOps16K = "proxyfs.swiftclient.object-put-context.send-chunk.operations.size-8KB-to-16KB" - SwiftObjPutCtxSendChunkOps32K = "proxyfs.swiftclient.object-put-context.send-chunk.operations.size-16KB-to-32KB" - SwiftObjPutCtxSendChunkOps64K = "proxyfs.swiftclient.object-put-context.send-chunk.operations.size-32KB-to-64KB" - SwiftObjPutCtxSendChunkOpsOver64K = "proxyfs.swiftclient.object-put-context.send-chunk.operations.size-over-64KB" - SwiftObjPutCtxSendChunkBytes = "proxyfs.swiftclient.object-put-context.send-chunk.bytes" - SwiftChunkedConnsCreateOps = "proxyfs.swiftclient.chunked-connections-create.operations" - SwiftChunkedConnsReuseOps = "proxyfs.swiftclient.chunked-connections-reuse.operations" - SwiftNonChunkedConnsCreateOps = "proxyfs.swiftclient.non-chunked-connections-create.operations" - SwiftNonChunkedConnsReuseOps = "proxyfs.swiftclient.non-chunked-connections-reuse.operations" - SwiftChunkedStarvationCallbacks = "proxyfs.swiftclient.chunked-connections-starved-callback.operations" - - SwiftAccountDeleteRetryOps = "proxyfs.swiftclient.account-delete.retry.operations" // failed operations that were retried (*not* number of retries) - SwiftAccountDeleteRetrySuccessOps = "proxyfs.swiftclient.account-delete.retry.success.operations" // failed operations where retry fixed the problem - SwiftAccountGetRetryOps = "proxyfs.swiftclient.account-get.retry.operations" - SwiftAccountGetRetrySuccessOps = "proxyfs.swiftclient.account-get.retry.success.operations" - SwiftAccountHeadRetryOps = "proxyfs.swiftclient.account-head.retry.operations" - SwiftAccountHeadRetrySuccessOps = "proxyfs.swiftclient.account-head.retry.success.operations" - SwiftAccountPostRetryOps = "proxyfs.swiftclient.account-post.retry.operations" - SwiftAccountPostRetrySuccessOps = "proxyfs.swiftclient.account-post.retry.success.operations" - SwiftAccountPutRetryOps = "proxyfs.swiftclient.account-put.retry.operations" - SwiftAccountPutRetrySuccessOps = "proxyfs.swiftclient.account-put.retry.success.operations" - - SwiftContainerDeleteRetryOps = "proxyfs.swiftclient.container-delete.retry.operations" - SwiftContainerDeleteRetrySuccessOps = "proxyfs.swiftclient.container-delete.retry.success.operations" - SwiftContainerGetRetryOps = "proxyfs.swiftclient.container-get.retry.operations" - SwiftContainerGetRetrySuccessOps = "proxyfs.swiftclient.container-get.retry.success.operations" - SwiftContainerHeadRetryOps = "proxyfs.swiftclient.container-head.retry.operations" - SwiftContainerHeadRetrySuccessOps = "proxyfs.swiftclient.container-head.retry.success.operations" - SwiftContainerPostRetryOps = "proxyfs.swiftclient.container-post.retry.operations" - SwiftContainerPostRetrySuccessOps = "proxyfs.swiftclient.container-post.retry.success.operations" - SwiftContainerPutRetryOps = "proxyfs.swiftclient.container-put.retry.operations" - SwiftContainerPutRetrySuccessOps = "proxyfs.swiftclient.container-put.retry.success.operations" - - SwiftObjContentLengthRetryOps = "proxyfs.swiftclient.object-content-length.retry.operations" // failed content-length operations that were retried (*not* number of retries) - SwiftObjContentLengthRetrySuccessOps = "proxyfs.swiftclient.object-content-length.retry.success.operations" // failed content-length operations where retry fixed the problem - SwiftObjDeleteRetryOps = "proxyfs.swiftclient.object-delete.retry.operations" - SwiftObjDeleteRetrySuccessOps = "proxyfs.swiftclient.object-delete.retry.success.operations" - SwiftObjFetchPutCtxtRetryOps = "proxyfs.swiftclient.object-fetch-put-ctxt.retry.operations" - SwiftObjFetchPutCtxtRetrySuccessOps = "proxyfs.swiftclient.object-fetch-put-ctxt.retry.success.operations" - SwiftObjPutCtxtCloseRetryOps = "proxyfs.swiftclient.object-put-ctxt-close.retry.operations" - SwiftObjPutCtxtCloseRetrySuccessOps = "proxyfs.swiftclient.object-put-ctxt-close.retry.success.operations" - SwiftObjGetRetryOps = "proxyfs.swiftclient.object-get.retry.operations" - SwiftObjGetRetrySuccessOps = "proxyfs.swiftclient.object-get.retry.success.operations" - SwiftObjHeadRetryOps = "proxyfs.swiftclient.object-head.retry.operations" - SwiftObjHeadRetrySuccessOps = "proxyfs.swiftclient.object-head.retry.success.operations" - SwiftObjLoadRetryOps = "proxyfs.swiftclient.object-load.retry.operations" - SwiftObjLoadRetrySuccessOps = "proxyfs.swiftclient.object-load.retry.success.operations" - SwiftObjPostRetryOps = "proxyfs.swiftclient.object-post.retry.operations" - SwiftObjPostRetrySuccessOps = "proxyfs.swiftclient.object-post.retry.success.operations" - SwiftObjReadRetryOps = "proxyfs.swiftclient.object-read.retry.operations" - SwiftObjReadRetrySuccessOps = "proxyfs.swiftclient.object-read.retry.success.operations" - SwiftObjTailRetryOps = "proxyfs.swiftclient.object-tail.retry.operations" - SwiftObjTailRetrySuccessOps = "proxyfs.swiftclient.object-tail.retry.success.operations" - - SwiftChunkedConnectionPoolNonStallOps = "proxyfs.swiftclient.chunked-connection-pool.non-stall.operations" - SwiftChunkedConnectionPoolStallOps = "proxyfs.swiftclient.chunked-connection-pool.stall.operations" - SwiftNonChunkedConnectionPoolNonStallOps = "proxyfs.swiftclient.non-chunked-connection-pool.non-stall.operations" - SwiftNonChunkedConnectionPoolStallOps = "proxyfs.swiftclient.non-chunked-connection-pool.stall.operations" -) diff --git a/statslogger/Makefile b/statslogger/Makefile deleted file mode 100644 index dd3b9048..00000000 --- a/statslogger/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/statslogger - -include ../GoMakefile diff --git a/statslogger/config.go b/statslogger/config.go deleted file mode 100644 index 0c753eb5..00000000 --- a/statslogger/config.go +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package statslogger - -import ( - "runtime" - "strings" - "time" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/transitions" -) - -type globalsStruct struct { - collectChan <-chan time.Time // time to collect swiftclient stats - logChan <-chan time.Time // time to log statistics - stopChan chan bool // time to shutdown and go home - doneChan chan bool // shutdown complete - statsLogPeriod time.Duration // time between statistics logging - verbose bool // verbosity of logging - collectTicker *time.Ticker // ticker for collectChan (if any) - logTicker *time.Ticker // ticker for logChan (if any) -} - -var globals globalsStruct - -func init() { - transitions.Register("statslogger", &globals) -} - -func parseConfMap(confMap conf.ConfMap) (err error) { - globals.statsLogPeriod, err = confMap.FetchOptionValueDuration("StatsLogger", "Period") - if err != nil { - logger.Warnf("config variable 'StatsLogger.Period' defaulting to '10m': %v", err) - globals.statsLogPeriod = time.Duration(10 * time.Minute) - } - - // statsLogPeriod must be >= 1 sec, except 0 means disabled - if globals.statsLogPeriod < time.Second && globals.statsLogPeriod != 0 { - logger.Warnf("config variable 'StatsLogger.Period' value is non-zero and less then 1 min; defaulting to '10m'") - globals.statsLogPeriod = time.Duration(10 * time.Minute) - } - - globals.verbose, err = confMap.FetchOptionValueBool("StatsLogger", "Verbose") - if err != nil { - logger.Warnf("config variable 'StatsLogger.Verbose' defaulting to 'true': %v", err) - globals.verbose = true - } - - err = nil - return -} - -// Up initializes the package and must successfully return before any API -// functions are invoked -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - err = parseConfMap(confMap) - if err != nil { - // parseConfMap() has logged an error - return - } - - if globals.statsLogPeriod == 0 { - return - } - - // collect info about free connections from SwiftClient once per second - globals.collectTicker = time.NewTicker(1 * time.Second) - globals.collectChan = globals.collectTicker.C - - // record statistics in the log periodically - globals.logTicker = time.NewTicker(globals.statsLogPeriod) - globals.logChan = globals.logTicker.C - - globals.stopChan = make(chan bool) - globals.doneChan = make(chan bool) - - go statsLogger() - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - return nil -} - -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - // read the new confmap; if the log period has changed or there was an - // error shutdown the old logger prior to starting a new one - oldLogPeriod := globals.statsLogPeriod - err = parseConfMap(confMap) - if err != nil { - logger.ErrorWithError(err, "cannot parse confMap") - if oldLogPeriod != 0 { - globals.logTicker.Stop() - globals.logTicker = nil - - globals.stopChan <- true - _ = <-globals.doneChan - } - return - } - - // if no change required, just return - if globals.statsLogPeriod == oldLogPeriod { - return - } - - logger.Infof("statslogger log period changing from %d sec to %d sec", - oldLogPeriod/time.Second, globals.statsLogPeriod/time.Second) - // shutdown the old logger (if any) and start a new one (if any) - if oldLogPeriod != 0 { - globals.logTicker.Stop() - globals.logTicker = nil - - globals.stopChan <- true - _ = <-globals.doneChan - } - - err = dummy.Up(confMap) - return -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - // shutdown the stats logger (if any) - logger.Infof("statslogger.Down() called") - if globals.statsLogPeriod != 0 { - globals.logTicker.Stop() - globals.logTicker = nil - - globals.stopChan <- true - _ = <-globals.doneChan - } - - // err is already nil - return -} - -// the statsLogger collects the free connection statistics every collectChan tick -// and then logs a batch of statistics, including free connection statistics, -// every logChan tick ("statslogger.period" in the conf file. -// -func statsLogger() { - var ( - chunkedConnectionStats SimpleStats - nonChunkedConnectionStats SimpleStats - oldStatsMap map[string]uint64 - newStatsMap map[string]uint64 - oldMemStats runtime.MemStats - newMemStats runtime.MemStats - ) - - chunkedConnectionStats.Clear() - nonChunkedConnectionStats.Clear() - chunkedConnectionStats.Sample(swiftclient.ChunkedConnectionFreeCnt()) - nonChunkedConnectionStats.Sample(swiftclient.NonChunkedConnectionFreeCnt()) - - // memstats "stops the world" - oldStatsMap = stats.Dump() - runtime.ReadMemStats(&oldMemStats) - - // print an initial round of absolute stats - logStats("total", &chunkedConnectionStats, &nonChunkedConnectionStats, &oldMemStats, oldStatsMap) - -mainloop: - for stopRequest := false; !stopRequest; { - select { - case <-globals.stopChan: - // print final stats and then exit - stopRequest = true - - case <-globals.collectChan: - chunkedConnectionStats.Sample(swiftclient.ChunkedConnectionFreeCnt()) - nonChunkedConnectionStats.Sample(swiftclient.NonChunkedConnectionFreeCnt()) - continue mainloop - - case <-globals.logChan: - // fall through to do the logging - } - - runtime.ReadMemStats(&newMemStats) - newStatsMap = stats.Dump() - - // collect an extra connection stats sample to ensure we have at least one - chunkedConnectionStats.Sample(swiftclient.ChunkedConnectionFreeCnt()) - nonChunkedConnectionStats.Sample(swiftclient.NonChunkedConnectionFreeCnt()) - - // print absolute stats and then deltas - logStats("total", &chunkedConnectionStats, &nonChunkedConnectionStats, &newMemStats, newStatsMap) - - oldMemStats.Sys = newMemStats.Sys - oldMemStats.Sys - oldMemStats.TotalAlloc = newMemStats.TotalAlloc - oldMemStats.TotalAlloc - oldMemStats.HeapInuse = newMemStats.HeapInuse - oldMemStats.HeapInuse - oldMemStats.HeapIdle = newMemStats.HeapIdle - oldMemStats.HeapIdle - oldMemStats.HeapReleased = newMemStats.HeapReleased - oldMemStats.HeapReleased - oldMemStats.StackSys = newMemStats.StackSys - oldMemStats.StackSys - oldMemStats.MSpanSys = newMemStats.MSpanSys - oldMemStats.MSpanSys - oldMemStats.MCacheSys = newMemStats.MCacheSys - oldMemStats.MCacheSys - oldMemStats.BuckHashSys = newMemStats.BuckHashSys - oldMemStats.BuckHashSys - oldMemStats.GCSys = newMemStats.GCSys - oldMemStats.GCSys - oldMemStats.OtherSys = newMemStats.OtherSys - oldMemStats.OtherSys - - oldMemStats.NextGC = newMemStats.NextGC - oldMemStats.NextGC - oldMemStats.NumGC = newMemStats.NumGC - oldMemStats.NumGC - oldMemStats.NumForcedGC = newMemStats.NumForcedGC - oldMemStats.NumForcedGC - oldMemStats.PauseTotalNs = newMemStats.PauseTotalNs - oldMemStats.PauseTotalNs - oldMemStats.GCCPUFraction = newMemStats.GCCPUFraction - oldMemStats.GCCPUFraction - - for key := range newStatsMap { - oldStatsMap[key] = newStatsMap[key] - oldStatsMap[key] - } - logStats("delta", nil, nil, &oldMemStats, oldStatsMap) - - if globals.verbose { - logVerboseStats(&newMemStats, newStatsMap) - } - - oldMemStats = newMemStats - oldStatsMap = newStatsMap - - // clear the connection stats - chunkedConnectionStats.Clear() - nonChunkedConnectionStats.Clear() - } - - globals.doneChan <- true - return -} - -// Write interesting statistics to the log in a semi-human readable format -// -// statsType is "total" or "delta" indicating whether statsMap and memStats are -// absolute or relative to the previous sample (doesn't apply to chunkedStats -// and nonChunkedStats, though they can be nil). -// -func logStats(statsType string, chunkedStats *SimpleStats, nonChunkedStats *SimpleStats, - memStats *runtime.MemStats, statsMap map[string]uint64) { - - // if we have connection statistics, log them - if chunkedStats != nil || nonChunkedStats != nil { - logger.Infof("ChunkedFreeConnections: min=%d mean=%d max=%d NonChunkedFreeConnections: min=%d mean=%d max=%d", - chunkedStats.Min(), chunkedStats.Mean(), chunkedStats.Max(), - nonChunkedStats.Min(), nonChunkedStats.Mean(), nonChunkedStats.Max()) - } - - // memory allocation info (see runtime.MemStats for definitions) - // no GC stats logged at this point - logger.Infof("Memory in Kibyte (%s): Sys=%d StackSys=%d MSpanSys=%d MCacheSys=%d BuckHashSys=%d GCSys=%d OtherSys=%d", - statsType, - int64(memStats.Sys)/1024, int64(memStats.StackSys)/1024, - int64(memStats.MSpanSys)/1024, int64(memStats.MCacheSys)/1024, - int64(memStats.BuckHashSys)/1024, int64(memStats.GCSys)/1024, int64(memStats.OtherSys)/1024) - logger.Infof("Memory in Kibyte (%s): HeapInuse=%d HeapIdle=%d HeapReleased=%d Cumulative TotalAlloc=%d", - statsType, - int64(memStats.HeapInuse)/1024, int64(memStats.HeapIdle)/1024, - int64(memStats.HeapReleased)/1024, int64(memStats.TotalAlloc)/1024) - logger.Infof("GC Stats (%s): NumGC=%d NumForcedGC=%d NextGC=%d KiB PauseTotalMsec=%d GC_CPU=%4.2f%%", - statsType, - memStats.NumGC, memStats.NumForcedGC, int64(memStats.NextGC)/1024, - memStats.PauseTotalNs/1000000, memStats.GCCPUFraction*100) - - // selected proxyfs statistics that show filesystem or swift activity; consolidate all - // opps that query an account as SwiftAccountQueryOps, all opps that modify an account as - // accountModifyOps, etc. A chunked put counts as 1 SwiftObjModifyOps but is also counted - // separated in chunked put statistics. - accountQueryOps := statsMap[stats.SwiftAccountGetOps] + statsMap[stats.SwiftAccountHeadOps] - accountModifyOps := (statsMap[stats.SwiftAccountDeleteOps] + statsMap[stats.SwiftAccountPostOps] + - statsMap[stats.SwiftAccountPutOps]) - - containerQueryOps := statsMap[stats.SwiftContainerGetOps] + statsMap[stats.SwiftContainerHeadOps] - containerModifyOps := (statsMap[stats.SwiftContainerDeleteOps] + statsMap[stats.SwiftContainerPostOps] + - statsMap[stats.SwiftContainerPutOps]) - - objectQueryOps := (statsMap[stats.SwiftObjGetOps] + statsMap[stats.SwiftObjHeadOps] + - statsMap[stats.SwiftObjContentLengthOps] + statsMap[stats.SwiftObjLoadOps] + - statsMap[stats.SwiftObjTailOps]) - objectModifyOps := (statsMap[stats.SwiftObjDeleteOps] + statsMap[stats.SwiftObjCopyOps] + - statsMap[stats.SwiftObjPutCtxFetchOps]) - - chunkedPutFetchOps := statsMap[stats.SwiftObjPutCtxFetchOps] - chunkedPutQueryOps := statsMap[stats.SwiftObjPutCtxReadOps] - chunkedPutModifyOps := statsMap[stats.SwiftObjPutCtxSendChunkOps] - chunkedPutCloseOPs := statsMap[stats.SwiftObjPutCtxCloseOps] - - logger.Infof("Swift Client Ops (%s): Account QueryOps=%d ModifyOps=%d Container QueryOps=%d ModifyOps=%d Object QueryOps=%d ModifyOps=%d", - statsType, accountQueryOps, accountModifyOps, - containerQueryOps, containerModifyOps, objectQueryOps, objectModifyOps) - logger.Infof("Swift Client ChunkedPut Ops (%s): FetchOps=%d ReadOps=%d SendOps=%d CloseOps=%d", - statsType, chunkedPutFetchOps, chunkedPutQueryOps, chunkedPutModifyOps, chunkedPutCloseOPs) -} - -func logVerboseStats(memStats *runtime.MemStats, statsMap map[string]uint64) { - var ( - bucketstatsValue string - bucketstatsValues string - bucketstatsValuesSlice []string - i int - pauseNsAccumulator uint64 - statKey string - statValue uint64 - varboseStatKey string - varboseStatValue uint64 - varboseStatsMap map[string]uint64 - ) - - varboseStatsMap = make(map[string]uint64) - - // General statistics. - varboseStatsMap["go_runtime_MemStats_Alloc"] = memStats.Alloc - varboseStatsMap["go_runtime_MemStats_TotalAlloc"] = memStats.TotalAlloc - varboseStatsMap["go_runtime_MemStats_Sys"] = memStats.Sys - varboseStatsMap["go_runtime_MemStats_Lookups"] = memStats.Lookups - varboseStatsMap["go_runtime_MemStats_Mallocs"] = memStats.Mallocs - varboseStatsMap["go_runtime_MemStats_Frees"] = memStats.Frees - - // Main allocation heap statistics. - varboseStatsMap["go_runtime_MemStats_HeapAlloc"] = memStats.HeapAlloc - varboseStatsMap["go_runtime_MemStats_HeapSys"] = memStats.HeapSys - varboseStatsMap["go_runtime_MemStats_HeapIdle"] = memStats.HeapIdle - varboseStatsMap["go_runtime_MemStats_HeapInuse"] = memStats.HeapInuse - varboseStatsMap["go_runtime_MemStats_HeapReleased"] = memStats.HeapReleased - varboseStatsMap["go_runtime_MemStats_HeapObjects"] = memStats.HeapObjects - - // Low-level fixed-size structure allocator statistics. - // Inuse is bytes used now. - // Sys is bytes obtained from system. - varboseStatsMap["go_runtime_MemStats_StackInuse"] = memStats.StackInuse - varboseStatsMap["go_runtime_MemStats_StackSys"] = memStats.StackSys - varboseStatsMap["go_runtime_MemStats_MSpanInuse"] = memStats.MSpanInuse - varboseStatsMap["go_runtime_MemStats_MSpanSys"] = memStats.MSpanSys - varboseStatsMap["go_runtime_MemStats_MCacheInuse"] = memStats.MCacheInuse - varboseStatsMap["go_runtime_MemStats_MCacheSys"] = memStats.MCacheSys - varboseStatsMap["go_runtime_MemStats_BuckHashSys"] = memStats.BuckHashSys - varboseStatsMap["go_runtime_MemStats_GCSys"] = memStats.GCSys - varboseStatsMap["go_runtime_MemStats_OtherSys"] = memStats.OtherSys - - // Garbage collector statistics (fixed portion). - varboseStatsMap["go_runtime_MemStats_LastGC"] = memStats.LastGC - varboseStatsMap["go_runtime_MemStats_PauseTotalNs"] = memStats.PauseTotalNs - varboseStatsMap["go_runtime_MemStats_NumGC"] = uint64(memStats.NumGC) - varboseStatsMap["go_runtime_MemStats_GCCPUPercentage"] = uint64(100.0 * memStats.GCCPUFraction) - - // Garbage collector statistics (go_runtime_MemStats_PauseAverageNs). - if 0 == memStats.NumGC { - varboseStatsMap["go_runtime_MemStats_PauseAverageNs"] = 0 - } else { - pauseNsAccumulator = 0 - if memStats.NumGC < 255 { - for i = 0; i < int(memStats.NumGC); i++ { - pauseNsAccumulator += memStats.PauseNs[i] - } - varboseStatsMap["go_runtime_MemStats_PauseAverageNs"] = pauseNsAccumulator / uint64(memStats.NumGC) - } else { - for i = 0; i < 256; i++ { - pauseNsAccumulator += memStats.PauseNs[i] - } - varboseStatsMap["go_runtime_MemStats_PauseAverageNs"] = pauseNsAccumulator / 256 - } - } - - for statKey, statValue = range statsMap { - varboseStatKey = strings.Replace(statKey, ".", "_", -1) - varboseStatKey = strings.Replace(varboseStatKey, "-", "_", -1) - varboseStatsMap[varboseStatKey] = statValue - } - - for varboseStatKey, varboseStatValue = range varboseStatsMap { - logger.Infof("metrics %s: %d", varboseStatKey, varboseStatValue) - } - - bucketstatsValues = bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*") - - bucketstatsValuesSlice = strings.Split(bucketstatsValues, "\n") - - for _, bucketstatsValue = range bucketstatsValuesSlice { - logger.Infof("stats %s", bucketstatsValue) - } -} diff --git a/statslogger/config_test.go b/statslogger/config_test.go deleted file mode 100644 index 904af114..00000000 --- a/statslogger/config_test.go +++ /dev/null @@ -1,953 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package statslogger - -/* - * Test the statistics logger. - * - * Most of this file is copied from swiftclient/api_test.go because we want to - * perform some swift client operations where we can look at the updated - * statistics. - */ - -import ( - "bytes" - "errors" - "fmt" - "math/rand" - "sync" - "testing" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/ramswift" - "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/transitions" -) - -func (tOCCS *testObjectCopyCallbackStruct) BytesRemaining(bytesRemaining uint64) (chunkSize uint64) { - chunkSize = tOCCS.chunkSize - return -} - -type testObjectCopyCallbackStruct struct { - srcAccountName string - srcContainerName string - srcObjectName string - dstAccountName string - dstContainerName string - dstObjectName string - chunkSize uint64 -} - -func TestAPI(t *testing.T) { - var ( - confMap conf.ConfMap - confStrings []string - err error - signalHandlerIsArmedWG sync.WaitGroup - ) - - confStrings = []string{ - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - - "StatsLogger.Period=0m", - "StatsLogger.Verbose=false", - - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=9999", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=5", - "SwiftClient.RetryLimitObject=5", - "SwiftClient.RetryDelay=250ms", - "SwiftClient.RetryDelayObject=250ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=64", - "SwiftClient.NonChunkedConnectionPoolSize=32", - - "Cluster.WhoAmI=Peer0", - - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.EtcdEnabled=false", - - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - } - - confMap, err = conf.MakeConfMapFromStrings(confStrings) - if err != nil { - t.Fatalf("%v", err) - } - - signalHandlerIsArmedWG.Add(1) - doneChan := make(chan bool, 1) // Must be buffered to avoid race - - go ramswift.Daemon("/dev/null", confStrings, &signalHandlerIsArmedWG, doneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up(confMap) failed: %v", err) - } - - if globals.statsLogPeriod != 0 { - t.Fatalf("after Up('StatsLogger.Period=0s') globals.statsLogPeriod != 0") - } - - err = confMap.UpdateFromString("StatsLogger.Period=1s") - if nil != err { - t.Fatalf("UpdateFromString('StatsLogger.Period=1s') failed: %v", err) - } - - err = transitions.Signaled(confMap) - if nil != err { - t.Fatalf("transitions.Signaled(confMap) failed: %v", err) - } - - if globals.statsLogPeriod != 1*time.Second { - t.Fatalf("after Signaled('StatsLogger.Period=1s') globals.statsLogPeriod != 1 sec") - } - - // Run the tests - // - // "Real" unit tests would verify the information written into the log - // - // t.Run("testRetry", testRetry) - // t.Run("testOps", testOps) - testOps(t) - testChunkedPut(t) - testReload(t, confMap) - - // Shutdown packages - - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("transitions.Down(confMap) failed: %v", err) - } - - // Send ourself a SIGTERM to terminate ramswift.Daemon() - - unix.Kill(unix.Getpid(), unix.SIGTERM) - - _ = <-doneChan -} - -// Test normal Swift client operations so we have something in the log -// -func testOps(t *testing.T) { - // headers for testing - - catDogHeaderMap := make(map[string][]string) - catDogHeaderMap["Cat"] = []string{"Dog"} - - mouseBirdHeaderMap := make(map[string][]string) - mouseBirdHeaderMap["Mouse"] = []string{"Bird"} - - mouseDeleteHeaderMap := make(map[string][]string) - mouseDeleteHeaderMap["Mouse"] = []string{""} - - // Send a PUT for account "TestAccount" and header Cat: Dog - - err := swiftclient.AccountPut("TestAccount", catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPut(\"TestAccount\", catDogHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog - - accountHeaders, err := swiftclient.AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok := accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - - // Send a GET for account "TestAccount" expecting header Cat: Dog and containerList []string{} - - accountHeaders, containerList, err := swiftclient.AccountGet("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountGet(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if 0 != len(containerList) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return expected containerList") - } - - // Send a POST for account "TestAccount" adding header Mouse: Bird - - err = swiftclient.AccountPost("TestAccount", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPost(\"TestAccount\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird - - accountHeaders, err = swiftclient.AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - mouseBirdHeader, ok := accountHeaders["Mouse"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Mouse\"") - } - if (1 != len(mouseBirdHeader)) || ("Bird" != mouseBirdHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Bird\" having value []string{\"Mouse\"}") - } - - // Send a POST for account "TestAccount" deleting header Mouse - - err = swiftclient.AccountPost("TestAccount", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPost(\"TestAccount\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header - - accountHeaders, err = swiftclient.AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - _, ok = accountHeaders["Mouse"] - if ok { - t.Fatalf("AccountHead(\"TestAccount\") shouldn't have returned Header \"Mouse\"") - } - - // Send a PUT for account "TestAccount" adding header Mouse: Bird - - err = swiftclient.AccountPut("TestAccount", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPut(\"TestAccount\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird - - accountHeaders, err = swiftclient.AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - mouseBirdHeader, ok = accountHeaders["Mouse"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Mouse\"") - } - if (1 != len(mouseBirdHeader)) || ("Bird" != mouseBirdHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Bird\" having value []string{\"Mouse\"}") - } - - // Send a PUT for account "TestAccount" deleting header Mouse - - err = swiftclient.AccountPut("TestAccount", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPut(\"TestAccount\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header - - accountHeaders, err = swiftclient.AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - _, ok = accountHeaders["Mouse"] - if ok { - t.Fatalf("AccountHead(\"TestAccount\") shouldn't have returned Header \"Mouse\"") - } - - // Send a PUT for container "TestContainer" and header Cat: Dog - - err = swiftclient.ContainerPut("TestAccount", "TestContainer", catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPut(\"TestAccount\", \"TestContainer\", catDogHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for account "TestAccount" expecting header Cat: Dog and containerList []string{"TestContainer"} - - accountHeaders, containerList, err = swiftclient.AccountGet("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountGet(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if (1 != len(containerList)) || ("TestContainer" != containerList[0]) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return expected containerList") - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog - - containerHeaders, err := swiftclient.ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok := containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{} - - containerHeaders, objectList, err := swiftclient.ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if 0 != len(objectList) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a POST for container "TestContainer" adding header Mouse: Bird - - err = swiftclient.ContainerPost("TestAccount", "TestContainer", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPost(\"TestAccount\", \"TestContainer\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird - - containerHeaders, err = swiftclient.ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - containerMouseHeader, ok := containerHeaders["Mouse"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Mouse\"") - } - if (1 != len(containerMouseHeader)) || ("Bird" != containerMouseHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Mouse\" having value []string{\"Bird\"}") - } - - // Send a POST for container "TestContainer" deleting header Mouse - - err = swiftclient.ContainerPost("TestAccount", "TestContainer", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPost(\"TestAccount\", \"TestContainer\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header - - containerHeaders, err = swiftclient.ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - _, ok = containerHeaders["Mouse"] - if ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") shouldn't have returned Header \"Mouse\"") - } - - // Send a PUT for container "TestContainer" adding header Mouse: Bird - - err = swiftclient.ContainerPut("TestAccount", "TestContainer", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPut(\"TestAccount\", \"TestContainer\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird - - containerHeaders, err = swiftclient.ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - containerMouseHeader, ok = containerHeaders["Mouse"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Mouse\"") - } - if (1 != len(containerMouseHeader)) || ("Bird" != containerMouseHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Mouse\" having value []string{\"Bird\"}") - } - - // Send a PUT for container "TestContainer" deleting header Mouse - - err = swiftclient.ContainerPut("TestAccount", "TestContainer", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPut(\"TestAccount\", \"TestContainer\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header - - containerHeaders, err = swiftclient.ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - _, ok = containerHeaders["Mouse"] - if ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") shouldn't have returned Header \"Mouse\"") - } - - // Start a chunked PUT for object "FooBar" - - chunkedPutContext, err := swiftclient.ObjectFetchChunkedPutContext("TestAccount", "TestContainer", "FooBar", "") - if nil != err { - tErr := fmt.Sprintf("ObjectFetchChunkedPutContext(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - - // Send a chunk for object "FooBar" of []byte{0xAA, 0xBB} - - err = chunkedPutContext.SendChunk([]byte{0xAA, 0xBB}) - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.SendChunk([]byte{0xAA, 0xBB}) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a chunk for object "FooBar" of []byte{0xCC, 0xDD, 0xEE} - - err = chunkedPutContext.SendChunk([]byte{0xCC, 0xDD, 0xEE}) - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.SendChunk([]byte{0xCC, 0xDD, 0xEE}) failed: %v", err) - t.Fatalf(tErr) - } - - // Fetch BytesPut for object "FooBar" expecting 5 - - bytesPut, err := chunkedPutContext.BytesPut() - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.BytesPut() failed: %v", err) - t.Fatalf(tErr) - } - if 5 != bytesPut { - t.Fatalf("chunkedPutContext.BytesPut() didn't return expected bytesPut") - } - - // Read back chunked PUT data at offset 1 for length 3 expecting []byte{0xBB, 0xCC, 0xDD} - - readBuf, err := chunkedPutContext.Read(uint64(1), uint64(3)) - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.Read(uint64(1), uint64(3)) failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, readBuf) { - t.Fatalf("chunkedPutContext.Read(uint64(1), uint64(3)) didn't return expected []byte") - } - - // Finish the chunked PUT for object "FooBar" - - err = chunkedPutContext.Close() - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.Close() failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{"FooBar"} - - containerHeaders, objectList, err = swiftclient.ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if (1 != len(objectList)) || ("FooBar" != objectList[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a HEAD for object "FooBar" expecting Content-Length: 5 - - objectHeaders, err := swiftclient.ObjectHead("TestAccount", "TestContainer", "FooBar") - if nil != err { - tErr := fmt.Sprintf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBar\") failed: %v", err) - t.Fatalf(tErr) - } - contentLengthHeader, ok := objectHeaders["Content-Length"] - if !ok { - t.Fatalf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBar\") didn't return Header \"Content-Length\"") - } - if (1 != len(contentLengthHeader)) || ("5" != contentLengthHeader[0]) { - t.Fatalf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBar\") didn't return Header \"Content-Length\" having value []string{\"5\"}") - } - - // Fetch Content-Length for object "FooBar" expecting 5 - - objectLength, err := swiftclient.ObjectContentLength("TestAccount", "TestContainer", "FooBar") - if nil != err { - tErr := fmt.Sprintf("ObjectContentLength(\"TestAccount\", \"TestContainer\", \"FooBar\") failed: %v", err) - t.Fatalf(tErr) - } - if uint64(5) != objectLength { - tErr := fmt.Sprintf("ObjectContentLength(\"TestAccount\", \"TestContainer\", \"FooBar\") didn't return expected Content-Length") - t.Fatalf(tErr) - } - - // Send a range GET of bytes at offset 1 for length 3 for object "FooBar" expecting []byte{0xBB, 0xCC, 0xDD} - - getBuf, err := swiftclient.ObjectGet("TestAccount", "TestContainer", "FooBar", uint64(1), uint64(3)) - if nil != err { - tErr := fmt.Sprintf("ObjectGet(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(1), uint64(3)) failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, getBuf) { - tErr := fmt.Sprintf("ObjectGet(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(1), uint64(3)) didn't return expected []byte") - t.Fatalf(tErr) - } - - // Send a full GET for object "FooBar" expecting []byte{0xAA, 0xBB, 0xCC, 0xDD, OxEE} - - loadBuf, err := swiftclient.ObjectLoad("TestAccount", "TestContainer", "FooBar") - if nil != err { - tErr := fmt.Sprintf("ObjectLoad(\"TestAccount\", \"TestContainer\", \"FooBar\") failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE}, loadBuf) { - tErr := fmt.Sprintf("ObjectLoad(\"TestAccount\", \"TestContainer\", \"FooBar\") didn't return expected []byte") - t.Fatalf(tErr) - } - - // Send a tail GET of the last 2 bytes for object "FooBar" expecting []byte{0xDD, 0xEE} - - tailBuf, err := swiftclient.ObjectTail("TestAccount", "TestContainer", "FooBar", uint64(2)) - if nil != err { - tErr := fmt.Sprintf("ObjectTail(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(2)) failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xDD, 0xEE}, tailBuf) { - tErr := fmt.Sprintf("ObjectTail(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(2)) didn't return expected []byte") - t.Fatalf(tErr) - } - - // Copy object "FooBar" to object "FooBarCopy" - - tOCCS := &testObjectCopyCallbackStruct{ - srcAccountName: "TestAccount", - srcContainerName: "TestContainer", - srcObjectName: "FooBar", - dstAccountName: "TestAccount", - dstContainerName: "TestContainer", - dstObjectName: "FooBarCopy", - chunkSize: 1, // Causes a callback for each of the five bytes of FooBar - } - - err = swiftclient.ObjectCopy(tOCCS.srcAccountName, tOCCS.srcContainerName, tOCCS.srcObjectName, tOCCS.dstAccountName, tOCCS.dstContainerName, tOCCS.dstObjectName, tOCCS) - if nil != err { - tErr := fmt.Sprintf("ObjectCopy(\"TestAccount/TestContainer/FooBar\" to \"TestAccount/TestContainer/FooBarCopy\") failed: %v", err) - t.Fatalf(tErr) - } - - // Send a full GET for object "FooBarCopy" expecting []byte{0xAA, 0xBB, 0xCC, 0xDD, OxEE} - - loadBuf, err = swiftclient.ObjectLoad("TestAccount", "TestContainer", "FooBarCopy") - if nil != err { - tErr := fmt.Sprintf("ObjectLoad(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE}, loadBuf) { - tErr := fmt.Sprintf("ObjectLoad(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") didn't return expected []byte") - t.Fatalf(tErr) - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{"FooBar", "FooBarCopy"} - - containerHeaders, objectList, err = swiftclient.ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if (2 != len(objectList)) || ("FooBar" != objectList[0]) || ("FooBarCopy" != objectList[1]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a DELETE for object "FooBar" - - err = swiftclient.ObjectDelete("TestAccount", "TestContainer", "FooBar", 0) - if nil != err { - tErr := fmt.Sprintf("ObjectDelete(\"TestAccount\", \"TestContainer\". \"FooBar\", 0) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{"FooBarCopy"} - - containerHeaders, objectList, err = swiftclient.ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if (1 != len(objectList)) || ("FooBarCopy" != objectList[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a DELETE for object "FooBarCopy" - - err = swiftclient.ObjectDelete("TestAccount", "TestContainer", "FooBarCopy", 0) - if nil != err { - tErr := fmt.Sprintf("ObjectDelete(\"TestAccount\", \"TestContainer\". \"FooBarCopy\", 0) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{} - - containerHeaders, objectList, err = swiftclient.ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if 0 != len(objectList) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a DELETE for container "TestContainer" - - err = swiftclient.ContainerDelete("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerDelete(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - - // create and delete container "TestContainer" again so we're sure the retry code is hit - - err = swiftclient.ContainerPut("TestAccount", "TestContainer", catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPut(\"TestAccount\", \"TestContainer\", catDogHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - err = swiftclient.ContainerDelete("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerDelete(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for account "TestAccount" expecting header Cat: Dog and containerList []string{} - - accountHeaders, containerList, err = swiftclient.AccountGet("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountGet(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if 0 != len(containerList) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return expected containerList") - } - - // Send a DELETE for account "TestAccount" - - err = swiftclient.AccountDelete("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountDelete(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - - // Create and delete "TestAccount" again so we're sure the retry code is hit - - err = swiftclient.AccountPut("TestAccount", catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPut(\"TestAccount\", catDogHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - err = swiftclient.AccountDelete("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountDelete(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } -} - -// Extended testing of chunked put interface to exercise internal retries -// -func testChunkedPut(t *testing.T) { - var ( - accountName = "TestAccount" - containerName = "TestContainer" - objNameFmt = "chunkObj%d" - objName string - ) - - // (lack of) headers for putting - catDogHeaderMap := make(map[string][]string) - - // (re)create the test account and continer - - err := swiftclient.AccountPut(accountName, catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("testChunkedPut.AccountPut('%s', catDogHeaderMap) failed: %v", accountName, err) - t.Fatalf(tErr) - } - - err = swiftclient.ContainerPut(accountName, containerName, catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("testChunkedPut.ContainerPut('%s', '%s', catDogHeaderMap) failed: %v", - accountName, containerName, err) - t.Fatalf(tErr) - } - - for i := 0; i < 5; i++ { - objName = fmt.Sprintf(objNameFmt, i) - - err = testObjectWriteVerify(t, accountName, containerName, objName, 4096, 4) - if nil != err { - tErr := fmt.Sprintf("testChunkedPut.testObjectWriteVerify('%s/%s/%s', %d, %d ) failed: %v", - accountName, containerName, objName, 4096, 4, err) - t.Fatalf(tErr) - } - } - - // cleanup the mess we made (objects, container, and account) - for i := 0; i < 5; i++ { - objName = fmt.Sprintf(objNameFmt, i) - - err = swiftclient.ObjectDelete(accountName, containerName, objName, 0) - if nil != err { - tErr := fmt.Sprintf("ObjectDelete('%s', '%s', '%s') failed: %v", - accountName, containerName, objName, err) - t.Fatalf(tErr) - } - } - - err = swiftclient.ContainerDelete(accountName, containerName) - if nil != err { - tErr := fmt.Sprintf("ContainerDelete('%s', '%s') failed: %v", accountName, containerName, err) - t.Fatalf(tErr) - } - - err = swiftclient.AccountDelete(accountName) - if nil != err { - tErr := fmt.Sprintf("AccountDelete('%s') failed: %v", accountName, err) - t.Fatalf(tErr) - } -} - -// write objSize worth of random bytes to the object using nWrite calls to -// SendChunk() and then read it back to verify. -// -func testObjectWriteVerify(t *testing.T, accountName string, containerName string, objName string, - objSize int, nwrite int) (err error) { - - writeBuf := make([]byte, objSize) - readBuf := make([]byte, 0) - - for i := 0; i < objSize; i++ { - writeBuf[i] = byte(rand.Uint32()) - } - if writeBuf[0] == 0 && writeBuf[1] == 0 && writeBuf[2] == 0 && writeBuf[3] == 0 { - tErr := "unix.GetRandom() is not very random" - t.Fatalf(tErr) - } - if writeBuf[objSize-1] == 0 && writeBuf[objSize-2] == 0 && writeBuf[objSize-3] == 0 && - writeBuf[objSize-4] == 0 { - tErr := "unix.GetRandom() is not very radnom at end of buffer" - t.Fatalf(tErr) - } - - // Start a chunked PUT for the object - chunkedPutContext, err := swiftclient.ObjectFetchChunkedPutContext(accountName, containerName, objName, "") - if nil != err { - tErr := fmt.Sprintf("ObjectFetchChunkedPutContext('%s', '%s', '%s') failed: %v", - accountName, containerName, objName, err) - return errors.New(tErr) - } - - wsz := len(writeBuf) / nwrite - for off := 0; off < len(writeBuf); off += wsz { - if off+wsz < objSize { - err = chunkedPutContext.SendChunk(writeBuf[off : off+wsz]) - } else { - err = chunkedPutContext.SendChunk(writeBuf[off:]) - } - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.SendChunk(writeBuf[%d:%d]) failed: %v", - off, off+wsz, err) - return errors.New(tErr) - } - } - err = chunkedPutContext.Close() - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.Close('%s/%s/%s') failed: %v", - accountName, containerName, objName, err) - return errors.New(tErr) - } - - // read and compare - readBuf, err = swiftclient.ObjectLoad(accountName, containerName, objName) - if nil != err { - tErr := fmt.Sprintf("ObjectLoad('%s/%s/%s') failed: %v", accountName, containerName, objName, err) - return errors.New(tErr) - } - if !bytes.Equal(readBuf, writeBuf) { - tErr := fmt.Sprintf("Object('%s/%s/%s') read back something different then written", - accountName, containerName, objName) - return errors.New(tErr) - } - - return nil -} - -// Make sure we can shutdown and re-enable the statsLogger -// -func testReload(t *testing.T, confMap conf.ConfMap) { - var err error - - // Reload statslogger with logging disabled - err = confMap.UpdateFromString("StatsLogger.Period=0s") - if nil != err { - tErr := fmt.Sprintf("UpdateFromString('StatsLogger.Period=0s') failed: %v", err) - t.Fatalf(tErr) - } - - err = transitions.Signaled(confMap) - if nil != err { - t.Fatalf("transitions.Signaled(confMap) failed: %v", err) - } - - // Enable logging again - err = confMap.UpdateFromString("StatsLogger.Period=1s") - if nil != err { - tErr := fmt.Sprintf("UpdateFromString('StatsLogger.Period=1s') failed: %v", err) - t.Fatalf(tErr) - } - - err = transitions.Signaled(confMap) - if nil != err { - t.Fatalf("transitions.Signaled(confMap) failed: %v", err) - } -} diff --git a/statslogger/simplestats.go b/statslogger/simplestats.go deleted file mode 100644 index 0ffef403..00000000 --- a/statslogger/simplestats.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package statslogger - -// Track statistics for the min, max, and mean number of free connections -// over a time interval (yes, this could be generalized) -// -// This is very simple, but still seems worth breaking out into a separate file. -// -type SimpleStats struct { - min int64 - max int64 - total int64 - samples int64 -} - -func (sp *SimpleStats) Clear() { - sp.min = 0 - sp.max = 0 - sp.total = 0 - sp.samples = 0 -} - -func (sp *SimpleStats) Sample(cnt int64) { - if sp.samples == 0 { - sp.min = cnt - } - if sp.min > cnt { - sp.min = cnt - } - if sp.max < cnt { - sp.max = cnt - } - sp.total += cnt - sp.samples++ -} - -func (sp *SimpleStats) Mean() int64 { - if sp.samples == 0 { - return 0 - } - return sp.total / sp.samples -} - -func (sp *SimpleStats) Min() int64 { - return sp.min -} - -func (sp *SimpleStats) Max() int64 { - return sp.max -} - -func (sp *SimpleStats) Samples() int64 { - return sp.samples -} - -func (sp *SimpleStats) Total() int64 { - return sp.total -} diff --git a/swiftclient/Makefile b/swiftclient/Makefile deleted file mode 100644 index d9255703..00000000 --- a/swiftclient/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/swiftclient - -include ../GoMakefile diff --git a/swiftclient/account.go b/swiftclient/account.go deleted file mode 100644 index 33bec5d9..00000000 --- a/swiftclient/account.go +++ /dev/null @@ -1,474 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Swift Account-specific API access implementation - -package swiftclient - -import ( - "fmt" - "net/url" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" -) - -func accountDeleteWithRetry(accountName string) (err error) { - // request is a function that, through the miracle of closure, calls - // accountDelete() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - request := func() (bool, error) { - var err error - err = accountDelete(accountName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - opname string = fmt.Sprintf("swiftclient.accountDelete(\"%v\")", accountName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftAccountDeleteRetryOps, - retrySuccessCnt: &stats.SwiftAccountDeleteRetrySuccessOps, - clientRequestTime: &globals.AccountDeleteUsec, - clientFailureCnt: &globals.AccountDeleteFailure, - swiftRequestTime: &globals.SwiftAccountDeleteUsec, - swiftRetryOps: &globals.SwiftAccountDeleteRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return err -} - -func accountDelete(accountName string) (err error) { - var ( - connection *connectionStruct - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "DELETE", "/"+swiftVersion+"/"+pathEscape(accountName), nil) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPDeleteError) - logger.WarnfWithError(err, "swiftclient.accountDelete(\"%v\") got writeHTTPRequestLineAndHeaders() error", accountName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPDeleteError) - logger.WarnfWithError(err, "swiftclient.accountDelete(\"%v\") got readHTTPStatusAndHeaders() error", accountName) - return - } - evtlog.Record(evtlog.FormatAccountDelete, accountName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "DELETE %s returned HTTP StatusCode %d Payload %s", accountName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.accountDelete(\"%v\") got readHTTPStatusAndHeaders() bad status", accountName) - return - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperations(&stats.SwiftAccountDeleteOps) - - return -} - -func accountGetWithRetry(accountName string) (headers map[string][]string, containerList []string, err error) { - // request is a function that, through the miracle of closure, calls - // accountGet() with the paramaters passed to this function, stashes the - // relevant return values into the local variables of this function, and - // then returns err and whether it is retriable to RequestWithRetry() - var ( - connection *connectionStruct - marker string - opname string - retryObj *RetryCtrl - statnm requestStatistics - toAddContainerList []string - toAddHeaders map[string][]string - ) - - retryObj = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - statnm = requestStatistics{ - retryCnt: &stats.SwiftAccountGetRetryOps, - retrySuccessCnt: &stats.SwiftAccountGetRetrySuccessOps, - clientRequestTime: &globals.AccountGetUsec, - clientFailureCnt: &globals.AccountGetFailure, - swiftRequestTime: &globals.SwiftAccountGetUsec, - swiftRetryOps: &globals.SwiftAccountGetRetryOps, - } - - request := func() (bool, error) { - var err error - toAddHeaders, toAddContainerList, err = accountGet(connection, accountName, marker) - return true, err - } - - headers = make(map[string][]string) - containerList = make([]string, 0) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - marker = "" - - for { - opname = fmt.Sprintf("swiftclient.accountGet(,\"%v\",\"%v\")", accountName, marker) - - err = retryObj.RequestWithRetry(request, &opname, &statnm) - - if nil == err { - mergeHeadersAndList(headers, &containerList, toAddHeaders, &toAddContainerList) - - if 0 == len(toAddContainerList) { - releaseNonChunkedConnection(connection, parseConnection(headers)) - - break - } else { - marker = toAddContainerList[len(toAddContainerList)-1] - } - } else { - releaseNonChunkedConnection(connection, false) - - break - } - } - - stats.IncrementOperations(&stats.SwiftAccountGetOps) - - return -} - -func accountGet(connection *connectionStruct, accountName string, marker string) (headers map[string][]string, containerList []string, err error) { - var ( - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - ) - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "GET", "/"+swiftVersion+"/"+pathEscape(accountName)+"?marker="+url.QueryEscape(marker), nil) - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.accountGet(,\"%v\",\"%v\") got writeHTTPRequestLineAndHeaders() error", accountName, marker) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.accountGet(,\"%v\",\"%v\") got readHTTPStatusAndHeaders() error", accountName, marker) - return - } - evtlog.Record(evtlog.FormatAccountGet, accountName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - err = blunder.NewError(fsErr, "GET %s returned HTTP StatusCode %d Payload %s", accountName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.accountGet(,\"%v\",\"%v\") got readHTTPStatusAndHeaders() bad status", accountName, marker) - return - } - - containerList, err = readHTTPPayloadLines(connection.tcpConn, headers) - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.accountGet(,\"%v\",\"%v\") got readHTTPPayloadLines() error", accountName, marker) - return - } - - return -} - -func accountHeadWithRetry(accountName string) (map[string][]string, error) { - // request is a function that, through the miracle of closure, calls - // accountHead() with the paramaters passed to this function, stashes the - // relevant return values into the local variables of this function, and - // then returns err and whether it is retriable to RequestWithRetry() - var ( - headers map[string][]string - ) - request := func() (bool, error) { - var err error - headers, err = accountHead(accountName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - opname string = fmt.Sprintf("swiftclient.accountHead(\"%v\")", accountName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftAccountHeadRetryOps, - retrySuccessCnt: &stats.SwiftAccountHeadRetrySuccessOps, - clientRequestTime: &globals.AccountHeadUsec, - clientFailureCnt: &globals.AccountHeadFailure, - swiftRequestTime: &globals.SwiftAccountHeadUsec, - swiftRetryOps: &globals.SwiftAccountHeadRetryOps, - } - ) - err := retryObj.RequestWithRetry(request, &opname, &statnm) - return headers, err -} - -func accountHead(accountName string) (headers map[string][]string, err error) { - var ( - connection *connectionStruct - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "HEAD", "/"+swiftVersion+"/"+pathEscape(accountName), nil) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.accountHead(\"%v\") got writeHTTPRequestLineAndHeaders() error", accountName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.accountHead(\"%v\") got readHTTPStatusAndHeaders() error", accountName) - return - } - evtlog.Record(evtlog.FormatAccountHead, accountName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "HEAD %s returned HTTP StatusCode %d Payload %s", accountName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.accountHead(\"%v\") got readHTTPStatusAndHeaders() bad status", accountName) - return - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperations(&stats.SwiftAccountHeadOps) - - return -} - -func accountPostWithRetry(accountName string, requestHeaders map[string][]string) (err error) { - // request is a function that, through the miracle of closure, calls - // accountPost() with the paramaters passed to this function, stashes the - // relevant return values into the local variables of this function, and - // then returns err and whether it is retriable to RequestWithRetry() - request := func() (bool, error) { - var err error - err = accountPost(accountName, requestHeaders) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - - opname string = fmt.Sprintf("swiftclient.accountPost(\"%v\")", accountName) - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftAccountPostRetryOps, - retrySuccessCnt: &stats.SwiftAccountPostRetrySuccessOps, - clientRequestTime: &globals.AccountPostUsec, - clientFailureCnt: &globals.AccountPostFailure, - swiftRequestTime: &globals.SwiftAccountPostUsec, - swiftRetryOps: &globals.SwiftAccountPostRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return err -} - -func accountPost(accountName string, requestHeaders map[string][]string) (err error) { - var ( - connection *connectionStruct - contentLength int - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - responseHeaders map[string][]string - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - requestHeaders["Content-Length"] = []string{"0"} - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "POST", "/"+swiftVersion+"/"+pathEscape(accountName), requestHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.accountPost(\"%v\") got writeHTTPRequestLineAndHeaders() error", accountName) - return - } - - httpStatus, responseHeaders, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.accountPost(\"%v\") got readHTTPStatusAndHeaders() error", accountName) - return - } - evtlog.Record(evtlog.FormatAccountPost, accountName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, responseHeaders) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "POST %s returned HTTP StatusCode %d Payload %s", accountName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.accountPost(\"%v\") got readHTTPStatusAndHeaders() bad status", accountName) - return - } - contentLength, err = parseContentLength(responseHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.accountPost(\"%v\") got parseContentLength() error", accountName) - return - } - if 0 < contentLength { - _, err = readBytesFromTCPConn(connection.tcpConn, contentLength) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.accountPost(\"%v\") got readBytesFromTCPConn() error", accountName) - return - } - } - - releaseNonChunkedConnection(connection, parseConnection(responseHeaders)) - - stats.IncrementOperations(&stats.SwiftAccountPostOps) - - return -} - -func accountPutWithRetry(accountName string, requestHeaders map[string][]string) (err error) { - // request is a function that, through the miracle of closure, calls - // accountPut() with the paramaters passed to this function, stashes the - // relevant return values into the local variables of this function, and - // then returns err and whether it is retriable to RequestWithRetry() - request := func() (bool, error) { - var err error - err = accountPut(accountName, requestHeaders) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - opname string = fmt.Sprintf("swiftclient.accountPut(\"%v\")", accountName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftAccountPutRetryOps, - retrySuccessCnt: &stats.SwiftAccountPutRetrySuccessOps, - clientRequestTime: &globals.AccountPutUsec, - clientFailureCnt: &globals.AccountPutFailure, - swiftRequestTime: &globals.SwiftAccountPutUsec, - swiftRetryOps: &globals.SwiftAccountPutRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return err -} - -func accountPut(accountName string, requestHeaders map[string][]string) (err error) { - var ( - connection *connectionStruct - contentLength int - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - responseHeaders map[string][]string - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - requestHeaders["Content-Length"] = []string{"0"} - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "PUT", "/"+swiftVersion+"/"+pathEscape(accountName), requestHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.accountPut(\"%v\") got writeHTTPRequestLineAndHeaders() error", accountName) - return - } - - httpStatus, responseHeaders, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.accountPut(\"%v\") got readHTTPStatusAndHeaders() error", accountName) - return - } - evtlog.Record(evtlog.FormatAccountPut, accountName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, responseHeaders) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "PUT %s returned HTTP StatusCode %d Payload %s", accountName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.accountPut(\"%v\") got readHTTPStatusAndHeaders() bad status", accountName) - return - } - contentLength, err = parseContentLength(responseHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.accountPut(\"%v\") got parseContentLength() error", accountName) - return - } - if 0 < contentLength { - _, err = readBytesFromTCPConn(connection.tcpConn, contentLength) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.accountPut(\"%v\") got readBytesFromTCPConn() error", accountName) - return - } - } - - releaseNonChunkedConnection(connection, parseConnection(responseHeaders)) - - stats.IncrementOperations(&stats.SwiftAccountPutOps) - - return -} diff --git a/swiftclient/api.go b/swiftclient/api.go deleted file mode 100644 index a376ea59..00000000 --- a/swiftclient/api.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package swiftclient provides API access to the local Swift NoAuth Pipeline. -package swiftclient - -type OperationOptions uint64 - -const ( - SkipRetry OperationOptions = 1 << iota -) - -// ChunkedCopyContext provides a callback context to use for Object Copies. -// -// A returned chunkSize == 0 says to stop copying -type ChunkedCopyContext interface { - BytesRemaining(bytesRemaining uint64) (chunkSize uint64) -} - -// ChunkedPutContext provides a context to use for Object HTTP PUTs using "chunked" Transfer-Encoding. -// -// The slice(s) passed to SendChunk() must not be modified until Close() has -// been called and has returned success or failure. Close() must always be -// called on a successfullly opened ChunkedPutContext, even if SendChunk() -// returns an error, or else we will leak open connections (although SendChunk() -// does its best). -type ChunkedPutContext interface { - Active() (active bool) // Report if currently active - BytesPut() (bytesPut uint64, err error) // Report how many bytes have been sent via SendChunk() for this ChunkedPutContext - Close() (err error) // Finish the "chunked" HTTP PUT for this ChunkedPutContext (with possible retry) - Read(offset uint64, length uint64) (buf []byte, err error) // Read back bytes previously sent via SendChunk() - SendChunk(buf []byte) (err error) // Send the supplied "chunk" via this ChunkedPutContext -} - -// StarvationParameters is used to report the current details pertaining to both -// the chunkedConnectionPool and nonChunkedConnectionPool. As this is a snapshot of -// a typically rapidly evolving state, it should only be use in well controlled test -// and debugging situations. -type StarvationParameters struct { - ChunkedConnectionPoolCapacity uint16 - ChunkedConnectionPoolInUse uint16 - ChunkedConnectionPoolNumWaiters uint64 - NonChunkedConnectionPoolCapacity uint16 - NonChunkedConnectionPoolInUse uint16 - NonChunkedConnectionPoolNumWaiters uint64 -} - -// GetStarvationParameters returns the the current details pertaining to both -// the chunkedConnectionPool and nonChunkedConnectionPool. -func GetStarvationParameters() (starvationParameters *StarvationParameters) { - starvationParameters = getStarvationParameters() - return -} - -// StarvationCallbackFunc specifies the signature of a callback function to be invoked when -// the Chunked PUT Connection Pool is exhausted and would like the callback function to -// relieve this exhaustion. -type StarvationCallbackFunc func() - -// SetStarvationCallbackFunc sets (or resets, if passed nil) the callback function to be -// invoked when the Chunked PUT Connection Pool is exhausted and would like the callback -// function to relieve this exhaustion. -func SetStarvationCallbackFunc(starvationCallback StarvationCallbackFunc) { - globals.starvationCallback = starvationCallback -} - -// AccountDelete invokes HTTP DELETE on the named Swift Account. -func AccountDelete(accountName string) (err error) { - return accountDeleteWithRetry(accountName) -} - -// AccountGet invokes HTTP GET on the named Swift Account. -func AccountGet(accountName string) (headers map[string][]string, containerList []string, err error) { - return accountGetWithRetry(accountName) -} - -// AccountHead invokes HTTP HEAD on the named Swift Account. -func AccountHead(accountName string) (headers map[string][]string, err error) { - return accountHeadWithRetry(accountName) -} - -// AccountPost invokes HTTP POST on the named Swift Account. -func AccountPost(accountName string, headers map[string][]string) (err error) { - return accountPostWithRetry(accountName, headers) -} - -// AccountPut invokes HTTP PUT on the named Swift Account. -func AccountPut(accountName string, headers map[string][]string) (err error) { - return accountPutWithRetry(accountName, headers) -} - -// ContainerDelete invokes HTTP DELETE on the named Swift Container. -func ContainerDelete(accountName string, containerName string) (err error) { - return containerDeleteWithRetry(accountName, containerName) -} - -// ContainerGet invokes HTTP GET on the named Swift Container. -func ContainerGet(accountName string, containerName string) (headers map[string][]string, objectList []string, err error) { - return containerGetWithRetry(accountName, containerName) -} - -// ContainerHead invokes HTTP HEAD on the named Swift Container. -func ContainerHead(accountName string, containerName string) (headers map[string][]string, err error) { - return containerHeadWithRetry(accountName, containerName) -} - -// ContainerPost invokes HTTP POST on the named Swift Container. -func ContainerPost(accountName string, containerName string, headers map[string][]string) (err error) { - return containerPostWithRetry(accountName, containerName, headers) -} - -// ContainerPut invokes HTTP PUT on the named Swift Container. -func ContainerPut(accountName string, containerName string, headers map[string][]string) (err error) { - return containerPutWithRetry(accountName, containerName, headers) -} - -// ObjectContentLength invokes HTTP HEAD on the named Swift Object and returns value of Content-Length Header. -func ObjectContentLength(accountName string, containerName string, objectName string) (length uint64, err error) { - return objectContentLengthWithRetry(accountName, containerName, objectName) -} - -// ObjectCopy asynchronously creates a copy of the named Swift Object Source called the named Swift Object Destination. -func ObjectCopy(srcAccountName string, srcContainerName string, srcObjectName string, dstAccountName string, dstContainerName string, dstObjectName string, chunkedCopyContext ChunkedCopyContext) (err error) { - return objectCopy(srcAccountName, srcContainerName, srcObjectName, dstAccountName, dstContainerName, dstObjectName, chunkedCopyContext) -} - -// ObjectDelete invokes HTTP DELETE on the named Swift Object. -func ObjectDelete(accountName string, containerName string, objectName string, operationOptions OperationOptions) (err error) { - return objectDelete(accountName, containerName, objectName, operationOptions) -} - -// ObjectFetchChunkedPutContext provisions a context to use for an HTTP PUT using "chunked" Transfer-Encoding on the named Swift Object. -// The useReserveForVolumeName string argument should be set to "" if normal chunked connection pool is to be used. -// If a reserved connection is to be used, note that there is only one per useReserveForVolumeName allowed at a time. -func ObjectFetchChunkedPutContext(accountName string, containerName string, objectName string, useReserveForVolumeName string) (chunkedPutContext ChunkedPutContext, err error) { - return objectFetchChunkedPutContextWithRetry(accountName, containerName, objectName, useReserveForVolumeName) -} - -// ObjectGet invokes HTTP GET on the named Swift Object for the specified byte range. -func ObjectGet(accountName string, containerName string, objectName string, offset uint64, length uint64) (buf []byte, err error) { - return objectGetWithRetry(accountName, containerName, objectName, offset, length) -} - -// ObjectHead invokes HTTP HEAD on the named Swift Object. -func ObjectHead(accountName string, containerName string, objectName string) (headers map[string][]string, err error) { - return objectHeadWithRetry(accountName, containerName, objectName) -} - -// ObjectLoad invokes HTTP GET on the named Swift Object for the entire object. -func ObjectLoad(accountName string, containerName string, objectName string) (buf []byte, err error) { - return objectLoadWithRetry(accountName, containerName, objectName) -} - -// ObjectPost invokes HTTP POST on the named Swift Object. -func ObjectPost(accountName string, containerName string, objectName string, headers map[string][]string) (err error) { - return objectPostWithRetry(accountName, containerName, objectName, headers) -} - -// ObjectRead invokes HTTP GET on the named Swift Object at the specified offset filling in the specified byte slice. -// Note that the byte slice must already have the desired length even though those bytes will be overwritten. -func ObjectRead(accountName string, containerName string, objectName string, offset uint64, buf []byte) (len uint64, err error) { - return objectReadWithRetry(accountName, containerName, objectName, offset, buf) -} - -// ObjectTail invokes HTTP GET on the named Swift Object with a byte range selecting the specified length of trailing bytes. -func ObjectTail(accountName string, containerName string, objectName string, length uint64) (buf []byte, err error) { - return objectTailWithRetry(accountName, containerName, objectName, length) -} - -// Number of chunked connections that are idle -func ChunkedConnectionFreeCnt() (freeChunkedConnections int64) { - return chunkedConnectionFreeCnt() -} - -// Number of non-chunked connections that are idle -func NonChunkedConnectionFreeCnt() (freeNonChunkedConnections int64) { - return nonChunkedConnectionFreeCnt() -} diff --git a/swiftclient/api_test.go b/swiftclient/api_test.go deleted file mode 100644 index 7ed025c0..00000000 --- a/swiftclient/api_test.go +++ /dev/null @@ -1,1732 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package swiftclient - -import ( - "bytes" - "errors" - "fmt" - "math/rand" - "os" - "regexp" - "runtime" - "sync" - "sync/atomic" - "testing" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/ramswift" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/transitions" -) - -type testObjectCopyCallbackStruct struct { - srcAccountName string - srcContainerName string - srcObjectName string - dstAccountName string - dstContainerName string - dstObjectName string - chunkSize uint64 -} - -func (tOCCS *testObjectCopyCallbackStruct) BytesRemaining(bytesRemaining uint64) (chunkSize uint64) { - chunkSize = tOCCS.chunkSize - return -} - -func TestAPI(t *testing.T) { - var ( - confMap conf.ConfMap - confStrings []string - doneChan chan bool - err error - signalHandlerIsArmedWG sync.WaitGroup - ) - - confStrings = []string{ - "TrackedLock.LockHoldTimeLimit=0s", - "TrackedLock.LockCheckPeriod=0s", - - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=9999", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=3", - "SwiftClient.RetryLimitObject=3", - "SwiftClient.RetryDelay=25ms", - "SwiftClient.RetryDelayObject=25ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - - // small pool sizes so test hangs if we leak connections - "SwiftClient.ChunkedConnectionPoolSize=1", - "SwiftClient.NonChunkedConnectionPoolSize=1", - - // checksum chunked put buffers - "SwiftClient.ChecksumChunkedPutChunks=true", - - "Cluster.WhoAmI=Peer0", - - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.EtcdEnabled=false", - - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - //"Logging.LogToConsole=true", - - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=1024", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - - "RamSwiftChaos.AccountDeleteFailureRate=2", - "RamSwiftChaos.AccountGetFailureRate=2", - "RamSwiftChaos.AccountHeadFailureRate=2", - "RamSwiftChaos.AccountPostFailureRate=2", - "RamSwiftChaos.AccountPutFailureRate=2", - - "RamSwiftChaos.ContainerDeleteFailureRate=2", - "RamSwiftChaos.ContainerGetFailureRate=2", - "RamSwiftChaos.ContainerHeadFailureRate=2", - "RamSwiftChaos.ContainerPostFailureRate=2", - "RamSwiftChaos.ContainerPutFailureRate=2", - - "RamSwiftChaos.ObjectDeleteFailureRate=3", - "RamSwiftChaos.ObjectGetFailureRate=2", - "RamSwiftChaos.ObjectHeadFailureRate=2", - "RamSwiftChaos.ObjectPostFailureRate=2", - "RamSwiftChaos.ObjectPutFailureRate=3", - - "RamSwiftChaos.FailureHTTPStatus=599", - } - - confMap, err = conf.MakeConfMapFromStrings(confStrings) - if err != nil { - t.Fatalf("%v", err) - } - - signalHandlerIsArmedWG.Add(1) - doneChan = make(chan bool, 1) // Must be buffered to avoid race - - go ramswift.Daemon("/dev/null", confStrings, &signalHandlerIsArmedWG, doneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - err = transitions.Up(confMap) - if nil != err { - t.Fatalf("transitions.Up(confMap) failed: %v", err) - } - - // additional error injection settings - atomic.StoreUint64(&globals.chaosSendChunkFailureRate, 7) - atomic.StoreUint64(&globals.chaosFetchChunkedPutFailureRate, 2) - - // Run the tests - // t.Run("testRetry", testRetry) - // t.Run("testOps", testOps) - testRetry(t) - testOps(t) - testChunkedPut(t) - testBadConnection(t) - - // Shutdown packages - - err = transitions.Down(confMap) - if nil != err { - t.Fatalf("logger.transitions() failed: %v", err) - } - - // Send ourself a SIGTERM to terminate ramswift.Daemon() - - unix.Kill(unix.Getpid(), unix.SIGTERM) - - _ = <-doneChan -} - -// Test the use of the API for normal Swift operations -// -func testOps(t *testing.T) { - - // headers for testing - - catDogHeaderMap := make(map[string][]string) - catDogHeaderMap["Cat"] = []string{"Dog"} - - mouseBirdHeaderMap := make(map[string][]string) - mouseBirdHeaderMap["Mouse"] = []string{"Bird"} - - mouseDeleteHeaderMap := make(map[string][]string) - mouseDeleteHeaderMap["Mouse"] = []string{""} - - // Send a PUT for account "TestAccount" and header Cat: Dog - - err := AccountPut("TestAccount", catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPut(\"TestAccount\", catDogHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog - - accountHeaders, err := AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok := accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - - // Send a GET for account "TestAccount" expecting header Cat: Dog and containerList []string{} - - accountHeaders, containerList, err := AccountGet("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountGet(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if 0 != len(containerList) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return expected containerList") - } - - // Send a POST for account "TestAccount" adding header Mouse: Bird - - err = AccountPost("TestAccount", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPost(\"TestAccount\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird - - accountHeaders, err = AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - mouseBirdHeader, ok := accountHeaders["Mouse"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Mouse\"") - } - if (1 != len(mouseBirdHeader)) || ("Bird" != mouseBirdHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Bird\" having value []string{\"Mouse\"}") - } - - // Send a POST for account "TestAccount" deleting header Mouse - - err = AccountPost("TestAccount", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPost(\"TestAccount\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header - - accountHeaders, err = AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - _, ok = accountHeaders["Mouse"] - if ok { - t.Fatalf("AccountHead(\"TestAccount\") shouldn't have returned Header \"Mouse\"") - } - - // Send a PUT for account "TestAccount" adding header Mouse: Bird - - err = AccountPut("TestAccount", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPut(\"TestAccount\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird - - accountHeaders, err = AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - mouseBirdHeader, ok = accountHeaders["Mouse"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Mouse\"") - } - if (1 != len(mouseBirdHeader)) || ("Bird" != mouseBirdHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Bird\" having value []string{\"Mouse\"}") - } - - // Send a PUT for account "TestAccount" deleting header Mouse - - err = AccountPut("TestAccount", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPut(\"TestAccount\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header - - accountHeaders, err = AccountHead("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountHead(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountHead(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - _, ok = accountHeaders["Mouse"] - if ok { - t.Fatalf("AccountHead(\"TestAccount\") shouldn't have returned Header \"Mouse\"") - } - - // Send a PUT for container "TestContainer" and header Cat: Dog - - err = ContainerPut("TestAccount", "TestContainer", catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPut(\"TestAccount\", \"TestContainer\", catDogHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a PUT for container "Z" - - err = ContainerPut("TestAccount", "Z", map[string][]string{}) - if nil != err { - t.Fatalf("ContainerPut(\"TestAccount\", \"Z\", nil) failed: %v", err) - } - - // Send a GET for account "TestAccount" expecting header Cat: Dog and containerList []string{"TestContainer", "Z"} - - accountHeaders, containerList, err = AccountGet("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountGet(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if (2 != len(containerList)) || ("TestContainer" != containerList[0]) || ("Z" != containerList[1]) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return expected containerList") - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog - - containerHeaders, err := ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok := containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{} - - containerHeaders, objectList, err := ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if 0 != len(objectList) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a POST for container "TestContainer" adding header Mouse: Bird - - err = ContainerPost("TestAccount", "TestContainer", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPost(\"TestAccount\", \"TestContainer\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird - - containerHeaders, err = ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - containerMouseHeader, ok := containerHeaders["Mouse"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Mouse\"") - } - if (1 != len(containerMouseHeader)) || ("Bird" != containerMouseHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Mouse\" having value []string{\"Bird\"}") - } - - // Send a POST for container "TestContainer" deleting header Mouse - - err = ContainerPost("TestAccount", "TestContainer", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPost(\"TestAccount\", \"TestContainer\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header - - containerHeaders, err = ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - _, ok = containerHeaders["Mouse"] - if ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") shouldn't have returned Header \"Mouse\"") - } - - // Send a PUT for container "TestContainer" adding header Mouse: Bird - - err = ContainerPut("TestAccount", "TestContainer", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPut(\"TestAccount\", \"TestContainer\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird - - containerHeaders, err = ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - containerMouseHeader, ok = containerHeaders["Mouse"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Mouse\"") - } - if (1 != len(containerMouseHeader)) || ("Bird" != containerMouseHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Mouse\" having value []string{\"Bird\"}") - } - - // Send a PUT for container "TestContainer" deleting header Mouse - - err = ContainerPut("TestAccount", "TestContainer", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPut(\"TestAccount\", \"TestContainer\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header - - containerHeaders, err = ContainerHead("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerHead(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - _, ok = containerHeaders["Mouse"] - if ok { - t.Fatalf("ContainerHead(\"TestAccount\", \"TestContainer\") shouldn't have returned Header \"Mouse\"") - } - - // Start a chunked PUT for object "FooBar" - - chunkedPutContext, err := ObjectFetchChunkedPutContext("TestAccount", "TestContainer", "FooBar", "") - if nil != err { - tErr := fmt.Sprintf("ObjectFetchChunkedPutContext(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - - // Send a chunk for object "FooBar" of []byte{0xAA, 0xBB} - - err = chunkedPutContext.SendChunk([]byte{0xAA, 0xBB}) - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.SendChunk([]byte{0xAA, 0xBB}) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a chunk for object "FooBar" of []byte{0xCC, 0xDD, 0xEE} - - err = chunkedPutContext.SendChunk([]byte{0xCC, 0xDD, 0xEE}) - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.SendChunk([]byte{0xCC, 0xDD, 0xEE}) failed: %v", err) - t.Fatalf(tErr) - } - - // Fetch BytesPut for object "FooBar" expecting 5 - - bytesPut, err := chunkedPutContext.BytesPut() - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.BytesPut() failed: %v", err) - t.Fatalf(tErr) - } - if 5 != bytesPut { - t.Fatalf("chunkedPutContext.BytesPut() didn't return expected bytesPut") - } - - // Read back chunked PUT data at offset 1 for length 3 expecting []byte{0xBB, 0xCC, 0xDD} - - readBuf, err := chunkedPutContext.Read(uint64(1), uint64(3)) - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.Read(uint64(1), uint64(3)) failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, readBuf) { - t.Fatalf("chunkedPutContext.Read(uint64(1), uint64(3)) didn't return expected []byte") - } - - // Finish the chunked PUT for object "FooBar" - - err = chunkedPutContext.Close() - if nil != err { - tErr := fmt.Sprintf("chunkedPutContext.Close() failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{"FooBar"} - - containerHeaders, objectList, err = ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if (1 != len(objectList)) || ("FooBar" != objectList[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a HEAD for object "FooBar" expecting Content-Length: 5 - - objectHeaders, err := ObjectHead("TestAccount", "TestContainer", "FooBar") - if nil != err { - tErr := fmt.Sprintf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBar\") failed: %v", err) - t.Fatalf(tErr) - } - contentLengthHeader, ok := objectHeaders["Content-Length"] - if !ok { - t.Fatalf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBar\") didn't return Header \"Content-Length\"") - } - if (1 != len(contentLengthHeader)) || ("5" != contentLengthHeader[0]) { - t.Fatalf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBar\") didn't return Header \"Content-Length\" having value []string{\"5\"}") - } - - // Fetch Content-Length for object "FooBar" expecting 5 - - objectLength, err := ObjectContentLength("TestAccount", "TestContainer", "FooBar") - if nil != err { - tErr := fmt.Sprintf("ObjectContentLength(\"TestAccount\", \"TestContainer\", \"FooBar\") failed: %v", err) - t.Fatalf(tErr) - } - if uint64(5) != objectLength { - tErr := fmt.Sprintf("ObjectContentLength(\"TestAccount\", \"TestContainer\", \"FooBar\") didn't return expected Content-Length") - t.Fatalf(tErr) - } - - // Send a range GET of bytes at offset 1 for length 3 for object "FooBar" expecting []byte{0xBB, 0xCC, 0xDD} - - getBuf, err := ObjectGet("TestAccount", "TestContainer", "FooBar", uint64(1), uint64(3)) - if nil != err { - tErr := fmt.Sprintf("ObjectGet(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(1), uint64(3)) failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, getBuf) { - tErr := fmt.Sprintf("ObjectGet(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(1), uint64(3)) didn't return expected []byte") - t.Fatalf(tErr) - } - - // Send a range READ of bytes at offset 1 for length 3 for object "FooBar" expecting []byte{0xBB, 0xCC, 0xDD} - - readLen, err := ObjectRead("TestAccount", "TestContainer", "FooBar", uint64(1), readBuf) - if nil != err { - tErr := fmt.Sprintf("ObjectRead(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(1), readBuf) failed: %v", err) - t.Fatalf(tErr) - } - if 3 != readLen { - tErr := fmt.Sprintf("ObjectRead(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(1), readBuf) didn't return expected len") - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, readBuf) { - tErr := fmt.Sprintf("ObjectRead(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(1), readBuf) didn't return expected []byte") - t.Fatalf(tErr) - } - - // Send a full GET for object "FooBar" expecting []byte{0xAA, 0xBB, 0xCC, 0xDD, OxEE} - - loadBuf, err := ObjectLoad("TestAccount", "TestContainer", "FooBar") - if nil != err { - tErr := fmt.Sprintf("ObjectLoad(\"TestAccount\", \"TestContainer\", \"FooBar\") failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE}, loadBuf) { - tErr := fmt.Sprintf("ObjectLoad(\"TestAccount\", \"TestContainer\", \"FooBar\") didn't return expected []byte") - t.Fatalf(tErr) - } - - // Send a tail GET of the last 2 bytes for object "FooBar" expecting []byte{0xDD, 0xEE} - - tailBuf, err := ObjectTail("TestAccount", "TestContainer", "FooBar", uint64(2)) - if nil != err { - tErr := fmt.Sprintf("ObjectTail(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(2)) failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xDD, 0xEE}, tailBuf) { - tErr := fmt.Sprintf("ObjectTail(\"TestAccount\", \"TestContainer\", \"FooBar\", uint64(2)) didn't return expected []byte") - t.Fatalf(tErr) - } - - // Copy object "FooBar" to object "FooBarCopy" - - tOCCS := &testObjectCopyCallbackStruct{ - srcAccountName: "TestAccount", - srcContainerName: "TestContainer", - srcObjectName: "FooBar", - dstAccountName: "TestAccount", - dstContainerName: "TestContainer", - dstObjectName: "FooBarCopy", - chunkSize: 1, // Causes a callback for each of the five bytes of FooBar - } - - err = ObjectCopy(tOCCS.srcAccountName, tOCCS.srcContainerName, tOCCS.srcObjectName, tOCCS.dstAccountName, tOCCS.dstContainerName, tOCCS.dstObjectName, tOCCS) - if nil != err { - tErr := fmt.Sprintf("ObjectCopy(\"TestAccount/TestContainer/FooBar\" to \"TestAccount/TestContainer/FooBarCopy\") failed: %v", err) - t.Fatalf(tErr) - } - - // Send a full GET for object "FooBarCopy" expecting []byte{0xAA, 0xBB, 0xCC, 0xDD, OxEE} - - loadBuf, err = ObjectLoad("TestAccount", "TestContainer", "FooBarCopy") - if nil != err { - tErr := fmt.Sprintf("ObjectLoad(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") failed: %v", err) - t.Fatalf(tErr) - } - if 0 != bytes.Compare([]byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE}, loadBuf) { - tErr := fmt.Sprintf("ObjectLoad(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") didn't return expected []byte") - t.Fatalf(tErr) - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{"FooBar", "FooBarCopy"} - - containerHeaders, objectList, err = ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if (2 != len(objectList)) || ("FooBar" != objectList[0]) || ("FooBarCopy" != objectList[1]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a DELETE for object "FooBar" - - err = ObjectDelete("TestAccount", "TestContainer", "FooBar", 0) - if nil != err { - tErr := fmt.Sprintf("ObjectDelete(\"TestAccount\", \"TestContainer\". \"FooBar\", 0) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{"FooBarCopy"} - - containerHeaders, objectList, err = ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if (1 != len(objectList)) || ("FooBarCopy" != objectList[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a POST for object "FooBarCopy" adding header Mouse: Bird - - err = ObjectPost("TestAccount", "TestContainer", "FooBarCopy", mouseBirdHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ObjectPost(\"TestAccount\", \"TestContainer\", \"FooBarCopy\", mouseBirdHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for object "FooBarCopy" expecting header Mouse: Bird - - objectHeaders, err = ObjectHead("TestAccount", "TestContainer", "FooBarCopy") - if nil != err { - tErr := fmt.Sprintf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") failed: %v", err) - t.Fatalf(tErr) - } - objectMouseHeader, ok := objectHeaders["Mouse"] - if !ok { - t.Fatalf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") didn't return Header \"Mouse\"") - } - if (1 != len(objectMouseHeader)) || ("Bird" != objectMouseHeader[0]) { - t.Fatalf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") didn't return Header \"Mouse\" having value []string{\"Bird\"}") - } - - // Send a POST for object "FooBarCopy" deleting header Mouse - - err = ObjectPost("TestAccount", "TestContainer", "FooBarCopy", mouseDeleteHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ObjectPost(\"TestAccount\", \"TestContainer\", \"FooBarCopy\", mouseDeleteHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a HEAD for object "FooBarCopy" expecting no Mouse Header - - objectHeaders, err = ObjectHead("TestAccount", "TestContainer", "FooBarCopy") - if nil != err { - tErr := fmt.Sprintf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") failed: %v", err) - t.Fatalf(tErr) - } - _, ok = objectHeaders["Mouse"] - if ok { - t.Fatalf("ObjectHead(\"TestAccount\", \"TestContainer\", \"FooBarCopy\") shouldn't have returned Header \"Mouse\"") - } - - // Send a DELETE for object "FooBarCopy" - - err = ObjectDelete("TestAccount", "TestContainer", "FooBarCopy", 0) - if nil != err { - tErr := fmt.Sprintf("ObjectDelete(\"TestAccount\", \"TestContainer\". \"FooBarCopy\", 0) failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for container "TestContainer" expecting header Cat: Dog and objectList []string{} - - containerHeaders, objectList, err = ContainerGet("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerGet(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - containerCatHeader, ok = containerHeaders["Cat"] - if !ok { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\"") - } - if (1 != len(containerCatHeader)) || ("Dog" != containerCatHeader[0]) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if 0 != len(objectList) { - t.Fatalf("ContainerGet(\"TestAccount\", \"TestContainer\") didn't return expected objectList") - } - - // Send a DELETE for container "TestContainer" - - err = ContainerDelete("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerDelete(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - - // Send a DELETE for container "Z" - - err = ContainerDelete("TestAccount", "Z") - if nil != err { - t.Fatalf("ContainerDelete(\"TestAccount\", \"Z\") failed: %v", err) - } - - // create and delete container "TestContainer" again so we're sure the retry code is hit - - err = ContainerPut("TestAccount", "TestContainer", catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("ContainerPut(\"TestAccount\", \"TestContainer\", catDogHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - err = ContainerDelete("TestAccount", "TestContainer") - if nil != err { - tErr := fmt.Sprintf("ContainerDelete(\"TestAccount\", \"TestContainer\") failed: %v", err) - t.Fatalf(tErr) - } - - // Send a GET for account "TestAccount" expecting header Cat: Dog and containerList []string{} - - accountHeaders, containerList, err = AccountGet("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountGet(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - accountCatHeader, ok = accountHeaders["Cat"] - if !ok { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\"") - } - if (1 != len(accountCatHeader)) || ("Dog" != accountCatHeader[0]) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return Header \"Cat\" having value []string{\"Dog\"}") - } - if 0 != len(containerList) { - t.Fatalf("AccountGet(\"TestAccount\") didn't return expected containerList") - } - - // Send a DELETE for account "TestAccount" - - err = AccountDelete("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountDelete(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - - // Create and delete "TestAccount" again so we're sure the retry code is hit - - err = AccountPut("TestAccount", catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("AccountPut(\"TestAccount\", catDogHeaderMap) failed: %v", err) - t.Fatalf(tErr) - } - err = AccountDelete("TestAccount") - if nil != err { - tErr := fmt.Sprintf("AccountDelete(\"TestAccount\") failed: %v", err) - t.Fatalf(tErr) - } - -} - -// Extended testing of chunked put interface to exercise internal retries -// -func testChunkedPut(t *testing.T) { - - // preserve the original settings of these globals that we change - var ( - chaosFetchChunkedPutFailureRate = atomic.LoadUint64(&globals.chaosFetchChunkedPutFailureRate) - chaosSendChunkFailureRate = atomic.LoadUint64(&globals.chaosSendChunkFailureRate) - chaosCloseChunkFailureRate = atomic.LoadUint64(&globals.chaosCloseChunkFailureRate) - retryLimitObject = globals.retryLimitObject - chunkedConnectionPoolSize = globals.chunkedConnectionPool.poolCapacity - - cleanup = func() { - atomic.StoreUint64(&globals.chaosFetchChunkedPutFailureRate, chaosFetchChunkedPutFailureRate) - atomic.StoreUint64(&globals.chaosSendChunkFailureRate, chaosSendChunkFailureRate) - atomic.StoreUint64(&globals.chaosCloseChunkFailureRate, chaosCloseChunkFailureRate) - globals.retryLimitObject = retryLimitObject - resizeChunkedConnectionPool(uint(chunkedConnectionPoolSize)) - } - ) - defer cleanup() - - var ( - accountName = "TestAccount" - containerName = "TestContainer" - objNameFmt = "chunkObj%d" - objName string - ) - - // increase the pool sizes to get some concurrency (non-chunked isn't tested here) - resizeChunkedConnectionPool(12) - - // (lack of) headers for putting - catDogHeaderMap := make(map[string][]string) - - // (re)create the test account and container - - err := AccountPut(accountName, catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("testChunkedPut.AccountPut('%s', catDogHeaderMap) failed: %v", accountName, err) - t.Fatalf(tErr) - } - - err = ContainerPut(accountName, containerName, catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("testChunkedPut.ContainerPut('%s', '%s', catDogHeaderMap) failed: %v", - accountName, containerName, err) - t.Fatalf(tErr) - } - - // Create an object and perform a sequence of randomly sized writes that - // total upto 11 Mibyte (maximum). Inject errors every 23rd - // SendChunk(), every 3rd FetchContext(), and every 5th Close(). - // Because ramwswift is also injecting an error every 3rd chunked put - // increase globals.retryLimitObject for this test because a series of - // failures at different levels pushes us over the base limit. - // - // Since the size of the writes is random it could require more than 23 - // writes to reach 11 Mibyte, it which case the test would fail. - // However, the sequence of "random" numbers from randGen is the same - // every time so this test always passes (unless the seed is changed). - atomic.StoreUint64(&globals.chaosFetchChunkedPutFailureRate, 3) - atomic.StoreUint64(&globals.chaosSendChunkFailureRate, 20) - atomic.StoreUint64(&globals.chaosCloseChunkFailureRate, 5) - globals.retryLimitObject = 6 - - randGen := rand.New(rand.NewSource(2)) - doneChan := make(chan error, 1) - for i := 0; i < 6; i++ { - objName = fmt.Sprintf(objNameFmt, i) - objSize := ((i % 11) + 1) * 1024 * 1024 - - // start an ObjectWriteVerify and wait for it to complete - testObjectWriteVerifyThreaded(t, accountName, containerName, objName, objSize, - randGen, doneChan) - err = <-doneChan - if err != nil { - fmt.Printf("testChunkedPut: FATAL ERROR: %v\n", err) - t.Error(err) - } - } - - // Now do the same thing with concurrent threads. Because the threads - // are running non-deterministically there is no real bound on the - // number of simulated errors a thread might see, so crank up the - // failure rates (less failures) and increase allowed retries - // significantly. Even so, its possible that a spurious error might - // occur (in which case the failure rates could be made even higher). - atomic.StoreUint64(&globals.chaosSendChunkFailureRate, atomic.LoadUint64(&globals.chaosSendChunkFailureRate)*3) - atomic.StoreUint64(&globals.chaosCloseChunkFailureRate, 11) - globals.retryLimitObject = 10 - - maxThread := 6 - doneChan = make(chan error, maxThread) - threads := 0 - for i := 0; i < 18; i++ { - objName = fmt.Sprintf(objNameFmt, i) - randGen = rand.New(rand.NewSource(int64(i) + 3)) - - objSize := randGen.Intn(12 * 1024 * 1024) - - // fire off a go routine to do the work and count it - threads++ - go testObjectWriteVerifyThreaded(t, accountName, containerName, objName, objSize, - randGen, doneChan) - - // if we've hit the maximum number of concurrent threads - // (requests) wait for 1 to finish - for threads >= maxThread { - - // trigger garbage collection - runtime.GC() - - err = <-doneChan - threads-- - if err != nil { - t.Error(err) - } - } - } - for threads > 0 { - err = <-doneChan - threads-- - if err != nil { - t.Error(err) - } - } - - // cleanup the mess we made (objects, container, and account) - err = ContainerDelete(accountName, containerName) - if nil != err { - tErr := fmt.Sprintf("ContainerDelete('%s', '%s') failed: %v", accountName, containerName, err) - t.Error(tErr) - } - - err = AccountDelete(accountName) - if nil != err { - tErr := fmt.Sprintf("AccountDelete('%s') failed: %v", accountName, err) - t.Error(tErr) - } -} - -// Only the "top" testing goroutine can call t.Fatal() or t.Error() and be -// heard; others are ignored. Since this is called by a child, it can't call -// one of those routines and all errors must be returned for handling. -// -func testObjectWriteVerifyThreaded(t *testing.T, accountName string, containerName string, objName string, - objSize int, randGen *rand.Rand, doneChan chan error) { - - // write the object and delete it (even if the verify failed) - verifyErr := testObjectWriteVerify(t, accountName, containerName, objName, objSize, randGen) - deleteErr := ObjectDelete(accountName, containerName, objName, 0) - - // if the write failed report it - if verifyErr != nil { - tErr := fmt.Errorf("testChunkedPut.testObjectWriteVerify('%s/%s/%s', %d) failed: %v", - accountName, containerName, objName, objSize, verifyErr) - doneChan <- tErr - return - } - - // if the write succeeded but the verify failed, report it - if deleteErr != nil { - tErr := fmt.Errorf("ObjectDelete('%s/%s/%s', %d) failed: %v", - accountName, containerName, objName, objSize, deleteErr) - doneChan <- tErr - return - } - - doneChan <- nil - return -} - -// write objSize worth of random bytes to the object using nWrite calls to -// SendChunk() and then read it back to verify. -// -// Only the "top" testing goroutine can call t.Fatal() or t.Error() and be -// heard; others are ignored. Since this is called by a child, it can't call -// one of those routines and all errors must be returned for handling. -// -func testObjectWriteVerify(t *testing.T, accountName string, containerName string, objName string, - objSize int, randGen *rand.Rand) (err error) { - - writeBuf := make([]byte, objSize) - readBuf := make([]byte, 0) - - for i := 0; i < objSize; i++ { - writeBuf[i] = byte(randGen.Uint32()) - } - if writeBuf[0] == 0 && writeBuf[1] == 0 && writeBuf[2] == 0 && writeBuf[3] == 0 { - err = fmt.Errorf("testObjectWriteVerify(): randGen is not very random") - return - } - if writeBuf[objSize-1] == 0 && writeBuf[objSize-2] == 0 && writeBuf[objSize-3] == 0 && - writeBuf[objSize-4] == 0 { - err = fmt.Errorf("testObjectWriteVerify(): randGen is not very random at end of buffer") - return - } - - // Start a chunked PUT for the object - chunkedPutContext, err := ObjectFetchChunkedPutContext(accountName, containerName, objName, "") - if nil != err { - err = fmt.Errorf("testObjectWriteVerify(): ObjectFetchChunkedPutContext('%s/%s/%s') failed: %v", - accountName, containerName, objName, err) - return - } - - var ( - off int - sz int - nio int - ) - - for off = 0; off < objSize; off += sz { - // write a random number of bytes, but at least 32 - sz = randGen.Intn(objSize-off) + 1 - if sz < 32 { - sz = 32 - if off+sz > objSize { - sz = objSize - off - } - } - - err = chunkedPutContext.SendChunk(writeBuf[off : off+sz]) - if nil != err { - tErr := fmt.Errorf("testObjectWriteVerify('%s/%s/%s'): "+ - "chunkedPutContext.SendChunk(writeBuf[%d:%d]) failed: %v", - accountName, containerName, objName, off, off+sz, err) - return tErr - } - nio++ - - // every 16 i/o requests, or so, wait upto 30 sec - // if randGen.Intn(16) == 0 { - // waitSec := randGen.Intn(31) - // fmt.Printf("object %10s sleeping %d sec after send\n", objName, waitSec) - // time.Sleep(time.Duration(waitSec) * time.Second) - // } - } - - // every 8 i/o requests, or so, wait upto 60 sec before Close() - // if randGen.Intn(4) == 0 { - // waitSec := randGen.Intn(61) - // fmt.Printf("object %10s sleeping %d sec before close\n", objName, waitSec) - // time.Sleep(time.Duration(waitSec) * time.Second) - // } - - // verify bytes using Read() at 16 random ranges - for nio = 0; nio < 16; nio++ { - off = randGen.Intn(objSize) - sz = randGen.Intn(objSize-off) + 1 - - readBuf, err = chunkedPutContext.Read(uint64(off), uint64(sz)) - if err != nil { - tErr := fmt.Errorf("testObjectWriteVerify('%s/%s/%s'): "+ - "chunkedPutContext.Read(%d, %d) failed: %v", - accountName, containerName, objName, off, sz, err) - // panic because this should *never* happen - panic(tErr) - // return tErr - } - - if !bytes.Equal(readBuf, writeBuf[off:off+sz]) { - tErr := fmt.Errorf("testObjectWriteVerify('%s/%s/%s'): "+ - "chunkedPutContext.Read(%d, %d) dat does not match SendChunk()", - accountName, containerName, objName, off, sz) - // panic because this should *never* happen - panic(tErr) - // return tErr - } - } - - err = chunkedPutContext.Close() - if nil != err { - tErr := fmt.Errorf("testObjectWriteVerify('%s/%s/%s'): chunkedPutContext.Close() failed: %v", - accountName, containerName, objName, err) - return tErr - } - - // read and compare - readBuf, err = ObjectLoad(accountName, containerName, objName) - if nil != err { - tErr := fmt.Errorf("testObjectWriteVerify('%s/%s/%s'): ObjectLoad() failed: %v", - accountName, containerName, objName, err) - return tErr - } - if !bytes.Equal(readBuf, writeBuf) { - tErr := fmt.Errorf("testObjectWriteVerify('%s/%s/%s'): "+ - "Object() read back something different then written", - accountName, containerName, objName) - // panic because this should *never* happen - panic(tErr) - // return tErr - } - - return nil -} - -// Parse a log entry generated by testRetry and return the important values as -// strings, where the definition of what's important is somewhat arbitrary. -// -// This makes use of ParseLogEntry() to parse the "msg" from the log entry and -// then breaks it down further. -// -// matching log entries look like: -// -// time="2017-07-27T01:30:46.060080Z" level=info msg="retry.RequestWithRetry(): swiftclient.testRetry.request(1) succeeded after 4 attempts in 0.031 sec" function=RequestWithRetry goroutine=6 package=swiftclient -// -// time="2017-07-27T02:18:19.214012Z" level=error msg="retry.RequestWithRetry(): swiftclient.testRetry.request(1) failed after 7 attempts in 0.053 sec with retriable error" error="Simulate a retriable errror" function=RequestWithRetry goroutine=6 package=swiftclient -// -// time="2017-07-27T02:09:32.259383Z" level=error msg="retry.RequestWithRetry(): swiftclient.testRetry.request(1) failed after 6 attempts in 0.054 sec with unretriable error" error="Simulate an unretriable error" function=RequestWithRetry goroutine=20 package=swiftclient -// -func parseRetryLogEntry(entry string) map[string]string { - var ( - fields = make(map[string]string) - matches []string - err error - ) - - fields, err = logger.ParseLogEntry(entry) - if err != nil { - fmt.Fprintf(os.Stderr, "log entry not matched: '%s'\n", entry) - } - - var msgRE = regexp.MustCompile( - `^retry.RequestWithRetry\(\): swiftclient.testRetry.request\((\d+)\) (succeeded|failed) after (\d+) attempts in (\d+\.\d*) sec( with (retriable|unretriable) error)?$`) - - matches = msgRE.FindStringSubmatch(fields["msg"]) - if matches == nil { - return nil - } - fields["reqno"] = matches[1] - fields["result"] = matches[2] - fields["attempts"] = matches[3] - fields["seconds"] = matches[4] - fields["retriable"] = matches[6] - - return fields -} - -// Test the retry handler -func testRetry(t *testing.T) { - - // a new log target to capture messages written to the log - var logcopy logger.LogTarget - logcopy.Init(50) - - // logcopy will live on long after this function returns - logger.AddLogTarget(logcopy) - - // request is a function that, through the miracle of closure, uses and - // updates these variables so we can use them to affect its behavior - // when its called by by RequestWithRetry() - var ( - callCnt int = 0 - successOn int = 2 - unretriableOn int = 2 - ) - request := func() (bool, error) { - callCnt++ - if successOn == callCnt { - return false, nil - } - if unretriableOn == callCnt { - return false, errors.New("Simulate an unretriable error") - } - return true, errors.New("Simulate a retriable errror") - } - - type requestStatisticsIncarnate struct { - RetryOps string - RetrySuccessCnt string - ClientRequestTime bucketstats.BucketLog2Round - ClientFailureCnt bucketstats.Total - SwiftRequestTime bucketstats.BucketLog2Round - SwiftRetryOps bucketstats.Average - } - - var ( - opname string - retryObj *RetryCtrl - - reqStat = requestStatisticsIncarnate{ - RetryOps: "proxyfs.switclient.test.operations", - RetrySuccessCnt: "proxyfs.switclient.test.success.operations", - } - statNm = requestStatistics{ - retryCnt: &reqStat.RetryOps, - retrySuccessCnt: &reqStat.RetrySuccessCnt, - clientRequestTime: &reqStat.ClientRequestTime, - clientFailureCnt: &reqStat.ClientFailureCnt, - swiftRequestTime: &reqStat.SwiftRequestTime, - swiftRetryOps: &reqStat.SwiftRetryOps, - } - ) - - // the statistics should be registered before use - bucketstats.Register("swiftclient", "api_test", &reqStat) - defer bucketstats.UnRegister("swiftclient", "api_test") - - // requests succeeds on first try (no log entry, stats not updated) - // - opname = "swiftclient.testRetry.request(1)" - retryObj = NewRetryCtrl(5, 1*time.Second, 2.0) - callCnt = 0 - successOn = 1 - unretriableOn = 0 // never happens - testRetrySucceeds(t, &logcopy, opname, retryObj, statNm, request, successOn, 0.000, 1.000) - - // request succeeds after 3 retries (4th attempt) which should take - // [30 msec, 60 msec) with expBackoff set to 1.0 - // - opname = "swiftclient.testRetry.request(2)" - retryObj = NewRetryCtrl(5, 10*time.Millisecond, 1.0) - callCnt = 0 - successOn = 4 - unretriableOn = 0 // never happens - testRetrySucceeds(t, &logcopy, opname, retryObj, statNm, request, successOn, 0.030, 0.060) - - // requests fails after 4 retries (5th attempt) with a retriable error - // which should take [150 msec, 300 msec) with expBackoff set to 2.0 - // - opname = "swiftclient.testRetry.request(3)" - retryObj = NewRetryCtrl(4, 10*time.Millisecond, 2.0) - callCnt = 0 - successOn = 0 // no success - unretriableOn = 0 // no unretriable failure - testRetryFails(t, &logcopy, opname, retryObj, statNm, request, 5, 0.150, 0.300, "retriable") - - // requests fails after 2 retries (3rd attempt) with an unretriable - // error, which should take [30 msec, 60 msec) with expBackoff set to 2.0 - // - opname = "swiftclient.testRetry.request(4)" - retryObj = NewRetryCtrl(4, 10*time.Millisecond, 2.0) - callCnt = 0 - successOn = 0 // no success - unretriableOn = 3 - testRetryFails(t, &logcopy, opname, retryObj, statNm, request, 3, 0.030, 0.060, "unretriable") - - // retries disabled, no errors - // - opname = "swiftclient.testRetry.request(5)" - retryObj = NewRetryCtrl(0, 10*time.Millisecond, 2.0) - callCnt = 0 - successOn = 1 // success on first try - unretriableOn = 0 - testRetrySucceeds(t, &logcopy, opname, retryObj, statNm, request, successOn, 0.000, 0.010) - - // retries disabled and request fails with retriable error - // - opname = "swiftclient.testRetry.request(6)" - retryObj = NewRetryCtrl(0, 10*time.Millisecond, 2.0) - callCnt = 0 - successOn = 0 // no success - unretriableOn = 0 - testRetryFails(t, &logcopy, opname, retryObj, statNm, request, 1, 0.000, 0.010, "retriable") - - // retries disabled and request fails withan unretriable error - // - opname = "swiftclient.testRetry.request(7)" - retryObj = NewRetryCtrl(0, 10*time.Millisecond, 2.0) - callCnt = 0 - successOn = 0 // no success - unretriableOn = 1 - testRetryFails(t, &logcopy, opname, retryObj, statNm, request, 1, 0.000, 0.010, "unretriable") -} - -// Test an operation that succeeds on attempt number successOn after at least -// minSec of delay...but less than maxSec of delay. -// -// successOn may be 1, in which case no log messages should be generated and the -// retry counters are unchanged (because no retries occurred), else we expect -// properly formatted log messages and updated retry counters. -// -func testRetrySucceeds(t *testing.T, logcopy *logger.LogTarget, - opname string, retryObj *RetryCtrl, reqStat requestStatistics, - request func() (bool, error), successOn int, minSec float32, maxSec float32) { - - var ( - totalEntriesPre int = logcopy.LogBuf.TotalEntries - retryCntPre uint64 - retryCntPost uint64 - retrySuccessCntPre uint64 - retrySuccessCntPost uint64 - statMap map[string]uint64 - logEntry string - logval map[string]string - err error - ) - statMap = stats.Dump() - retryCntPre, _ = statMap[*reqStat.retryCnt] - retrySuccessCntPre, _ = statMap[*reqStat.retrySuccessCnt] - - err = retryObj.RequestWithRetry(request, &opname, &reqStat) - if err != nil { - t.Errorf("%s: should have succeeded, error: %s", opname, err) - } - - // validate proper log entry found, including number of attempts and - // elapsed time - logEntry = logcopy.LogBuf.LogEntries[0] - if logEntry != "" { - logval = parseRetryLogEntry(logEntry) - } else { - logval = nil - } - - switch { - case successOn == 1: - if logcopy.LogBuf.TotalEntries != totalEntriesPre { - t.Errorf("%s: should not have created a log entry", opname) - } - case logcopy.LogBuf.TotalEntries != totalEntriesPre+1: - t.Errorf("%s: should have created exactly one log entry", opname) - case logval == nil: - t.Errorf("%s: log entry should exist, instead found: %s", opname, logEntry) - case logval["result"] != "succeeded" || logval["retriable"] != "" || - logval["attempts"] == "" || logval["seconds"] == "": - t.Errorf("%s: proper log entry should exist, instead found: %s", opname, logEntry) - default: - var ( - sec float32 - attempts int - ) - cnt, err := fmt.Sscanf(logval["seconds"], "%f", &sec) - if err != nil || cnt != 1 { - t.Errorf("%s: seconds not found in log entry: %s", opname, logEntry) - } - cnt, err = fmt.Sscanf(logval["attempts"], "%d", &attempts) - if err != nil || cnt != 1 { - t.Errorf("%s: attempts not found in log entry: %s", opname, logEntry) - } - - if attempts != successOn { - t.Errorf("%s: should have succeeded after %d attempts, log entry shows: %s", - opname, successOn, logEntry) - } - - if sec < minSec { - t.Errorf("%s: elapsed time %4.3f sec outside bounds, should be [%4.3f, %4.3f)", - opname, sec, minSec, maxSec) - } - if sec >= maxSec { - t.Logf("%s: elapsed time %4.3f sec outside bounds, should be [%4.3f, %4.3f)", - opname, sec, minSec, maxSec) - } - } - - // insure stats are updated correctly (unchanged if no retry occurred, - // otherwise both are incremented) - if successOn > 1 { - retryCntPre++ - retrySuccessCntPre++ - } - - // stats sometimes take a little while to update, so wait a bit if we don't - // get the right answer on the first try - for try := 0; try < 500; try++ { - statMap = stats.Dump() - retryCntPost, _ = statMap[*reqStat.retryCnt] - retrySuccessCntPost, _ = statMap[*reqStat.retrySuccessCnt] - - if retryCntPost == retryCntPost && retrySuccessCntPost == retrySuccessCntPre { - break - } - time.Sleep(20 * time.Millisecond) - } - if retryCntPost != retryCntPre { - t.Errorf("%s: stats updated incorrectly: retryOps is %d should be %d", - opname, retryCntPost, retryCntPre) - } - if retrySuccessCntPost != retrySuccessCntPre { - t.Errorf("%s: stats updated incorrectly: retrySuccessOps is %d should be %d", - opname, retrySuccessCntPost, retrySuccessCntPre) - } -} - -// Test an operation that ultimately fails after some number of retries -// (possibly 0). It should fail on the attempt number failOn, after at least -// minSec of delay...but less than maxSec of delay. -// -func testRetryFails(t *testing.T, logcopy *logger.LogTarget, - opname string, retryObj *RetryCtrl, reqStat requestStatistics, - request func() (bool, error), failOn int, minSec float32, maxSec float32, retryStr string) { - - var ( - totalEntriesPre int = logcopy.LogBuf.TotalEntries - retryCntPre uint64 - retryCntPost uint64 - retrySuccessCntPre uint64 - retrySuccessCntPost uint64 - statMap map[string]uint64 - logEntry string - logval map[string]string - err error - ) - statMap = stats.Dump() - retryCntPre, _ = statMap[*reqStat.retryCnt] - retrySuccessCntPre, _ = statMap[*reqStat.retrySuccessCnt] - - err = retryObj.RequestWithRetry(request, &opname, &reqStat) - if err == nil { - t.Errorf("%s: should have failed, error: %s", opname, err) - } - - // validate proper log entry found, including number of attempts and - // elapsed time - logEntry = logcopy.LogBuf.LogEntries[0] - logval = parseRetryLogEntry(logEntry) - switch { - case logcopy.LogBuf.TotalEntries != totalEntriesPre+1: - t.Errorf("%s: should have created exactly one log entry", opname) - case logval == nil: - t.Errorf("%s: log entry should exist, instead found: %s", opname, logEntry) - case logval["result"] != "failed" || logval["retriable"] != retryStr || - logval["attempts"] == "" || logval["seconds"] == "": - t.Errorf("%s: proper log entry should exist, instead found: %s", opname, logEntry) - default: - var ( - sec float32 - attempts int - ) - cnt, err := fmt.Sscanf(logval["seconds"], "%f", &sec) - if err != nil || cnt != 1 { - t.Errorf("%s: seconds not found in log entry: %s", opname, logEntry) - } - cnt, err = fmt.Sscanf(logval["attempts"], "%d", &attempts) - if err != nil || cnt != 1 { - t.Errorf("%s: attempts not found in log entry: %s", opname, logEntry) - } - - if attempts != failOn { - t.Errorf("%s: should have failed after %d attempts, log entry shows: %s", - opname, failOn, logEntry) - } - - if sec < minSec { - t.Errorf("%s: elapsed time %4.3f sec outside bounds, should be [%4.3f, %4.3f)", - opname, sec, minSec, maxSec) - } - if sec >= maxSec { - t.Logf("%s: elapsed time %4.3f sec outside bounds, should be [%4.3f, %4.3f)", - opname, sec, minSec, maxSec) - } - } - - // insure stats are updated correctly: retrySuccessOps is never incremented; - // retryOps incremented only if retries are enabled - if retryObj.attemptMax > 0 { - retryCntPre++ - } - - // stats sometimes take a little while to update, so wait a bit if we don't - // get the right answer on the first try - for try := 0; try < 10; try++ { - statMap = stats.Dump() - retryCntPost, _ = statMap[*reqStat.retryCnt] - retrySuccessCntPost, _ = statMap[*reqStat.retrySuccessCnt] - - if retryCntPost == retryCntPost && retrySuccessCntPost == retrySuccessCntPre { - break - } - time.Sleep(time.Second) - } - if retryCntPost != retryCntPre { - t.Errorf("%s: stats updated incorrectly: retryOps is %d should be %d", - opname, retryCntPost, retryCntPre) - } - if retrySuccessCntPost != retrySuccessCntPre { - t.Errorf("%s: stats updated incorrectly: retrySuccessOps is %d should be %d", - opname, retrySuccessCntPost, retrySuccessCntPre) - } -} - -// Test intermittent bad connections (can't connect to server) -// -func testBadConnection(t *testing.T) { - - // preserve the original settings of these globals that we change - var ( - chaosOpenConnectionFailureRate = atomic.LoadUint32(&globals.chaosOpenConnectionFailureRate) - chaosSendChunkFailureRate = atomic.LoadUint64(&globals.chaosSendChunkFailureRate) - chaosFetchChunkedPutFailureRate = atomic.LoadUint64(&globals.chaosFetchChunkedPutFailureRate) - - retryLimitObject = globals.retryLimitObject - retryDelay = globals.retryDelay - nonChunkedConnectionPoolSize = globals.nonChunkedConnectionPool.poolCapacity - - cleanup = func() { - atomic.StoreUint32(&globals.chaosOpenConnectionFailureRate, chaosOpenConnectionFailureRate) - atomic.StoreUint64(&globals.chaosSendChunkFailureRate, chaosSendChunkFailureRate) - atomic.StoreUint64(&globals.chaosFetchChunkedPutFailureRate, chaosFetchChunkedPutFailureRate) - - globals.retryLimitObject = retryLimitObject - globals.retryDelay = retryDelay - resizeNonChunkedConnectionPool(uint(nonChunkedConnectionPoolSize)) - } - ) - defer cleanup() - - var ( - accountName = "TestAccount" - containerName = "TestContainer" - objNameFmt = "chunkObj%d" - objName string - ) - - // increase the pool sizes to get some concurrency (chunked isn't tested here) - resizeNonChunkedConnectionPool(12) - - // (lack of) headers for putting - catDogHeaderMap := make(map[string][]string) - - // (re)create the test account and container - - err := AccountPut(accountName, catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("testBadConnection.AccountPut('%s', catDogHeaderMap) failed: %v", accountName, err) - t.Fatalf(tErr) - } - - err = ContainerPut(accountName, containerName, catDogHeaderMap) - if nil != err { - tErr := fmt.Sprintf("testBadConnection.ContainerPut('%s', '%s', catDogHeaderMap) failed: %v", - accountName, containerName, err) - t.Fatalf(tErr) - } - - // create a couple of dozen objects to work with - nObject := 24 - atomic.StoreUint64(&globals.chaosSendChunkFailureRate, 0) - atomic.StoreUint64(&globals.chaosFetchChunkedPutFailureRate, 0) - - objSize := 64 * 1024 - randGen := rand.New(rand.NewSource(2)) - for i := 0; i < nObject; i++ { - objName = fmt.Sprintf(objNameFmt, i) - - // write an object and verify it - err = testObjectWriteVerify(t, accountName, containerName, objName, objSize, randGen) - if err != nil { - fmt.Printf("testBadConnection: FATAL ERROR: %v\n", err) - t.Error(err) - } - } - - // verify we can access the last object - _, err = ObjectHead(accountName, containerName, objName) - if err != nil { - fmt.Printf("testBadConnection: HEAD of object %s failed: %v\n", objName, err) - t.Error(err) - } - - // cause all openConnection calls to fails and close any cached connections - atomic.StoreUint32(&globals.chaosOpenConnectionFailureRate, 1) - drainConnections() - - _, err = ObjectHead(accountName, containerName, objName) - if err == nil { - fmt.Printf("testBadConnection: HEAD of object %s should have failed: %v\n", - objName, err) - t.Error(err) - } - - // object operations will retry for: - // globals.retryDelayObjects * (globals.RetryExpBackoffObject ^ globals.retryLimitObject - 1) - // - // before they fail. once globals.retryLimitObject is set to 6, with the existing - // paramaters this works out to 25 msec * (2.0^6 - 1) == 1.575 sec. - // - // disable openConnection() for 500 msec and then enable it. the HEAD - // request should succeed during the last 2 or 3 retries. - globals.retryLimitObject = 6 - - var rendezvous sync.WaitGroup - - rendezvous.Add(1) - go func() { - _, err = ObjectHead(accountName, containerName, objName) - if err != nil { - fmt.Printf("testBadConnection: HEAD of object %s failed: %v\n", objName, err) - t.Error(err) - } - - rendezvous.Done() - }() - time.Sleep(500 * time.Millisecond) - - atomic.StoreUint32(&globals.chaosOpenConnectionFailureRate, 0) - rendezvous.Wait() - - // try a batch of HEAD operations in parallel in the same scenario. - // - // unfortunately, we also have ramswift Chaos settings that cause every 2nd object - // HEAD request to fail ("RamSwiftChaos.ObjectHeadFailureRate=2" at the top of - // this file). that means that if we submit 4 HEAD requests, about 2 of them will - // fail. of those 4, the 2 that succeed will exit while the 2 that fail will try - // again after the retry interval expires, at which point 1 will succeed and the - // the other will fail. - // - // start with 8 HEAD requests. while chaosOpenConnectionFailureRate == 1 all of - // the requests will fail. after chaosOpenConnectionFailureRate == 0 then half of - // the requests will succeed, so we need 4 successful retry cycles for all 8 to - // succeed. waiting 60 msec before enabled connections should leave enough time - // for all 8 requests to complete. - atomic.StoreUint32(&globals.chaosOpenConnectionFailureRate, 1) - for i := 0; i < 8; i++ { - objName = fmt.Sprintf(objNameFmt, i) - - rendezvous.Add(1) - go func(localObjName string) { - var localErr error - _, localErr = ObjectHead(accountName, containerName, localObjName) - if localErr != nil { - fmt.Printf("testBadConnection: HEAD of object %s failed: %v\n", localObjName, localErr) - t.Error(localErr) - } - - rendezvous.Done() - }(objName) - } - time.Sleep(60 * time.Millisecond) - - atomic.StoreUint32(&globals.chaosOpenConnectionFailureRate, 0) - rendezvous.Wait() - - // cleanup the mess we made (objects, container, and account) - atomic.StoreUint32(&globals.chaosOpenConnectionFailureRate, 0) - for i := 0; i < nObject; i++ { - objName = fmt.Sprintf(objNameFmt, i) - - err = ObjectDelete(accountName, containerName, objName, 0) - if nil != err { - tErr := fmt.Sprintf("ObjectDelete('%s', '%s', '%s') failed: %v", - accountName, containerName, objName, err) - t.Error(tErr) - } - } - - err = ContainerDelete(accountName, containerName) - if nil != err { - tErr := fmt.Sprintf("ContainerDelete('%s', '%s') failed: %v", accountName, containerName, err) - t.Error(tErr) - } - err = AccountDelete(accountName) - if nil != err { - tErr := fmt.Sprintf("AccountDelete('%s') failed: %v", accountName, err) - t.Error(tErr) - } -} diff --git a/swiftclient/config.go b/swiftclient/config.go deleted file mode 100644 index a6b3176d..00000000 --- a/swiftclient/config.go +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package swiftclient - -import ( - "container/list" - "fmt" - "net" - "strconv" - "time" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/trackedlock" - "github.com/NVIDIA/proxyfs/transitions" -) - -type connectionStruct struct { - connectionNonce uint64 // globals.connectionNonce at time connection was established - tcpConn *net.TCPConn - reserveForVolumeName string -} - -type connectionPoolStruct struct { - trackedlock.Mutex - poolCapacity uint16 // Set to SwiftClient.{|Non}ChunkedConnectionPoolSize - poolInUse uint16 // Active (i.e. not in LIFO) *connectionStruct's - lifoIndex uint16 // Indicates where next released *connectionStruct will go - lifoOfAvailableConnections []*connectionStruct // LIFO of available active connections - numWaiters uint64 // Count of the number of blocked acquirers - waiters *list.List // Contains sync.Cond's of waiters - // At time of connection release: - // If poolInUse < poolCapacity, - // If keepAlive: connectionStruct pushed to lifoOfAvailableConnections - // If poolInUse == poolCapacity, - // If keepAlive: connectionStruct pushed to lifoOfAvailableConnections - // sync.Cond at front of waitors is awakened - // If poolInUse > poolCapacity, - // poolInUse is decremented and connection is discarded - // Note: waiters list is not used for when in starvation mode - // for the chunkedConnectionPool if a starvationCallback - // has been provided -} - -// Used to track client request times (client's of swiftclient) and Swift server -// request times using bucketized statistics -// -type requestTimeStatistic struct { - bucketstats.BucketLog2Round -} - -// Used to track retries using an average, where the count increases by 1 for -// each retry and the total increases by 1 for each successful retry -// -type requestRetryStatistic struct { - bucketstats.Average -} - -// Used to track client request failures -// -type requestFailureStatistic struct { - bucketstats.Total -} - -type globalsStruct struct { - noAuthStringAddr string - noAuthTCPAddr *net.TCPAddr - retryLimit uint16 // maximum retries - retryLimitObject uint16 // maximum retries for object ops - retryDelay time.Duration // delay before first retry - retryDelayObject time.Duration // delay before first retry for object ops - retryExpBackoff float64 // increase delay by this factor each try (exponential backoff) - retryExpBackoffObject float64 // increase delay by this factor each try for object ops - connectionNonce uint64 // incremented each SIGHUP... older connections always closed - chunkedConnectionPool connectionPoolStruct - nonChunkedConnectionPool connectionPoolStruct - starvationCallback StarvationCallbackFunc - starvationCallbackSerializer trackedlock.Mutex - reservedChunkedConnection map[string]*connectionStruct // Key: VolumeName - reservedChunkedConnectionMutex trackedlock.Mutex - maxIntAsUint64 uint64 - checksumChunkedPutChunks bool // compute and verify checksums (for testing) - chaosFetchChunkedPutFailureRate uint64 // set only during testing - chaosSendChunkFailureRate uint64 // set only during testing - chaosCloseChunkFailureRate uint64 // set only during testing - chaosOpenConnectionFailureRate uint32 // set only during testing - - // statistics for requests to swiftclient - AccountDeleteUsec bucketstats.BucketLog2Round // bucketized by time - AccountGetUsec bucketstats.BucketLog2Round // bucketized by time - AccountHeadUsec bucketstats.BucketLog2Round // bucketized by time - AccountPostUsec bucketstats.BucketLog2Round // bucketized by time - AccountPutUsec bucketstats.BucketLog2Round // bucketized by time - ContainerDeleteUsec bucketstats.BucketLog2Round // bucketized by time - ContainerGetUsec bucketstats.BucketLog2Round // bucketized by time - ContainerHeadUsec bucketstats.BucketLog2Round // bucketized by time - ContainerPostUsec bucketstats.BucketLog2Round // bucketized by time - ContainerPutUsec bucketstats.BucketLog2Round // bucketized by time - ObjectContentLengthUsec bucketstats.BucketLog2Round // bucketized by time - ObjectCopyUsec bucketstats.BucketLog2Round // bucketized by time - ObjectDeleteUsec bucketstats.BucketLog2Round // bucketized by time - ObjectGetUsec bucketstats.BucketLog2Round // bucketized by time - ObjectGetBytes bucketstats.BucketLog2Round // bucketized by byte count - ObjectHeadUsec bucketstats.BucketLog2Round // bucketized by time - ObjectLoadUsec bucketstats.BucketLog2Round // bucketized by time - ObjectLoadBytes bucketstats.BucketLog2Round // bucketized by byte count - ObjectPostUsec bucketstats.BucketLog2Round // bucketized by time - ObjectReadUsec bucketstats.BucketLog2Round // bucketized by time - ObjectReadBytes bucketstats.BucketLog2Round // bucketized by byte count - ObjectTailUsec bucketstats.BucketLog2Round // bucketized by time - ObjectTailBytes bucketstats.BucketLog2Round // bucketized by byte count - ObjectNonChunkedFreeConnection bucketstats.BucketLogRoot2Round // free non-chunked connections (at acquire time) - ObjectPutCtxtFetchUsec bucketstats.BucketLog2Round // bucketized by time - ObjectPutCtxtFreeConnection bucketstats.BucketLogRoot2Round // free chunked put connections (at acquite time) - ObjectPutCtxtBytesPut bucketstats.Total // number of calls to BytesPut() query - ObjectPutCtxtCloseUsec bucketstats.BucketLog2Round // bucketized by time - ObjectPutCtxtReadBytes bucketstats.BucketLog2Round // bucketized by bytes read - ObjectPutCtxtSendChunkUsec bucketstats.BucketLog2Round // bucketized by time - ObjectPutCtxtSendChunkBytes bucketstats.BucketLog2Round // bucketized by byte count - ObjectPutCtxtBytes bucketstats.BucketLog2Round // bucketized by total bytes put - ObjectPutCtxtFetchToCloseUsec bucketstats.BucketLog2Round // Fetch returns to Close called time - - // client request failures - AccountDeleteFailure bucketstats.Total - AccountGetFailure bucketstats.Total - AccountHeadFailure bucketstats.Total - AccountPostFailure bucketstats.Total - AccountPutFailure bucketstats.Total - ContainerDeleteFailure bucketstats.Total - ContainerGetFailure bucketstats.Total - ContainerHeadFailure bucketstats.Total - ContainerPostFailure bucketstats.Total - ContainerPutFailure bucketstats.Total - ObjectContentLengthFailure bucketstats.Total - ObjectDeleteFailure bucketstats.Total - ObjectGetFailure bucketstats.Total - ObjectHeadFailure bucketstats.Total - ObjectLoadFailure bucketstats.Total - ObjectPostFailure bucketstats.Total - ObjectReadFailure bucketstats.Total - ObjectTailFailure bucketstats.Total - ObjectPutCtxtFetchFailure bucketstats.Total - ObjectPutCtxtCloseFailure bucketstats.Total - - // statistics for swiftclient requests to Swift - SwiftAccountDeleteUsec bucketstats.BucketLog2Round // bucketized by time - SwiftAccountGetUsec bucketstats.BucketLog2Round // bucketized by time - SwiftAccountHeadUsec bucketstats.BucketLog2Round // bucketized by time - SwiftAccountPostUsec bucketstats.BucketLog2Round // bucketized by time - SwiftAccountPutUsec bucketstats.BucketLog2Round // bucketized by time - SwiftContainerDeleteUsec bucketstats.BucketLog2Round // bucketized by time - SwiftContainerGetUsec bucketstats.BucketLog2Round // bucketized by time - SwiftContainerHeadUsec bucketstats.BucketLog2Round // bucketized by time - SwiftContainerPostUsec bucketstats.BucketLog2Round // bucketized by time - SwiftContainerPutUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectContentLengthUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectDeleteUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectGetUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectHeadUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectLoadUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectPostUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectReadUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectTailUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectPutCtxtFetchUsec bucketstats.BucketLog2Round // bucketized by time - SwiftObjectPutCtxtCloseUsec bucketstats.BucketLog2Round // bucketized by time - - // statistics for retries to Swift, Count() is the number of retries and - // Total() is the number successful - SwiftAccountDeleteRetryOps bucketstats.Average - SwiftAccountGetRetryOps bucketstats.Average - SwiftAccountHeadRetryOps bucketstats.Average - SwiftAccountPostRetryOps bucketstats.Average - SwiftAccountPutRetryOps bucketstats.Average - SwiftContainerDeleteRetryOps bucketstats.Average - SwiftContainerGetRetryOps bucketstats.Average - SwiftContainerHeadRetryOps bucketstats.Average - SwiftContainerPostRetryOps bucketstats.Average - SwiftContainerPutRetryOps bucketstats.Average - SwiftObjectContentLengthRetryOps bucketstats.Average - SwiftObjectDeleteRetryOps bucketstats.Average - SwiftObjectGetRetryOps bucketstats.Average - SwiftObjectHeadRetryOps bucketstats.Average - SwiftObjectLoadRetryOps bucketstats.Average - SwiftObjectPostRetryOps bucketstats.Average - SwiftObjectReadRetryOps bucketstats.Average - SwiftObjectTailRetryOps bucketstats.Average - SwiftObjectPutCtxtFetchRetryOps bucketstats.Average - SwiftObjectPutCtxtCloseRetryOps bucketstats.Average -} - -var globals globalsStruct - -func init() { - transitions.Register("swiftclient", &globals) -} - -// Update the configuration variables that can be changed by sending proxyfs a -// SIGHUP. Note that others, like "SwiftClient.ChunkedConnectionPoolSize" -// cannot currently be changed. -// -func (dummy *globalsStruct) reloadConfig(confMap conf.ConfMap) (err error) { - - globals.retryLimit, err = confMap.FetchOptionValueUint16("SwiftClient", "RetryLimit") - if nil != err { - return - } - globals.retryLimitObject, err = confMap.FetchOptionValueUint16("SwiftClient", "RetryLimitObject") - if nil != err { - return - } - - globals.retryDelay, err = confMap.FetchOptionValueDuration("SwiftClient", "RetryDelay") - if nil != err { - return - } - globals.retryDelayObject, err = confMap.FetchOptionValueDuration("SwiftClient", "RetryDelayObject") - if nil != err { - return - } - - globals.retryExpBackoff, err = confMap.FetchOptionValueFloat64("SwiftClient", "RetryExpBackoff") - if nil != err { - return - } - globals.retryExpBackoffObject, err = confMap.FetchOptionValueFloat64("SwiftClient", "RetryExpBackoffObject") - if nil != err { - return - } - logger.Infof("SwiftClient.RetryLimit %d, SwiftClient.RetryDelay %4.3f sec, SwiftClient.RetryExpBackoff %2.1f", - globals.retryLimit, float64(globals.retryDelay)/float64(time.Second), globals.retryExpBackoff) - logger.Infof("SwiftClient.RetryLimitObject %d, SwiftClient.RetryDelayObject %4.3f sec, SwiftClient.RetryExpBackoffObject %2.1f", - globals.retryLimitObject, float64(globals.retryDelayObject)/float64(time.Second), - globals.retryExpBackoffObject) - - globals.checksumChunkedPutChunks, err = confMap.FetchOptionValueBool("SwiftClient", "ChecksumChunkedPutChunks") - if nil != err { - err = nil - globals.checksumChunkedPutChunks = false - } - checksums := "disabled" - if globals.checksumChunkedPutChunks { - checksums = "enabled" - } - logger.Infof("SwiftClient.ChecksumChunkedPutChunks %s\n", checksums) - - return -} - -func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { - var ( - chunkedConnectionPoolSize uint16 - freeConnectionIndex uint16 - noAuthIPAddr string - noAuthTCPPort uint16 - nonChunkedConnectionPoolSize uint16 - ) - - // register the bucketstats statistics tracked here - bucketstats.Register("proxyfs.swiftclient", "", &globals) - - noAuthIPAddr, err = confMap.FetchOptionValueString("SwiftClient", "NoAuthIPAddr") - if nil != err { - noAuthIPAddr = "127.0.0.1" // TODO: Eventually just return - } - - noAuthTCPPort, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") - if nil != err { - return - } - if uint16(0) == noAuthTCPPort { - err = fmt.Errorf("SwiftClient.NoAuthTCPPort must be a non-zero uint16") - return - } - - globals.noAuthStringAddr = noAuthIPAddr + ":" + strconv.Itoa(int(noAuthTCPPort)) - - globals.noAuthTCPAddr, err = net.ResolveTCPAddr("tcp4", globals.noAuthStringAddr) - if nil != err { - return - } - - globals.connectionNonce = 0 - - chunkedConnectionPoolSize, err = confMap.FetchOptionValueUint16("SwiftClient", "ChunkedConnectionPoolSize") - if nil != err { - return - } - if uint16(0) == chunkedConnectionPoolSize { - err = fmt.Errorf("SwiftClient.ChunkedConnectionPoolSize must be a non-zero uint16") - return - } - - globals.chunkedConnectionPool.poolCapacity = chunkedConnectionPoolSize - globals.chunkedConnectionPool.poolInUse = 0 - globals.chunkedConnectionPool.lifoIndex = 0 - globals.chunkedConnectionPool.lifoOfAvailableConnections = make([]*connectionStruct, chunkedConnectionPoolSize) - globals.chunkedConnectionPool.numWaiters = 0 - globals.chunkedConnectionPool.waiters = list.New() - - for freeConnectionIndex = uint16(0); freeConnectionIndex < chunkedConnectionPoolSize; freeConnectionIndex++ { - globals.chunkedConnectionPool.lifoOfAvailableConnections[freeConnectionIndex] = nil - } - - nonChunkedConnectionPoolSize, err = confMap.FetchOptionValueUint16("SwiftClient", "NonChunkedConnectionPoolSize") - if nil != err { - return - } - if uint16(0) == nonChunkedConnectionPoolSize { - err = fmt.Errorf("SwiftClient.NonChunkedConnectionPoolSize must be a non-zero uint16") - return - } - - globals.nonChunkedConnectionPool.poolCapacity = nonChunkedConnectionPoolSize - globals.nonChunkedConnectionPool.poolInUse = 0 - globals.nonChunkedConnectionPool.lifoIndex = 0 - globals.nonChunkedConnectionPool.lifoOfAvailableConnections = make([]*connectionStruct, nonChunkedConnectionPoolSize) - globals.nonChunkedConnectionPool.numWaiters = 0 - globals.nonChunkedConnectionPool.waiters = list.New() - - for freeConnectionIndex = uint16(0); freeConnectionIndex < nonChunkedConnectionPoolSize; freeConnectionIndex++ { - globals.nonChunkedConnectionPool.lifoOfAvailableConnections[freeConnectionIndex] = nil - } - - globals.starvationCallback = nil - - globals.reservedChunkedConnection = make(map[string]*connectionStruct) - - globals.maxIntAsUint64 = uint64(^uint(0) >> 1) - - // load the configuration tunables that can be changed after boot - err = dummy.reloadConfig(confMap) - - return -} - -func (dummy *globalsStruct) VolumeGroupCreated(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupMoved(confMap conf.ConfMap, volumeGroupName string, activePeer string, virtualIPAddr string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeGroupDestroyed(confMap conf.ConfMap, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeCreated(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeMoved(confMap conf.ConfMap, volumeName string, volumeGroupName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeDestroyed(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) UnserveVolume(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} -func (dummy *globalsStruct) VolumeToBeUnserved(confMap conf.ConfMap, volumeName string) (err error) { - return nil -} - -func (dummy *globalsStruct) SignaledStart(confMap conf.ConfMap) (err error) { - drainConnections() - globals.connectionNonce++ - - // load the configuration tunables that can be changed after boot - err = dummy.reloadConfig(confMap) - - return -} - -func (dummy *globalsStruct) SignaledFinish(confMap conf.ConfMap) (err error) { - return nil -} - -func (dummy *globalsStruct) Down(confMap conf.ConfMap) (err error) { - drainConnections() - globals.connectionNonce++ - bucketstats.UnRegister("proxyfs.swiftclient", "") - - err = nil - return -} diff --git a/swiftclient/container.go b/swiftclient/container.go deleted file mode 100644 index 0936dc51..00000000 --- a/swiftclient/container.go +++ /dev/null @@ -1,475 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Swift Container-specific API access implementation - -package swiftclient - -import ( - "fmt" - "net/url" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" -) - -func containerDeleteWithRetry(accountName string, containerName string) (err error) { - // request is a function that, through the miracle of closure, calls - // containerDelete() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - request := func() (bool, error) { - var err error - err = containerDelete(accountName, containerName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - opname string = fmt.Sprintf("swiftclient.containerDelete(\"%v/%v\")", accountName, containerName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftContainerDeleteRetryOps, - retrySuccessCnt: &stats.SwiftContainerDeleteRetrySuccessOps, - clientRequestTime: &globals.ContainerDeleteUsec, - clientFailureCnt: &globals.ContainerDeleteFailure, - swiftRequestTime: &globals.SwiftContainerDeleteUsec, - swiftRetryOps: &globals.SwiftContainerDeleteRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return err -} - -func containerDelete(accountName string, containerName string) (err error) { - var ( - connection *connectionStruct - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "DELETE", "/"+swiftVersion+"/"+pathEscape(accountName, containerName), nil) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPDeleteError) - logger.WarnfWithError(err, "swiftclient.containerDelete(\"%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPDeleteError) - logger.WarnfWithError(err, "swiftclient.containerDelete(\"%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName) - return - } - evtlog.Record(evtlog.FormatContainerDelete, accountName, containerName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "DELETE %s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.containerDelete(\"%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName) - return - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperations(&stats.SwiftContainerDeleteOps) - - return -} - -func containerGetWithRetry(accountName string, containerName string) (headers map[string][]string, objectList []string, err error) { - // request is a function that, through the miracle of closure, calls - // containerGet() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - var ( - connection *connectionStruct - marker string - opname string - retryObj *RetryCtrl - statnm requestStatistics - toAddHeaders map[string][]string - toAddObjectList []string - ) - - retryObj = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - statnm = requestStatistics{ - retryCnt: &stats.SwiftContainerGetRetryOps, - retrySuccessCnt: &stats.SwiftContainerGetRetrySuccessOps, - clientRequestTime: &globals.ContainerGetUsec, - clientFailureCnt: &globals.ContainerGetFailure, - swiftRequestTime: &globals.SwiftContainerGetUsec, - swiftRetryOps: &globals.SwiftContainerGetRetryOps, - } - - request := func() (bool, error) { - var err error - toAddHeaders, toAddObjectList, err = containerGet(connection, accountName, containerName, marker) - return true, err - } - - headers = make(map[string][]string) - objectList = make([]string, 0) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - marker = "" - - for { - opname = fmt.Sprintf("swiftclient.containerGet(,\"%v\",\"%v\",\"%v\")", accountName, containerName, marker) - - err = retryObj.RequestWithRetry(request, &opname, &statnm) - - if nil == err { - mergeHeadersAndList(headers, &objectList, toAddHeaders, &toAddObjectList) - - if 0 == len(toAddObjectList) { - releaseNonChunkedConnection(connection, parseConnection(headers)) - - break - } else { - marker = toAddObjectList[len(toAddObjectList)-1] - } - } else { - releaseNonChunkedConnection(connection, false) - - break - } - } - - stats.IncrementOperations(&stats.SwiftContainerGetOps) - - return -} - -func containerGet(connection *connectionStruct, accountName string, containerName string, marker string) (headers map[string][]string, objectList []string, err error) { - var ( - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - ) - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "GET", "/"+swiftVersion+"/"+pathEscape(accountName, containerName)+"?marker="+url.QueryEscape(marker), nil) - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.containerGet(,\"%v\",\"%v\",\"%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, marker) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.containerGet(,\"%v\",\"%v\",\"%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, marker) - return - } - evtlog.Record(evtlog.FormatContainerGet, accountName, containerName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - err = blunder.NewError(fsErr, "GET %s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.containerGet(,\"%v\",\"%v\",\"%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, marker) - return - } - - objectList, err = readHTTPPayloadLines(connection.tcpConn, headers) - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.containerGet(,\"%v\",\"%v\",\"%v\") got readHTTPPayloadLines() error", accountName, containerName, marker) - return - } - - return -} - -func containerHeadWithRetry(accountName string, containerName string) (map[string][]string, error) { - // request is a function that, through the miracle of closure, calls - // containerHead() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - var ( - headers map[string][]string - err error - ) - request := func() (bool, error) { - var err error - headers, err = containerHead(accountName, containerName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - opname string = fmt.Sprintf("swiftclient.containerHead(\"%v/%v\")", accountName, containerName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftContainerHeadRetryOps, - retrySuccessCnt: &stats.SwiftContainerHeadRetrySuccessOps, - clientRequestTime: &globals.ContainerHeadUsec, - clientFailureCnt: &globals.ContainerHeadFailure, - swiftRequestTime: &globals.SwiftContainerHeadUsec, - swiftRetryOps: &globals.SwiftContainerHeadRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return headers, err -} - -func containerHead(accountName string, containerName string) (headers map[string][]string, err error) { - var ( - connection *connectionStruct - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "HEAD", "/"+swiftVersion+"/"+pathEscape(accountName, containerName), nil) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.containerHead(\"%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.containerHead(\"%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName) - return - } - evtlog.Record(evtlog.FormatContainerHead, accountName, containerName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "HEAD %s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.containerHead(\"%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName) - return - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperations(&stats.SwiftContainerHeadOps) - - return -} - -func containerPostWithRetry(accountName string, containerName string, requestHeaders map[string][]string) (err error) { - // request is a function that, through the miracle of closure, calls - // containerPost() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - request := func() (bool, error) { - var err error - err = containerPost(accountName, containerName, requestHeaders) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - opname string = fmt.Sprintf("swiftclient.containerPost(\"%v/%v\")", accountName, containerName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftContainerPostRetryOps, - retrySuccessCnt: &stats.SwiftContainerPostRetrySuccessOps, - clientRequestTime: &globals.ContainerPostUsec, - clientFailureCnt: &globals.ContainerPostFailure, - swiftRequestTime: &globals.SwiftContainerPostUsec, - swiftRetryOps: &globals.SwiftContainerPostRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return err -} - -func containerPost(accountName string, containerName string, requestHeaders map[string][]string) (err error) { - var ( - connection *connectionStruct - contentLength int - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - responseHeaders map[string][]string - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - requestHeaders["Content-Length"] = []string{"0"} - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "POST", "/"+swiftVersion+"/"+pathEscape(accountName, containerName), requestHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.containerPost(\"%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName) - return - } - - httpStatus, responseHeaders, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.containerPost(\"%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName) - return - } - evtlog.Record(evtlog.FormatContainerPost, accountName, containerName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, responseHeaders) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "POST %s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.containerPost(\"%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName) - return - } - contentLength, err = parseContentLength(responseHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.containerPost(\"%v/%v\") got parseContentLength() error", accountName, containerName) - return - } - if 0 < contentLength { - _, err = readBytesFromTCPConn(connection.tcpConn, contentLength) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.containerPost(\"%v/%v\") got readBytesFromTCPConn() error", accountName, containerName) - return - } - } - - releaseNonChunkedConnection(connection, parseConnection(responseHeaders)) - - stats.IncrementOperations(&stats.SwiftContainerPostOps) - - return -} - -func containerPutWithRetry(accountName string, containerName string, requestHeaders map[string][]string) (err error) { - // request is a function that, through the miracle of closure, calls - // containerPut() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - request := func() (bool, error) { - var err error - err = containerPut(accountName, containerName, requestHeaders) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - opname string = fmt.Sprintf("swiftclient.containerPut(\"%v/%v\")", accountName, containerName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftContainerPutRetryOps, - retrySuccessCnt: &stats.SwiftContainerPutRetrySuccessOps, - clientRequestTime: &globals.ContainerPutUsec, - clientFailureCnt: &globals.ContainerPutFailure, - swiftRequestTime: &globals.SwiftContainerPutUsec, - swiftRetryOps: &globals.SwiftContainerPutRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return err -} - -func containerPut(accountName string, containerName string, requestHeaders map[string][]string) (err error) { - var ( - connection *connectionStruct - contentLength int - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - responseHeaders map[string][]string - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - requestHeaders["Content-Length"] = []string{"0"} - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "PUT", "/"+swiftVersion+"/"+pathEscape(accountName, containerName), requestHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.containerPut(\"%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName) - return - } - - httpStatus, responseHeaders, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.containerPut(\"%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName) - return - } - evtlog.Record(evtlog.FormatContainerPut, accountName, containerName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, responseHeaders) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "PUT %s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.containerPut(\"%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName) - return - } - contentLength, err = parseContentLength(responseHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.containerPut(\"%v/%v\") got parseContentLength() error", accountName, containerName) - return - } - if 0 < contentLength { - _, err = readBytesFromTCPConn(connection.tcpConn, contentLength) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.containerPut(\"%v/%v\") got readBytesFromTCPConn() error", accountName, containerName) - return - } - } - - releaseNonChunkedConnection(connection, parseConnection(responseHeaders)) - - stats.IncrementOperations(&stats.SwiftContainerPutOps) - - return -} diff --git a/swiftclient/http_status.go b/swiftclient/http_status.go deleted file mode 100644 index 8a685498..00000000 --- a/swiftclient/http_status.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// HTTP Status Code Decoder - -package swiftclient - -import ( - "github.com/NVIDIA/proxyfs/blunder" -) - -func httpStatusIsSuccess(httpStatus int) (isSuccess bool) { - return httpStatus >= 200 && httpStatus <= 299 -} - -func httpStatusIsError(httpStatus int) (isError bool, fsErr blunder.FsError) { - - switch { - - // 1xx Informational - // - // 100 Continue - // 101 Switching Protocols - // 102 Processing (WebDAV) - case httpStatus >= 100 && httpStatus <= 199: - return false, blunder.SuccessError - - // 2xx Success - // - // 200 OK - // 201 Created - // 202 Accepted - // 203 Non-Authoritative Information - // 204 No Content - // 205 Reset Content - // 206 Partial Content - // 207 Multi-Status (WebDAV) - // 208 Already Reported (WebDAV) - // 226 IM Used - case httpStatus >= 200 && httpStatus <= 299: - return false, blunder.SuccessError - - // 3xx Redirection - // - // 300 Multiple Choices - // 301 Moved Permanently - // 302 Found - // 303 See Other - // 304 Not Modified - // 305 Use Proxy - // 306 (Unused) - // 307 Temporary Redirect - // 308 Permanent Redirect (experiemental) - case httpStatus >= 300 && httpStatus <= 399: - // XXX TODO: Make error more unique? - return true, blunder.NotImplementedError - - // 4xx Client Error - // - - // 401 Unauthorized - case httpStatus == 401: - return true, blunder.NotPermError - // 403 Forbidden - case httpStatus == 403: - return true, blunder.NotPermError - // 404 Not Found - case httpStatus == 404: - return true, blunder.NotFoundError - - // 400 Bad Request - // 402 Payment Required - // 405 Method Not Allowed - // 406 Not Acceptable - // 407 Proxy Authentication Required - // 408 Request Timeout - - // 409 Conflict - case httpStatus == 409: - // Swift returns this error when one tries to delete a container that isn't empty - return true, blunder.NotEmptyError - - // 410 Gone - // 411 Length Required - // 412 Precondition Failed - // 413 Request Entity Too Large - // 414 Request-URI Too Long - // 415 Unsupported Media Type - // 416 Requested Range Not Satisfiable - // 417 Expectation Failed - // 418 I'm a teapot (RFC 2324) - // 420 Enhance Your Calm (Twitter) - // 422 Unprocessable Entity (WebDAV) - // 423 Locked (WebDAV) - // 424 Failed Dependency (WebDAV) - // 425 Reserved for WebDAV - // 426 Upgrade Required - // 428 Precondition Required - // 429 Too Many Requests - // 431 Request Header Fields Too Large - // 444 No Response (Nginx) - // 449 Retry With (Microsoft) - // 450 Blocked by Windows Parental Controls (Microsoft) - // 451 Unavailable For Legal Reasons - // 499 Client Closed Request (Nginx) - case httpStatus >= 400 && httpStatus <= 499: - // XXX TODO: Make error more unique? - return true, blunder.NotImplementedError - - // 5xx Server Error - // - // 500 Internal Server Error - // 501 Not Implemented - // 502 Bad Gateway - // 503 Service Unavailable - // 504 Gateway Timeout - // 505 HTTP Version Not Supported - // 506 Variant Also Negotiates (Experimental) - // 507 Insufficient Storage (WebDAV) - // 508 Loop Detected (WebDAV) - // 509 Bandwidth Limit Exceeded (Apache) - // 510 Not Extended - // 511 Network Authentication Required - // 598 Network read timeout error - // 599 Network connect timeout error - case httpStatus >= 500 && httpStatus <= 599: - // XXX TODO: Make error more unique? - return true, blunder.NotImplementedError - - default: - // XXX TODO: Make error more unique? - return true, blunder.NotImplementedError - } -} diff --git a/swiftclient/object.go b/swiftclient/object.go deleted file mode 100644 index 12fb5081..00000000 --- a/swiftclient/object.go +++ /dev/null @@ -1,1746 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Swift Object-specific API access implementation - -package swiftclient - -import ( - "fmt" - "runtime" - "strconv" - "sync/atomic" - "time" - - "github.com/NVIDIA/sortedmap" - "github.com/creachadair/cityhash" - - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/evtlog" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/trackedlock" -) - -func objectContentLengthWithRetry(accountName string, containerName string, objectName string) (uint64, error) { - // request is a function that, through the miracle of closure, calls - // objectContentLength() with the paramaters passed to this function, - // stashes the relevant return values into the local variables of this - // function, and then returns err and whether it is retriable to - // RequestWithRetry() - var ( - length uint64 - err error - ) - request := func() (bool, error) { - var err error - length, err = objectContentLength(accountName, containerName, objectName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl( - globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname string = fmt.Sprintf("swiftclient.objectContentLength(\"%v/%v/%v\")", - accountName, containerName, objectName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjContentLengthRetryOps, - retrySuccessCnt: &stats.SwiftObjContentLengthRetrySuccessOps, - clientRequestTime: &globals.ObjectContentLengthUsec, - clientFailureCnt: &globals.ObjectContentLengthFailure, - swiftRequestTime: &globals.SwiftObjectContentLengthUsec, - swiftRetryOps: &globals.SwiftObjectContentLengthRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return length, err -} - -func objectContentLength(accountName string, containerName string, objectName string) (length uint64, err error) { - var ( - connection *connectionStruct - contentLengthAsInt int - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "HEAD", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), nil) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.objectContentLength(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.objectContentLength(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, objectName) - return - } - evtlog.Record(evtlog.FormatObjectHead, accountName, containerName, objectName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "HEAD %s/%s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, objectName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.objectContentLength(\"%v/%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, objectName) - return - } - - contentLengthAsInt, err = parseContentLength(headers) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.objectContentLength(\"%v/%v/%v\") got parseContentLength() error", accountName, containerName, objectName) - return - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - length = uint64(contentLengthAsInt) - - stats.IncrementOperations(&stats.SwiftObjContentLengthOps) - - return -} - -func objectCopy(srcAccountName string, srcContainerName string, srcObjectName string, dstAccountName string, dstContainerName string, dstObjectName string, chunkedCopyContext ChunkedCopyContext) (err error) { - var ( - chunk []byte - chunkSize uint64 - dstChunkedPutContext ChunkedPutContext - srcObjectPosition = uint64(0) - srcObjectSize uint64 - clientReqTime time.Time - ) - - clientReqTime = time.Now() - defer func() { - elapsedUsec := time.Since(clientReqTime).Nanoseconds() / time.Microsecond.Nanoseconds() - globals.ObjectCopyUsec.Add(uint64(elapsedUsec)) - }() - - srcObjectSize, err = objectContentLengthWithRetry(srcAccountName, srcContainerName, srcObjectName) - if nil != err { - return - } - - dstChunkedPutContext, err = objectFetchChunkedPutContextWithRetry(dstAccountName, dstContainerName, dstObjectName, "") - if nil != err { - return - } - - for srcObjectPosition < srcObjectSize { - chunkSize = chunkedCopyContext.BytesRemaining(srcObjectSize - srcObjectPosition) - if 0 == chunkSize { - err = dstChunkedPutContext.Close() - return - } - - if (srcObjectPosition + chunkSize) > srcObjectSize { - chunkSize = srcObjectSize - srcObjectPosition - - chunk, err = objectTailWithRetry(srcAccountName, srcContainerName, srcObjectName, chunkSize) - } else { - chunk, err = objectGetWithRetry(srcAccountName, srcContainerName, srcObjectName, srcObjectPosition, chunkSize) - } - - srcObjectPosition += chunkSize - - err = dstChunkedPutContext.SendChunk(chunk) - if nil != err { - return - } - } - - err = dstChunkedPutContext.Close() - - stats.IncrementOperations(&stats.SwiftObjCopyOps) - - return -} - -func objectDelete(accountName string, containerName string, objectName string, operationOptions OperationOptions) (err error) { - if (operationOptions & SkipRetry) == SkipRetry { - err = objectDeleteOneTime(accountName, containerName, objectName) - } else { - // request is a function that, through the miracle of closure, calls - // objectDelete() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - request := func() (bool, error) { - var err error - err = objectDeleteOneTime(accountName, containerName, objectName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl( - globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname string = fmt.Sprintf( - "swiftclient.objectDeleteOneTime(\"%v/%v/%v\")", accountName, containerName, objectName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjDeleteRetryOps, - retrySuccessCnt: &stats.SwiftObjDeleteRetrySuccessOps, - clientRequestTime: &globals.ObjectDeleteUsec, - clientFailureCnt: &globals.ObjectDeleteFailure, - swiftRequestTime: &globals.SwiftObjectDeleteUsec, - swiftRetryOps: &globals.SwiftObjectDeleteRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - } - - return -} - -func objectDeleteOneTime(accountName string, containerName string, objectName string) (err error) { - var ( - connection *connectionStruct - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "DELETE", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), nil) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPDeleteError) - logger.WarnfWithError(err, "swiftclient.objectDelete(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPDeleteError) - logger.WarnfWithError(err, "swiftclient.objectDelete(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, objectName) - return - } - evtlog.Record(evtlog.FormatObjectDelete, accountName, containerName, objectName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "DELETE %s/%s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, objectName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.objectDelete(\"%v/%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, objectName) - return - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperations(&stats.SwiftObjDeleteOps) - - return -} - -func objectGetWithRetry(accountName string, containerName string, objectName string, - offset uint64, length uint64) ([]byte, error) { - - // request is a function that, through the miracle of closure, calls - // objectGet() with the paramaters passed to this function, stashes the - // relevant return values into the local variables of this function, and - // then returns err and whether it is retriable to RequestWithRetry() - var ( - buf []byte - err error - ) - request := func() (bool, error) { - var err error - buf, err = objectGet(accountName, containerName, objectName, offset, length) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl( - globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname string = fmt.Sprintf( - "swiftclient.objectGet(\"%v/%v/%v\")", accountName, containerName, objectName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjGetRetryOps, - retrySuccessCnt: &stats.SwiftObjGetRetrySuccessOps, - clientRequestTime: &globals.ObjectGetUsec, - clientFailureCnt: &globals.ObjectGetFailure, - swiftRequestTime: &globals.SwiftObjectGetUsec, - swiftRetryOps: &globals.SwiftObjectGetRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return buf, err -} - -func objectGet(accountName string, containerName string, objectName string, offset uint64, length uint64) (buf []byte, err error) { - var ( - connection *connectionStruct - chunk []byte - contentLength int - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - headers = make(map[string][]string) - headers["Range"] = []string{"bytes=" + strconv.FormatUint(offset, 10) + "-" + strconv.FormatUint((offset+length-1), 10)} - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "GET", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), headers) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectGet(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectGet(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, objectName) - return - } - evtlog.Record(evtlog.FormatObjectGet, accountName, containerName, objectName, offset, length, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "GET %s/%s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, objectName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.objectGet(\"%v/%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, objectName) - return - } - - if parseTransferEncoding(headers) { - buf = make([]byte, 0) - for { - chunk, err = readHTTPChunk(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectGet(\"%v/%v/%v\") got readHTTPChunk() error", accountName, containerName, objectName) - return - } - - if 0 == len(chunk) { - break - } - - buf = append(buf, chunk...) - } - } else { - contentLength, err = parseContentLength(headers) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectGet(\"%v/%v/%v\") got parseContentLength() error", accountName, containerName, objectName) - return - } - - if 0 == contentLength { - buf = make([]byte, 0) - } else { - buf, err = readBytesFromTCPConn(connection.tcpConn, contentLength) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectGet(\"%v/%v/%v\") got readBytesFromTCPConn() error", accountName, containerName, objectName) - return - } - } - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperationsAndBucketedBytes(stats.SwiftObjGet, uint64(len(buf))) - globals.ObjectGetBytes.Add(uint64(len(buf))) - return -} - -func objectHeadWithRetry(accountName string, containerName string, objectName string) (map[string][]string, error) { - // request is a function that, through the miracle of closure, calls - // objectHead() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - var ( - headers map[string][]string - err error - ) - request := func() (bool, error) { - var err error - headers, err = objectHead(accountName, containerName, objectName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl( - globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname string = fmt.Sprintf( - "swiftclient.objectHead(\"%v/%v/%v\")", accountName, containerName, objectName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjHeadRetryOps, - retrySuccessCnt: &stats.SwiftObjHeadRetrySuccessOps, - clientRequestTime: &globals.ObjectHeadUsec, - clientFailureCnt: &globals.ObjectHeadFailure, - swiftRequestTime: &globals.SwiftObjectHeadUsec, - swiftRetryOps: &globals.SwiftObjectHeadRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return headers, err -} - -func objectHead(accountName string, containerName string, objectName string) (headers map[string][]string, err error) { - var ( - connection *connectionStruct - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "HEAD", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), nil) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.objectHead(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPHeadError) - logger.WarnfWithError(err, "swiftclient.objectHead(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, objectName) - return - } - evtlog.Record(evtlog.FormatObjectHead, accountName, containerName, objectName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "HEAD %s/%s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, objectName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.objectHead(\"%v/%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, objectName) - return - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperations(&stats.SwiftObjHeadOps) - - return -} - -func objectLoadWithRetry(accountName string, containerName string, objectName string) ([]byte, error) { - // request is a function that, through the miracle of closure, calls - // objectLoad() with the paramaters passed to this function, stashes the - // relevant return values into the local variables of this function, and - // then returns err and whether it is retriable to RequestWithRetry() - var ( - buf []byte - err error - ) - request := func() (bool, error) { - var err error - buf, err = objectLoad(accountName, containerName, objectName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl( - globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname string = fmt.Sprintf( - "swiftclient.objectLoad(\"%v/%v/%v\")", accountName, containerName, objectName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjLoadRetryOps, - retrySuccessCnt: &stats.SwiftObjLoadRetrySuccessOps, - clientRequestTime: &globals.ObjectLoadUsec, - clientFailureCnt: &globals.ObjectLoadFailure, - swiftRequestTime: &globals.SwiftObjectLoadUsec, - swiftRetryOps: &globals.SwiftObjectLoadRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return buf, err -} - -func objectLoad(accountName string, containerName string, objectName string) (buf []byte, err error) { - var ( - connection *connectionStruct - chunk []byte - contentLength int - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "GET", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), nil) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectLoad(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectLoad(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, objectName) - return - } - evtlog.Record(evtlog.FormatObjectLoad, accountName, containerName, objectName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "GET %s/%s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, objectName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.objectLoad(\"%v/%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, objectName) - return - } - - if parseTransferEncoding(headers) { - buf = make([]byte, 0) - for { - chunk, err = readHTTPChunk(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectLoad(\"%v/%v/%v\") got readHTTPChunk() error", accountName, containerName, objectName) - return - } - - if 0 == len(chunk) { - break - } - - buf = append(buf, chunk...) - } - } else { - contentLength, err = parseContentLength(headers) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectLoad(\"%v/%v/%v\") got parseContentLength() error", accountName, containerName, objectName) - return - } - - if 0 == contentLength { - buf = make([]byte, 0) - } else { - buf, err = readBytesFromTCPConn(connection.tcpConn, contentLength) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectLoad(\"%v/%v/%v\") got readBytesFromTCPConn() error", accountName, containerName, objectName) - return - } - } - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperationsAndBucketedBytes(stats.SwiftObjLoad, uint64(len(buf))) - globals.ObjectLoadBytes.Add(uint64(len(buf))) - return -} - -func objectPostWithRetry(accountName string, containerName string, objectName string, requestHeaders map[string][]string) (err error) { - // request is a function that, through the miracle of closure, calls - // containerPost() with the paramaters passed to this function, stashes - // the relevant return values into the local variables of this function, - // and then returns err and whether it is retriable to RequestWithRetry() - request := func() (bool, error) { - var err error - err = objectPost(accountName, containerName, objectName, requestHeaders) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl(globals.retryLimit, globals.retryDelay, globals.retryExpBackoff) - opname string = fmt.Sprintf("swiftclient.objectPost(\"%v/%v/%v\")", accountName, containerName, objectName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjPostRetryOps, - retrySuccessCnt: &stats.SwiftObjPostRetrySuccessOps, - clientRequestTime: &globals.ObjectPostUsec, - clientFailureCnt: &globals.ObjectPostFailure, - swiftRequestTime: &globals.SwiftObjectPostUsec, - swiftRetryOps: &globals.SwiftObjectPostRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return err -} - -func objectPost(accountName string, containerName string, objectName string, requestHeaders map[string][]string) (err error) { - var ( - connection *connectionStruct - contentLength int - fsErr blunder.FsError - httpPayload string - httpStatus int - isError bool - responseHeaders map[string][]string - ) - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - requestHeaders["Content-Length"] = []string{"0"} - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "POST", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), requestHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.objectPost(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - httpStatus, responseHeaders, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.objectPost(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, objectName) - return - } - evtlog.Record(evtlog.FormatContainerPost, accountName, containerName, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, responseHeaders) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "POST %s/%s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, objectName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.objectPost(\"%v/%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, objectName) - return - } - contentLength, err = parseContentLength(responseHeaders) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.objectPost(\"%v/%v/%v\") got parseContentLength() error", accountName, containerName, objectName) - return - } - if 0 < contentLength { - _, err = readBytesFromTCPConn(connection.tcpConn, contentLength) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.objectPost(\"%v/%v/%v\") got readBytesFromTCPConn() error", accountName, containerName, objectName) - return - } - } - - releaseNonChunkedConnection(connection, parseConnection(responseHeaders)) - - stats.IncrementOperations(&stats.SwiftObjPostOps) - - return -} - -func objectReadWithRetry(accountName string, containerName string, objectName string, offset uint64, buf []byte) (uint64, error) { - // request is a function that, through the miracle of closure, calls - // objectRead() with the paramaters passed to this function, stashes the - // relevant return values into the local variables of this function, and - // then returns err and whether it is retriable to RequestWithRetry() - var ( - len uint64 - err error - ) - request := func() (bool, error) { - var err error - len, err = objectRead(accountName, containerName, objectName, offset, buf) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl( - globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname string = fmt.Sprintf( - "swiftclient.objectRead(\"%v/%v/%v\", offset=0x%016X, len(buf)=0x%016X)", accountName, containerName, objectName, offset, cap(buf)) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjReadRetryOps, - retrySuccessCnt: &stats.SwiftObjReadRetrySuccessOps, - clientRequestTime: &globals.ObjectReadUsec, - clientFailureCnt: &globals.ObjectReadFailure, - swiftRequestTime: &globals.SwiftObjectReadUsec, - swiftRetryOps: &globals.SwiftObjectReadRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return len, err -} - -func objectRead(accountName string, containerName string, objectName string, offset uint64, buf []byte) (len uint64, err error) { - var ( - capacity uint64 - chunkLen uint64 - chunkPos uint64 - connection *connectionStruct - contentLength int - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - capacity = uint64(cap(buf)) - - headers = make(map[string][]string) - headers["Range"] = []string{"bytes=" + strconv.FormatUint(offset, 10) + "-" + strconv.FormatUint((offset+capacity-1), 10)} - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "GET", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), headers) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectRead(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectRead(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, objectName) - return - } - evtlog.Record(evtlog.FormatObjectRead, accountName, containerName, objectName, offset, capacity, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "GET %s/%s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, objectName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.objectRead(\"%v/%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, objectName) - return - } - - if parseTransferEncoding(headers) { - chunkPos = 0 - for { - chunkLen, err = readHTTPChunkIntoBuf(connection.tcpConn, buf[chunkPos:]) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectRead(\"%v/%v/%v\") got readHTTPChunk() error", accountName, containerName, objectName) - return - } - - if 0 == chunkLen { - len = chunkPos - break - } - } - } else { - contentLength, err = parseContentLength(headers) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectRead(\"%v/%v/%v\") got parseContentLength() error", accountName, containerName, objectName) - return - } - - if 0 == contentLength { - len = 0 - err = nil - } else { - err = readBytesFromTCPConnIntoBuf(connection.tcpConn, buf) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectRead(\"%v/%v/%v\") got readBytesFromTCPConn() error", accountName, containerName, objectName) - return - } - len = capacity - } - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperationsAndBucketedBytes(stats.SwiftObjRead, len) - globals.ObjectReadBytes.Add(len) - return -} - -func objectTailWithRetry(accountName string, containerName string, objectName string, - length uint64) ([]byte, error) { - - // request is a function that, through the miracle of closure, calls - // objectTail() with the paramaters passed to this function, stashes the - // relevant return values into the local variables of this function, and - // then returns err and whether it is retriable to RequestWithRetry() - var ( - buf []byte - err error - ) - request := func() (bool, error) { - var err error - buf, err = objectTail(accountName, containerName, objectName, length) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl( - globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname string = fmt.Sprintf( - "swiftclient.objectTail(\"%v/%v/%v\")", accountName, containerName, objectName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjTailRetryOps, - retrySuccessCnt: &stats.SwiftObjTailRetrySuccessOps, - clientRequestTime: &globals.ObjectTailUsec, - clientFailureCnt: &globals.ObjectTailFailure, - swiftRequestTime: &globals.SwiftObjectTailUsec, - swiftRetryOps: &globals.SwiftObjectTailRetryOps, - } - ) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - return buf, err -} - -func objectTail(accountName string, containerName string, objectName string, length uint64) (buf []byte, err error) { - var ( - chunk []byte - connection *connectionStruct - contentLength int - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - headers = make(map[string][]string) - headers["Range"] = []string{"bytes=-" + strconv.FormatUint(length, 10)} - - connection, err = acquireNonChunkedConnection() - if err != nil { - // acquireNonChunkedConnection()/openConnection() logged a warning - return - } - - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "GET", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), headers) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectTail(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectTail(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", accountName, containerName, objectName) - return - } - evtlog.Record(evtlog.FormatObjectTail, accountName, containerName, objectName, length, uint32(httpStatus)) - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(connection.tcpConn, headers) - releaseNonChunkedConnection(connection, false) - err = blunder.NewError(fsErr, "GET %s/%s/%s returned HTTP StatusCode %d Payload %s", accountName, containerName, objectName, httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "swiftclient.objectTail(\"%v/%v/%v\") got readHTTPStatusAndHeaders() bad status", accountName, containerName, objectName) - return - } - - if parseTransferEncoding(headers) { - buf = make([]byte, 0) - for { - chunk, err = readHTTPChunk(connection.tcpConn) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectTail(\"%v/%v/%v\") got readHTTPChunk() error", accountName, containerName, objectName) - return - } - - if 0 == len(chunk) { - break - } - - buf = append(buf, chunk...) - } - } else { - contentLength, err = parseContentLength(headers) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectTail(\"%v/%v/%v\") got parseContentLength() error", accountName, containerName, objectName) - return - } - - if 0 == contentLength { - buf = make([]byte, 0) - } else { - buf, err = readBytesFromTCPConn(connection.tcpConn, contentLength) - if nil != err { - releaseNonChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPGetError) - logger.WarnfWithError(err, "swiftclient.objectTail(\"%v/%v/%v\") got readBytesFromTCPConn() error", accountName, containerName, objectName) - return - } - } - } - - releaseNonChunkedConnection(connection, parseConnection(headers)) - - stats.IncrementOperationsAndBytes(stats.SwiftObjTail, uint64(len(buf))) - globals.ObjectTailBytes.Add(uint64(len(buf))) - return -} - -// Checksum and related info for one chunked put chunk -type chunkedPutChunkInfo struct { - chunkBuf []byte - chunkCksum uint64 -} - -type chunkedPutContextStruct struct { - trackedlock.Mutex - accountName string - containerName string - objectName string - fetchTime time.Time - sendTime time.Time - active bool - err error - fatal bool - useReserveForVolumeName string - connection *connectionStruct - stillOpen bool - bytesPut uint64 - bytesPutTree sortedmap.LLRBTree // Key == objectOffset of start of chunk in object - // Value == []byte of bytes sent to SendChunk() - chunkInfoArray []chunkedPutChunkInfo -} - -func (chunkedPutContext *chunkedPutContextStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { - keyAsUint64, ok := key.(uint64) - if !ok { - err = fmt.Errorf("swiftclient.chunkedPutContext.DumpKey() could not parse key as a uint64") - return - } - - keyAsString = fmt.Sprintf("0x%016X", keyAsUint64) - - err = nil - return -} - -func (chunkedPutContext *chunkedPutContextStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { - valueAsByteSlice, ok := value.([]byte) - if !ok { - err = fmt.Errorf("swiftclient.chunkedPutContext.DumpValue() could not parse value as a []byte") - return - } - - valueAsString = string(valueAsByteSlice[:]) - - err = nil - return -} - -func objectFetchChunkedPutContextWithRetry(accountName string, containerName string, objectName string, useReserveForVolumeName string) (*chunkedPutContextStruct, error) { - // request is a function that, through the miracle of closure, calls - // objectFetchChunkedPutContext() with the paramaters passed to this - // function, stashes the relevant return values into the local variables of - // this function, and then returns err and whether it is retriable to - // RequestWithRetry() - var ( - chunkedPutContext *chunkedPutContextStruct - ) - request := func() (bool, error) { - var err error - chunkedPutContext, err = objectFetchChunkedPutContext(accountName, containerName, objectName, useReserveForVolumeName) - return true, err - } - - var ( - retryObj *RetryCtrl = NewRetryCtrl( - globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname string = fmt.Sprintf( - "swiftclient.objectFetchChunkedPutContext(\"%v/%v/%v\")", accountName, containerName, objectName) - - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjFetchPutCtxtRetryOps, - retrySuccessCnt: &stats.SwiftObjFetchPutCtxtRetrySuccessOps, - clientRequestTime: &globals.ObjectPutCtxtFetchUsec, - clientFailureCnt: &globals.ObjectPutCtxtFetchFailure, - swiftRequestTime: &globals.SwiftObjectPutCtxtFetchUsec, - swiftRetryOps: &globals.SwiftObjectPutCtxtFetchRetryOps, - } - ) - err := retryObj.RequestWithRetry(request, &opname, &statnm) - return chunkedPutContext, err -} - -// used during testing for error injection -var objectFetchChunkedPutContextCnt uint64 - -func objectFetchChunkedPutContext(accountName string, containerName string, objectName string, useReserveForVolumeName string) (chunkedPutContext *chunkedPutContextStruct, err error) { - var ( - connection *connectionStruct - headers map[string][]string - ) - - evtlog.Record(evtlog.FormatObjectPutChunkedStart, accountName, containerName, objectName) - - connection, err = acquireChunkedConnection(useReserveForVolumeName) - if err != nil { - // acquireChunkedConnection()/openConnection() logged a warning - return - } - - headers = make(map[string][]string) - headers["Transfer-Encoding"] = []string{"chunked"} - - // check for chaos error generation (testing only) - if atomic.LoadUint64(&globals.chaosFetchChunkedPutFailureRate) > 0 && - (atomic.AddUint64(&objectFetchChunkedPutContextCnt, 1)% - atomic.LoadUint64(&globals.chaosFetchChunkedPutFailureRate) == 0) { - err = fmt.Errorf("swiftclient.objectFetchChunkedPutContext returning simulated error") - } else { - err = writeHTTPRequestLineAndHeaders(connection.tcpConn, "PUT", "/"+swiftVersion+"/"+pathEscape(accountName, containerName, objectName), headers) - } - if nil != err { - releaseChunkedConnection(connection, false) - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.objectFetchChunkedPutContext(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", accountName, containerName, objectName) - return - } - - chunkedPutContext = &chunkedPutContextStruct{ - accountName: accountName, - containerName: containerName, - objectName: objectName, - err: nil, - active: true, - useReserveForVolumeName: useReserveForVolumeName, - connection: connection, - bytesPut: 0, - } - - chunkedPutContext.bytesPutTree = sortedmap.NewLLRBTree(sortedmap.CompareUint64, chunkedPutContext) - chunkedPutContext.chunkInfoArray = make([]chunkedPutChunkInfo, 0) - - stats.IncrementOperations(&stats.SwiftObjPutCtxFetchOps) - - // record the time that ObjectFetchChunkedPutContext() returns - chunkedPutContext.fetchTime = time.Now() - return -} - -func (chunkedPutContext *chunkedPutContextStruct) Active() (active bool) { - active = chunkedPutContext.active - - stats.IncrementOperations(&stats.SwiftObjPutCtxActiveOps) - - return -} - -func (chunkedPutContext *chunkedPutContextStruct) BytesPut() (bytesPut uint64, err error) { - chunkedPutContext.Lock() - bytesPut = chunkedPutContext.bytesPut - chunkedPutContext.Unlock() - - stats.IncrementOperations(&stats.SwiftObjPutCtxBytesPutOps) - globals.ObjectPutCtxtBytesPut.Add(1) - - err = nil - return -} - -func (chunkedPutContext *chunkedPutContextStruct) Close() (err error) { - - // track the amount of time the client spent holding the connection -- - // this includes time spent in SendChunk(), but not time spent in - // ObjectFetchChunkedPutContext() and not time that will be spent here - // in Close() - fetchToCloseUsec := time.Since(chunkedPutContext.fetchTime).Nanoseconds() / - time.Microsecond.Nanoseconds() - globals.ObjectPutCtxtFetchToCloseUsec.Add(uint64(fetchToCloseUsec)) - - chunkedPutContext.validateChunkChecksums() - - // request is a function that, through the miracle of closure, calls - // retry() and Close() with the paramaters passed to this function and - // stashes the return values into the local variables of this function - // and then returns the error and whether it is retriable to its caller, - // RequestWithRetry() - var firstAttempt bool = true - request := func() (bool, error) { - var err error - - // if this not the first attempt, retry() will redo all of the - // setup and SendChunk()s that were done before the first attempt - if !firstAttempt { - err = chunkedPutContext.retry() - if err != nil { - // closeHelper() will clean up the error, but - // only if it needs to know there was one - chunkedPutContext.err = err - } - } - firstAttempt = false - - err = chunkedPutContext.closeHelper() - - // fatal errors cannot be retried because we don't have the data - // that needs to be resent available (it could not be stored) - retriable := !chunkedPutContext.fatal - return retriable, err - } - - var ( - retryObj *RetryCtrl - opname string - statnm requestStatistics = requestStatistics{ - retryCnt: &stats.SwiftObjPutCtxtCloseRetryOps, - retrySuccessCnt: &stats.SwiftObjPutCtxtCloseRetrySuccessOps, - clientRequestTime: &globals.ObjectPutCtxtCloseUsec, - clientFailureCnt: &globals.ObjectPutCtxtCloseFailure, - swiftRequestTime: &globals.SwiftObjectPutCtxtCloseUsec, - swiftRetryOps: &globals.SwiftObjectPutCtxtCloseRetryOps, - } - ) - retryObj = NewRetryCtrl(globals.retryLimitObject, globals.retryDelayObject, globals.retryExpBackoffObject) - opname = fmt.Sprintf("swiftclient.chunkedPutContext.Close(\"%v/%v/%v\")", - chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - err = retryObj.RequestWithRetry(request, &opname, &statnm) - - // RequestWithRetry() tried the operation one or more times until it - // either: succeeded; failed with a non-retriable (fatal) error; or hit - // the retry limit. Regardless, its time to release the connection we - // got at the very start or acquired during retry (if any). - if err != nil { - if nil != chunkedPutContext.connection { - releaseChunkedConnection(chunkedPutContext.connection, false) - } - } else { - releaseChunkedConnection(chunkedPutContext.connection, chunkedPutContext.stillOpen) - } - chunkedPutContext.connection = nil - - return err -} - -func (chunkedPutContext *chunkedPutContextStruct) closeHelper() (err error) { - var ( - fsErr blunder.FsError - headers map[string][]string - httpPayload string - httpStatus int - isError bool - ) - - chunkedPutContext.Lock() - defer chunkedPutContext.Unlock() - defer chunkedPutContext.validateChunkChecksums() - - if !chunkedPutContext.active { - err = blunder.NewError(blunder.BadHTTPPutError, "called while inactive") - logger.PanicfWithError(err, "swiftclient.chunkedPutContext.closeHelper(\"%v/%v/%v\") called while inactive", - chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - chunkedPutContext.active = false - - // if an error occurred earlier there's no point in trying to send the closing chunk - if chunkedPutContext.err != nil { - err = chunkedPutContext.err - return - } - - err = writeHTTPPutChunk(chunkedPutContext.connection.tcpConn, []byte{}) - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.chunkedPutContext.closeHelper(\"%v/%v/%v\") got writeHTTPPutChunk() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - httpStatus, headers, err = readHTTPStatusAndHeaders(chunkedPutContext.connection.tcpConn) - - atomic.AddUint64(&chunkedPutCloseCnt, 1) - if atomic.LoadUint64(&globals.chaosCloseChunkFailureRate) > 0 && - atomic.LoadUint64(&chunkedPutCloseCnt)%atomic.LoadUint64(&globals.chaosCloseChunkFailureRate) == 0 { - err = fmt.Errorf("chunkedPutContextStruct.closeHelper(): readHTTPStatusAndHeaders() simulated error") - } - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.chunkedPutContext.closeHelper(\"%v/%v/%v\") got readHTTPStatusAndHeaders() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - evtlog.Record(evtlog.FormatObjectPutChunkedEnd, chunkedPutContext.accountName, - chunkedPutContext.containerName, chunkedPutContext.objectName, - chunkedPutContext.bytesPut, uint32(httpStatus)) - - isError, fsErr = httpStatusIsError(httpStatus) - if isError { - httpPayload, _ = readHTTPPayloadAsString(chunkedPutContext.connection.tcpConn, headers) - err = blunder.NewError(fsErr, - "chunkedPutContext.closeHelper(): PUT '%s/%s/%s' returned HTTP StatusCode %d Payload %s", - chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName, - httpStatus, httpPayload) - err = blunder.AddHTTPCode(err, httpStatus) - logger.WarnfWithError(err, "chunkedPutContext readHTTPStatusAndHeaders() returned error") - return - } - // even though the PUT succeeded the server may have decided to close - // the connection, something releaseChunkedConnection() needs to know - chunkedPutContext.stillOpen = parseConnection(headers) - - stats.IncrementOperations(&stats.SwiftObjPutCtxCloseOps) - globals.ObjectPutCtxtBytes.Add(chunkedPutContext.bytesPut) - - return -} - -func (chunkedPutContext *chunkedPutContextStruct) Read(offset uint64, length uint64) (buf []byte, err error) { - var ( - chunkBufAsByteSlice []byte - chunkBufAsValue sortedmap.Value - chunkOffsetAsKey sortedmap.Key - chunkOffsetAsUint64 uint64 - found bool - chunkIndex int - ok bool - readLimitOffset uint64 - ) - - readLimitOffset = offset + length - - chunkedPutContext.Lock() - - chunkedPutContext.validateChunkChecksums() - - if readLimitOffset > chunkedPutContext.bytesPut { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() called for invalid range") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") called for invalid range", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - chunkIndex, found, err = chunkedPutContext.bytesPutTree.BisectLeft(offset) - if nil != err { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.BisectLeft() failed: %v", err) - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got bytesPutTree.BisectLeft() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - if !found && (0 > chunkIndex) { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.BisectLeft() returned unexpected index/found") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") attempt to read past end", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - chunkOffsetAsKey, chunkBufAsValue, ok, err = chunkedPutContext.bytesPutTree.GetByIndex(chunkIndex) - if nil != err { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.GetByIndex() failed: %v", err) - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got initial bytesPutTree.GetByIndex() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - if !ok { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.GetByIndex() returned ok == false") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got initial bytesPutTree.GetByIndex() !ok", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - chunkOffsetAsUint64, ok = chunkOffsetAsKey.(uint64) - if !ok { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.GetByIndex() returned non-uint64 Key") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got initial bytesPutTree.GetByIndex() malformed Key", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - chunkBufAsByteSlice, ok = chunkBufAsValue.([]byte) - if !ok { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.GetByIndex() returned non-[]byte Value") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got initial bytesPutTree.GetByIndex() malformed Value", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - if (readLimitOffset - chunkOffsetAsUint64) <= uint64(len(chunkBufAsByteSlice)) { - // Trivial case: offset:readLimitOffset fits entirely within chunkBufAsByteSlice - - buf = chunkBufAsByteSlice[(offset - chunkOffsetAsUint64):(readLimitOffset - chunkOffsetAsUint64)] - } else { - // Complex case: offset:readLimit extends beyond end of chunkBufAsByteSlice - - buf = make([]byte, 0, length) - buf = append(buf, chunkBufAsByteSlice[(offset-chunkOffsetAsUint64):]...) - - for uint64(len(buf)) < length { - chunkIndex++ - - chunkOffsetAsKey, chunkBufAsValue, ok, err = chunkedPutContext.bytesPutTree.GetByIndex(chunkIndex) - if nil != err { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.GetByIndex() failed: %v", err) - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got next bytesPutTree.GetByIndex() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - if !ok { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.GetByIndex() returned ok == false") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got next bytesPutTree.GetByIndex() !ok", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - chunkOffsetAsUint64, ok = chunkOffsetAsKey.(uint64) - if !ok { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.GetByIndex() returned non-uint64 Key") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got next bytesPutTree.GetByIndex() malformed Key", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - chunkBufAsByteSlice, ok = chunkBufAsValue.([]byte) - if !ok { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPGetError, "swiftclient.chunkedPutContext.Read() bytesPutTree.GetByIndex() returned non-[]byte Value") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.Read(\"%v/%v/%v\") got next bytesPutTree.GetByIndex() malformed Key", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - if (readLimitOffset - chunkOffsetAsUint64) < uint64(len(chunkBufAsByteSlice)) { - buf = append(buf, chunkBufAsByteSlice[:(readLimitOffset-chunkOffsetAsUint64)]...) - } else { - buf = append(buf, chunkBufAsByteSlice...) - } - } - } - chunkedPutContext.Unlock() - - stats.IncrementOperationsAndBucketedBytes(stats.SwiftObjPutCtxRead, length) - globals.ObjectPutCtxtReadBytes.Add(length) - - err = nil - return -} - -func (chunkedPutContext *chunkedPutContextStruct) retry() (err error) { - var ( - chunkBufAsByteSlice []byte - chunkBufAsValue sortedmap.Value - chunkIndex int - headers map[string][]string - ok bool - ) - - chunkedPutContext.Lock() - - if chunkedPutContext.active { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPPutError, "called while active") - logger.PanicfWithError(err, "swiftclient.chunkedPutContext.retry(\"%v/%v/%v\") called while active", - chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - // clear error from the previous attempt and make active - chunkedPutContext.err = nil - chunkedPutContext.active = true - - // try to re-open chunked put connection - if nil != chunkedPutContext.connection { - releaseChunkedConnection(chunkedPutContext.connection, false) - } - chunkedPutContext.connection, err = acquireChunkedConnection(chunkedPutContext.useReserveForVolumeName) - if nil != err { - chunkedPutContext.Unlock() - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.chunkedPutContext.retry(\"%v/%v/%v\") acquireChunkedConnection() failed", - chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - headers = make(map[string][]string) - headers["Transfer-Encoding"] = []string{"chunked"} - - err = writeHTTPRequestLineAndHeaders(chunkedPutContext.connection.tcpConn, "PUT", "/"+swiftVersion+"/"+pathEscape(chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName), headers) - if nil != err { - chunkedPutContext.Unlock() - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, "swiftclient.chunkedPutContext.retry(\"%v/%v/%v\") got writeHTTPRequestLineAndHeaders() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - chunkIndex = 0 - - chunkedPutContext.validateChunkChecksums() - for { - _, chunkBufAsValue, ok, err = chunkedPutContext.bytesPutTree.GetByIndex(chunkIndex) - if nil != err { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPPutError, "bytesPutTree.GetByIndex() failed: %v", err) - logger.WarnfWithError(err, "swiftclient.chunkedPutContext.retry(\"%v/%v/%v\") got bytesPutTree.GetByIndex() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - if !ok { - // We've reached the end of bytesPutTree... so we (presumably) have now sent bytesPut bytes in total - break - } - - // Simply (re)send the chunk (assuming it is a []byte) - - chunkBufAsByteSlice, ok = chunkBufAsValue.([]byte) - if !ok { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPPutError, "bytesPutTree.GetByIndex() returned non-[]byte Value") - logger.WarnfWithError(err, "swiftclient.chunkedPutContext.retry(\"%v/%v/%v\") got bytesPutTree.GetByIndex(() malformed Value", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - // check for chaos error generation (testing only) - atomic.AddUint64(&chunkedPutSendRetryCnt, 1) - if atomic.LoadUint64(&globals.chaosSendChunkFailureRate) > 0 && - atomic.LoadUint64(&chunkedPutSendRetryCnt)%atomic.LoadUint64(&globals.chaosSendChunkFailureRate) == 0 { - - nChunk, _ := chunkedPutContext.bytesPutTree.Len() - err = fmt.Errorf("chunkedPutContextStruct.retry(): "+ - "writeHTTPPutChunk() simulated error with %d chunks", nChunk) - } else { - err = writeHTTPPutChunk(chunkedPutContext.connection.tcpConn, chunkBufAsByteSlice) - } - if nil != err { - chunkedPutContext.Unlock() - err = blunder.NewError(blunder.BadHTTPPutError, "writeHTTPPutChunk() failed: %v", err) - logger.WarnfWithError(err, "swiftclient.chunkedPutContext.retry(\"%v/%v/%v\") got writeHTTPPutChunk() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - return - } - - // See if there is (was) another chunk - - chunkIndex++ - } - chunkedPutContext.Unlock() - - stats.IncrementOperations(&stats.SwiftObjPutCtxRetryOps) - - return -} - -// If checksums are enabled validate the checksums for each chunk. -// -// The chunked put context is assumed to be locked (not actively changing), and -// is returned still locked. -// -func (chunkedPutContext *chunkedPutContextStruct) validateChunkChecksums() { - if !globals.checksumChunkedPutChunks { - return - } - if chunkedPutContext.bytesPut == 0 { - return - } - - var ( - chunkBufAsValue sortedmap.Value - chunkBuf []byte - chunkOffsetAsKey sortedmap.Key - chunkOffset uint64 - chunkIndex int - ok bool - panicErr error - offset uint64 - ) - - // print as much potentially useful information as we can before panic'ing - panicErr = nil - offset = 0 - for chunkIndex = 0; true; chunkIndex++ { - var err error - - chunkOffsetAsKey, chunkBufAsValue, ok, err = chunkedPutContext.bytesPutTree.GetByIndex(chunkIndex) - if err != nil { - panicErr = fmt.Errorf("bytesPutTree.GetByIndex(%d) offset %d at failed: %v", - chunkIndex, offset, err) - break - } - if !ok { - // We've reached the end of bytesPutTree... so we - // (presumably) have now check all the chunks - break - } - - chunkOffset, ok = chunkOffsetAsKey.(uint64) - if !ok { - panicErr = fmt.Errorf( - "bytesPutTree.GetByIndex(%d) offset %d returned incorrect key type: %T", - chunkIndex, offset, chunkOffsetAsKey) - break - } - - chunkBuf, ok = chunkBufAsValue.([]byte) - if !ok { - panicErr = fmt.Errorf( - "bytesPutTree.GetByIndex(%d) offset %d returned incorrect value type: %T", - chunkIndex, offset, chunkOffsetAsKey) - break - } - - // verify slice still points to the same memory and has correct checksum - if &chunkBuf[0] != &chunkedPutContext.chunkInfoArray[chunkIndex].chunkBuf[0] { - err = fmt.Errorf("chunkInfoArray(%d) offset %d returned buf %p != saved buf %p", - chunkIndex, offset, &chunkBuf[0], - &chunkedPutContext.chunkInfoArray[chunkIndex].chunkBuf[0]) - - } else if cityhash.Hash64(chunkBuf) != chunkedPutContext.chunkInfoArray[chunkIndex].chunkCksum { - err = fmt.Errorf( - "chunkInfoArray(%d) offset %d computed chunk checksum 0x%16x != saved checksum 0x%16x", - chunkIndex, offset, cityhash.Hash64(chunkBuf), - chunkedPutContext.chunkInfoArray[chunkIndex].chunkCksum) - } else if offset != chunkOffset { - err = fmt.Errorf("chunkInfoArray(%d) offset %d returned incorrect offset key %d", - chunkIndex, offset, chunkOffset) - } - - if err != nil { - logger.ErrorfWithError(err, - "validateChunkChecksums('%v/%v/%v') %d chunks %d bytes invalid chunk", - chunkedPutContext.accountName, chunkedPutContext.containerName, - chunkedPutContext.objectName, - len(chunkedPutContext.chunkInfoArray), chunkedPutContext.bytesPut) - if panicErr == nil { - panicErr = err - } - } - - offset += uint64(len(chunkBuf)) - } - - if panicErr == nil { - return - } - - // log time stamps for chunked put (and last checksum validation) - logger.Errorf( - "validateChunkChecksums('%v/%v/%v') fetched %f sec ago last SendChunk() %f sec ago", - chunkedPutContext.accountName, chunkedPutContext.containerName, - chunkedPutContext.objectName, - time.Since(chunkedPutContext.fetchTime).Seconds(), - time.Since(chunkedPutContext.sendTime).Seconds()) - - // there was an error; dump stack, other relevant information, and panic - stackBuf := make([]byte, 64*1024) - stackBuf = stackBuf[:runtime.Stack(stackBuf, false)] - logger.Errorf( - "validateChunkChecksums('%v/%v/%v') chunked buffer error: stack trace:\n%s", - chunkedPutContext.accountName, chunkedPutContext.containerName, - chunkedPutContext.objectName, stackBuf) - - logger.PanicfWithError(panicErr, - "validateChunkChecksums('%v/%v/%v') %d chunks %d chunk bytes chunked buffer error", - chunkedPutContext.accountName, chunkedPutContext.containerName, - chunkedPutContext.objectName, - len(chunkedPutContext.chunkInfoArray), chunkedPutContext.bytesPut) - panic(panicErr) -} - -// used during testing for error injection -var ( - chunkedPutSendCnt uint64 - chunkedPutSendRetryCnt uint64 - chunkedPutCloseCnt uint64 -) - -// SendChunk() tries to send the chunk of data to the Swift server and deal with -// any errors that occur. There is a retry mechanism in Close() that will -// attempt to resend the data if the first attempt here was not successful. -// -// For "normal" error cases SendChunk() will returns nil (success) instead of -// the error and stash the data to be sent away so that retry can attempt to -// send it. Whence Close() is called, it notices the pending error in -// ChunkedPutContext.err and retries the entire operation. If the retry works, -// it returns success. Otherwise, it returns the final error it encountered. - -// There are some cases where SendChunk() cannot stash away the data to be sent -// due to logic errors in the program or corruption of in memory data -// structures. These should probably just be dealt with by panic'ing, but -// instead ChunkedPutContext treats this as a "fatal" error, which it returns to -// the caller of SendChunk(). If Close() is called later, it does not attempt -// to retry but instead returns the same error. Just in case Close() is not -// called (current code does not call Close() after SendChunk() returns an -// error, SendChunk() also cleans up the TCP connection. -// -func (chunkedPutContext *chunkedPutContextStruct) SendChunk(buf []byte) (err error) { - - clientReqTime := time.Now() - - chunkedPutContext.Lock() - - // record the start of the most recent (modulo lock scheduling) send request - chunkedPutContext.sendTime = clientReqTime - - if !chunkedPutContext.active { - err = blunder.NewError(blunder.BadHTTPPutError, "called while inactive") - logger.PanicfWithError(err, "swiftclient.chunkedPutContext.SendChunk(\"%v/%v/%v\") logic error", - chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - chunkedPutContext.err = err - chunkedPutContext.fatal = true - - // connection should already be nil - chunkedPutContext.connection = nil - chunkedPutContext.Unlock() - return - } - - if 0 == len(buf) { - err = blunder.NewError(blunder.BadHTTPPutError, "called with zero-length buf") - logger.PanicfWithError(err, "swiftclient.chunkedPutContext.SendChunk(\"%v/%v/%v\") logic error", - chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - chunkedPutContext.err = err - chunkedPutContext.fatal = true - releaseChunkedConnection(chunkedPutContext.connection, false) - chunkedPutContext.connection = nil - chunkedPutContext.Unlock() - return - } - - // validate current checksums - chunkedPutContext.validateChunkChecksums() - - // if checksum verification is enabled, compute the checksum - if globals.checksumChunkedPutChunks { - var chunkInfo chunkedPutChunkInfo - - chunkInfo.chunkBuf = buf - chunkInfo.chunkCksum = cityhash.Hash64(buf) - chunkedPutContext.chunkInfoArray = append(chunkedPutContext.chunkInfoArray, chunkInfo) - } - - ok, err := chunkedPutContext.bytesPutTree.Put(chunkedPutContext.bytesPut, buf) - if nil != err { - err = blunder.NewError(blunder.BadHTTPPutError, "attempt to append chunk to LLRB Tree failed: %v", err) - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.SendChunk(\"%v/%v/%v\") got bytesPutTree.Put() error", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - chunkedPutContext.err = err - chunkedPutContext.fatal = true - releaseChunkedConnection(chunkedPutContext.connection, false) - chunkedPutContext.connection = nil - chunkedPutContext.Unlock() - return - } - if !ok { - err = blunder.NewError(blunder.BadHTTPPutError, "attempt to append chunk to LLRB Tree returned ok == false") - logger.ErrorfWithError(err, "swiftclient.chunkedPutContext.SendChunk(\"%v/%v/%v\") got bytesPutTree.Put() !ok", chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - chunkedPutContext.err = err - chunkedPutContext.fatal = true - releaseChunkedConnection(chunkedPutContext.connection, false) - chunkedPutContext.connection = nil - chunkedPutContext.Unlock() - return - } - - chunkedPutContext.bytesPut += uint64(len(buf)) - - // The prior errors are logical/programmatic errors that cannot be fixed - // by a retry so let them return with the error and let the caller abort - // (Close() will not be called, so retry() will not be called). - // - // However, if writeHTTPPutChunk() fails that is probably due to a - // problem with the TCP connection or storage which may be cured by a - // retry. Therefore, if the call fails stash the error in - // chunkedPutContext and return success to the caller so it will feed us - // the rest of the chunks. We need those chunks for the retry! - // - // If an error has already been seen, stash the data for use by retry - // and return success. - if chunkedPutContext.err != nil { - chunkedPutContext.Unlock() - return nil - } - - // check for chaos error generation (testing only) - atomic.AddUint64(&chunkedPutSendCnt, 1) - if atomic.LoadUint64(&globals.chaosSendChunkFailureRate) > 0 && - atomic.LoadUint64(&chunkedPutSendCnt)%atomic.LoadUint64(&globals.chaosSendChunkFailureRate) == 0 { - err = fmt.Errorf("chunkedPutContextStruct.SendChunk(): writeHTTPPutChunk() simulated error") - } else { - err = writeHTTPPutChunk(chunkedPutContext.connection.tcpConn, buf) - } - if nil != err { - err = blunder.AddError(err, blunder.BadHTTPPutError) - logger.WarnfWithError(err, - "swiftclient.chunkedPutContext.SendChunk(\"%v/%v/%v\") got writeHTTPPutChunk() error", - chunkedPutContext.accountName, chunkedPutContext.containerName, chunkedPutContext.objectName) - chunkedPutContext.err = err - chunkedPutContext.Unlock() - err = nil - return - } - - chunkedPutContext.Unlock() - - stats.IncrementOperationsAndBucketedBytes(stats.SwiftObjPutCtxSendChunk, uint64(len(buf))) - elapsedUsec := time.Since(clientReqTime).Nanoseconds() / time.Microsecond.Nanoseconds() - globals.ObjectPutCtxtSendChunkUsec.Add(uint64(elapsedUsec)) - globals.ObjectPutCtxtSendChunkBytes.Add(uint64(len(buf))) - - return -} diff --git a/swiftclient/retry.go b/swiftclient/retry.go deleted file mode 100644 index f6defc4f..00000000 --- a/swiftclient/retry.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Swift Object-specific API access implementation - -package swiftclient - -import ( - "fmt" - "time" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" -) - -// Swift requests should be retried if they fail with a retriable error. -// RetryCtrl is used to control the retry process, including exponential backoff -// on subsequent replay attempts. lastReq tracks the time that the last request -// was made so that time consumed by the request can be subtracted from the -// backoff amount (if a request takes 30 sec to timeout and the initial delay -// is 10 sec, we don't want 40 sec between requests). -// -type RetryCtrl struct { - attemptMax uint // maximum attempts - attemptCnt uint // number of attempts - delay time.Duration // backoff amount (grows each attempt) - expBackoff float64 // factor to increase delay by - clientReqTime time.Time // client (of swiftclient) request start time - swiftReqTime time.Time // most recent Swift request start time -} - -type requestStatistics struct { - retryCnt *string // increment stat for each operation that is retried (not each retry) - retrySuccessCnt *string // increment this for each operation where retry fixed the problem - clientRequestTime *bucketstats.BucketLog2Round - clientFailureCnt *bucketstats.Total - swiftRequestTime *bucketstats.BucketLog2Round - swiftRetryOps *bucketstats.Average -} - -func NewRetryCtrl(maxAttempt uint16, delay time.Duration, expBackoff float64) *RetryCtrl { - var ctrl = RetryCtrl{attemptCnt: 0, attemptMax: uint(maxAttempt), delay: delay, expBackoff: expBackoff} - ctrl.clientReqTime = time.Now() - ctrl.swiftReqTime = ctrl.clientReqTime - - return &ctrl -} - -// Wait until this.delay has elapsed since the last request started and then -// update the delay with the exponential backoff and record when the next -// request was started -// -func (this *RetryCtrl) RetryWait() { - var delay time.Duration = time.Now().Sub(this.swiftReqTime) - - if this.delay > delay { - time.Sleep(this.delay - delay) - } - this.delay = time.Duration(float64(this.delay) * this.expBackoff) - this.swiftReqTime = time.Now() - return -} - -// Perform a request until it suceeds, it fails with an unretriable error, or we -// hit the maximum retries. doRequest() will issue the request and return both -// an error indication and a boolean indicating whether the error is retriable -// or not (if there is an error). -// -// if a request fails, even if this.attemptMax == 0 (retry disabled) this will -// still log an Error message indicating RequestWithRetry() failed along with -// the operation identifier (name and paramaters) -// -func (this *RetryCtrl) RequestWithRetry(doRequest func() (bool, error), opid *string, reqstat *requestStatistics) (err error) { - var ( - lastErr error - retriable bool - elapsed time.Duration - ) - - // start performing the request now - this.attemptCnt = 1 - retriable, lastErr = doRequest() - - elapsed = time.Since(this.clientReqTime) - reqstat.swiftRequestTime.Add(uint64(elapsed.Nanoseconds() / time.Microsecond.Nanoseconds())) - - if lastErr == nil { - reqstat.clientRequestTime.Add(uint64(elapsed.Nanoseconds() / time.Microsecond.Nanoseconds())) - - return nil - } - - // doRequest(), above, counts as the first attempt though its not a - // retry, which is why this loop goes to <= this.attemptMax (consider - // this.attemptMax == 0 (retries disabled) and this.attemptMax == 1 - // cases). - // - // if retries are enabled but the first failure is not retriable then - // increment reqstat.retryCnt even though no retry is performed because - // users are likely to assume that: - // reqstat.retryCnt - reqstat.retrySuccess == count_of_failures - if this.attemptMax != 0 { - stats.IncrementOperations(reqstat.retryCnt) - } - for retriable && this.attemptCnt <= this.attemptMax { - this.RetryWait() - this.attemptCnt++ - retriable, lastErr = doRequest() - - // RetryWait() set this.swiftReqTime to Now() - elapsed = time.Since(this.swiftReqTime) - reqstat.swiftRequestTime.Add(uint64(elapsed.Nanoseconds() / time.Microsecond.Nanoseconds())) - - if lastErr == nil { - elapsed = time.Since(this.clientReqTime) - reqstat.clientRequestTime.Add(uint64(elapsed.Nanoseconds() / time.Microsecond.Nanoseconds())) - reqstat.swiftRetryOps.Add(1) - stats.IncrementOperations(reqstat.retrySuccessCnt) - - logger.Infof("retry.RequestWithRetry(): %s succeeded after %d attempts in %4.3f sec", - *opid, this.attemptCnt, elapsed.Seconds()) - return nil - } - - // bump retry count (as failure) - reqstat.swiftRetryOps.Add(0) - } - // lasterr != nil - - // the client request failed (and is finished) - elapsed = time.Since(this.clientReqTime) - reqstat.clientRequestTime.Add(uint64(elapsed.Nanoseconds() / time.Microsecond.Nanoseconds())) - reqstat.clientFailureCnt.Add(1) - - var errstring string - if !retriable { - errstring = fmt.Sprintf( - "retry.RequestWithRetry(): %s failed after %d attempts in %4.3f sec with unretriable error", - *opid, this.attemptCnt, elapsed.Seconds()) - } else { - errstring = fmt.Sprintf( - "retry.RequestWithRetry(): %s failed after %d attempts in %4.3f sec with retriable error", - *opid, this.attemptCnt, elapsed.Seconds()) - } - logger.ErrorWithError(lastErr, errstring) - return lastErr -} diff --git a/swiftclient/utils.go b/swiftclient/utils.go deleted file mode 100644 index ec344fb4..00000000 --- a/swiftclient/utils.go +++ /dev/null @@ -1,1131 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package swiftclient - -import ( - "bytes" - "container/list" - "fmt" - "net" - "net/url" - "strconv" - "strings" - "sync" - "sync/atomic" - - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/stats" - "github.com/NVIDIA/proxyfs/version" -) - -const swiftVersion = "v1" - -func drainConnections() { - var ( - connection *connectionStruct - reservedChunkedConnectionCopy map[string]*connectionStruct - volumeName string - ) - - globals.reservedChunkedConnectionMutex.Lock() - reservedChunkedConnectionCopy = make(map[string]*connectionStruct) - for volumeName, connection = range globals.reservedChunkedConnection { - reservedChunkedConnectionCopy[volumeName] = connection - } - for volumeName, connection = range reservedChunkedConnectionCopy { - _ = connection.tcpConn.Close() - delete(globals.reservedChunkedConnection, volumeName) - } - globals.reservedChunkedConnectionMutex.Unlock() - - globals.chunkedConnectionPool.Lock() - // The following should not be necessary so, as such, will remain commented out - /* - for 0 < globals.chunkedConnectionPool.poolInUse { - globals.chunkedConnectionPool.Unlock() - time.Sleep(100 * time.Millisecond) - globals.chunkedConnectionPool.Lock() - } - */ - for 0 < globals.chunkedConnectionPool.lifoIndex { - globals.chunkedConnectionPool.lifoIndex-- - connection = globals.chunkedConnectionPool.lifoOfAvailableConnections[globals.chunkedConnectionPool.lifoIndex] - globals.chunkedConnectionPool.lifoOfAvailableConnections[globals.chunkedConnectionPool.lifoIndex] = nil - _ = connection.tcpConn.Close() - } - globals.chunkedConnectionPool.Unlock() - - globals.nonChunkedConnectionPool.Lock() - // The following should not be necessary so, as such, will remain commented out - /* - for 0 < globals.nonChunkedConnectionPool.poolInUse { - globals.nonChunkedConnectionPool.Unlock() - time.Sleep(100 * time.Millisecond) - globals.nonChunkedConnectionPool.Lock() - } - */ - for 0 < globals.nonChunkedConnectionPool.lifoIndex { - globals.nonChunkedConnectionPool.lifoIndex-- - connection = globals.nonChunkedConnectionPool.lifoOfAvailableConnections[globals.nonChunkedConnectionPool.lifoIndex] - globals.nonChunkedConnectionPool.lifoOfAvailableConnections[globals.nonChunkedConnectionPool.lifoIndex] = nil - _ = connection.tcpConn.Close() - } - globals.nonChunkedConnectionPool.Unlock() -} - -func getStarvationParameters() (starvationParameters *StarvationParameters) { - starvationParameters = &StarvationParameters{ - ChunkedConnectionPoolCapacity: globals.chunkedConnectionPool.poolCapacity, - NonChunkedConnectionPoolCapacity: globals.nonChunkedConnectionPool.poolCapacity, - } - - globals.chunkedConnectionPool.Lock() - starvationParameters.ChunkedConnectionPoolInUse = globals.chunkedConnectionPool.poolInUse - starvationParameters.ChunkedConnectionPoolNumWaiters = globals.chunkedConnectionPool.numWaiters - globals.chunkedConnectionPool.Unlock() - - globals.nonChunkedConnectionPool.Lock() - starvationParameters.NonChunkedConnectionPoolInUse = globals.nonChunkedConnectionPool.poolInUse - starvationParameters.NonChunkedConnectionPoolNumWaiters = globals.nonChunkedConnectionPool.numWaiters - globals.nonChunkedConnectionPool.Unlock() - - return -} - -func acquireChunkedConnection(useReserveForVolumeName string) (connection *connectionStruct, err error) { - var ( - connectionToBeCreated bool - cv *sync.Cond - ok bool - swiftChunkedStarvationCallbacks uint64 - wasStalled bool - ) - - if "" != useReserveForVolumeName { - globals.reservedChunkedConnectionMutex.Lock() - - // track this statistic once per call to acquireChunkedConnection() - freeConnections := globals.chunkedConnectionPool.poolCapacity - globals.chunkedConnectionPool.poolInUse - globals.ObjectPutCtxtFreeConnection.Add(uint64(freeConnections)) - - connection, ok = globals.reservedChunkedConnection[useReserveForVolumeName] - if ok { - // Reuse connection from globals.reservedChunkedConnection map...removing it from there since it's in use - - delete(globals.reservedChunkedConnection, useReserveForVolumeName) - - globals.reservedChunkedConnectionMutex.Unlock() - - stats.IncrementOperations(&stats.SwiftChunkedConnsReuseOps) - } else { - // No connection available...create a new one - - globals.reservedChunkedConnectionMutex.Unlock() - - connection = &connectionStruct{connectionNonce: globals.connectionNonce, reserveForVolumeName: useReserveForVolumeName} - - err = openConnection(fmt.Sprintf("swiftclient.acquireChunkedConnection(\"%v\")", - useReserveForVolumeName), connection) - if err != nil { - connection = nil - - // if openConnection() failed then the cached connections are - // likely broken and using them will create further mischief - drainConnections() - } - - stats.IncrementOperations(&stats.SwiftChunkedConnsCreateOps) - } - - return - } - - connectionToBeCreated = false - wasStalled = false - - globals.chunkedConnectionPool.Lock() - - // track this statistic once per call to acquireChunkedConnection() - freeConnections := globals.chunkedConnectionPool.poolCapacity - globals.chunkedConnectionPool.poolInUse - globals.ObjectPutCtxtFreeConnection.Add(uint64(freeConnections)) - - swiftChunkedStarvationCallbacks = 0 - - for { - if globals.chunkedConnectionPool.poolInUse < globals.chunkedConnectionPool.poolCapacity { - break - } - - wasStalled = true - - globals.chunkedConnectionPool.numWaiters++ - - if nil == globals.starvationCallback { - // Wait for a connection to be released before retrying - cv = sync.NewCond(&globals.chunkedConnectionPool) - _ = globals.chunkedConnectionPool.waiters.PushBack(cv) - cv.Wait() - } else { - // Issue starvationCallback() (synchronously) before retrying - globals.chunkedConnectionPool.Unlock() - globals.starvationCallbackSerializer.Lock() - globals.starvationCallback() - globals.starvationCallbackSerializer.Unlock() - globals.chunkedConnectionPool.Lock() - swiftChunkedStarvationCallbacks++ - } - - globals.chunkedConnectionPool.numWaiters-- - } - - if 0 < swiftChunkedStarvationCallbacks { - stats.IncrementOperationsBy(&stats.SwiftChunkedStarvationCallbacks, swiftChunkedStarvationCallbacks) - } - - globals.chunkedConnectionPool.poolInUse++ - - if 0 == globals.chunkedConnectionPool.lifoIndex { - connectionToBeCreated = true - } else { - globals.chunkedConnectionPool.lifoIndex-- - connection = globals.chunkedConnectionPool.lifoOfAvailableConnections[globals.chunkedConnectionPool.lifoIndex] - globals.chunkedConnectionPool.lifoOfAvailableConnections[globals.chunkedConnectionPool.lifoIndex] = nil - } - - globals.chunkedConnectionPool.Unlock() - - if connectionToBeCreated { - connection = &connectionStruct{connectionNonce: globals.connectionNonce, reserveForVolumeName: ""} - err = openConnection("swiftclient.acquireChunkedConnection()", connection) - if err != nil { - connection = nil - globals.chunkedConnectionPool.Lock() - globals.chunkedConnectionPool.poolInUse-- - globals.chunkedConnectionPool.Unlock() - - // if openConnection() failed then the cached connections are - // likely broken and using them will create further mischief - drainConnections() - } - stats.IncrementOperations(&stats.SwiftChunkedConnsCreateOps) - } else { - stats.IncrementOperations(&stats.SwiftChunkedConnsReuseOps) - } - - if wasStalled { - stats.IncrementOperations(&stats.SwiftChunkedConnectionPoolStallOps) - } else { - stats.IncrementOperations(&stats.SwiftChunkedConnectionPoolNonStallOps) - } - - return -} - -func releaseChunkedConnection(connection *connectionStruct, keepAlive bool) { - var ( - connectionToBeClosed bool - cv *sync.Cond - waiter *list.Element - ) - - if "" != connection.reserveForVolumeName { - if keepAlive && - (connection.connectionNonce == globals.connectionNonce) { - // Re-insert connection in globals.reservedChunkedConnection map - - globals.reservedChunkedConnectionMutex.Lock() - globals.reservedChunkedConnection[connection.reserveForVolumeName] = connection - globals.reservedChunkedConnectionMutex.Unlock() - } else { - // Don't re-insert connection in globals.reservedChunkedConnection map... just Close() it - - _ = connection.tcpConn.Close() - } - - return - } - - connectionToBeClosed = false - - globals.chunkedConnectionPool.Lock() - - globals.chunkedConnectionPool.poolInUse-- - - if keepAlive && (connection.connectionNonce == globals.connectionNonce) { - globals.chunkedConnectionPool.lifoOfAvailableConnections[globals.chunkedConnectionPool.lifoIndex] = connection - globals.chunkedConnectionPool.lifoIndex++ - } else { - connectionToBeClosed = true - } - - if 0 < globals.chunkedConnectionPool.waiters.Len() { - // Note: If starvationCallback is armed, acquirers will be retrying (not cv.Wait()'ing) - waiter = globals.chunkedConnectionPool.waiters.Front() - cv = waiter.Value.(*sync.Cond) - _ = globals.chunkedConnectionPool.waiters.Remove(waiter) - cv.Signal() - } - - globals.chunkedConnectionPool.Unlock() - - if connectionToBeClosed { - _ = connection.tcpConn.Close() - } -} - -// Grow or shrink the chunked connection pool -// -func resizeChunkedConnectionPool(newPoolSize uint) { - var ( - index uint16 - newAvailableConnections []*connectionStruct - newSize uint16 - ) - - newSize = uint16(newPoolSize) - - if newSize != globals.chunkedConnectionPool.poolCapacity { - // Need to adjust the size of the pool - - globals.chunkedConnectionPool.Lock() - - if newSize < globals.chunkedConnectionPool.poolCapacity { - // Shrinking the pool... must close any open connections beyond newSize - - for globals.chunkedConnectionPool.lifoIndex > newSize { - globals.chunkedConnectionPool.lifoIndex-- - index = globals.chunkedConnectionPool.lifoIndex - globals.chunkedConnectionPool.lifoOfAvailableConnections[index].tcpConn.Close() - globals.chunkedConnectionPool.lifoOfAvailableConnections[index] = nil - } - } else { - // Growing the pool... so clone and extend existing pool - - newAvailableConnections = make([]*connectionStruct, newPoolSize) - for i := uint16(0); i < globals.chunkedConnectionPool.poolCapacity; i++ { - newAvailableConnections[i] = globals.chunkedConnectionPool.lifoOfAvailableConnections[i] - } - for i := globals.chunkedConnectionPool.poolCapacity; i < newSize; i++ { - newAvailableConnections[globals.chunkedConnectionPool.poolCapacity] = nil - } - - globals.chunkedConnectionPool.lifoOfAvailableConnections = newAvailableConnections - } - - globals.chunkedConnectionPool.poolCapacity = newSize - - globals.chunkedConnectionPool.Unlock() - } -} - -// Get a connection to the noauth server from the non-chunked connection pool. -// -// Return an error and a nil connection if it could not be opened. -// -func acquireNonChunkedConnection() (connection *connectionStruct, err error) { - var ( - connectionToBeCreated bool - cv *sync.Cond - wasStalled bool - ) - - connectionToBeCreated = false - wasStalled = false - - globals.nonChunkedConnectionPool.Lock() - - // track this statistic once per call to acquireNonChunkedConnection() - freeConnections := globals.nonChunkedConnectionPool.poolCapacity - globals.nonChunkedConnectionPool.poolInUse - globals.ObjectNonChunkedFreeConnection.Add(uint64(freeConnections)) - - for { - if globals.nonChunkedConnectionPool.poolInUse < globals.nonChunkedConnectionPool.poolCapacity { - break - } - wasStalled = true - globals.nonChunkedConnectionPool.numWaiters++ - cv = sync.NewCond(&globals.nonChunkedConnectionPool) - _ = globals.nonChunkedConnectionPool.waiters.PushBack(cv) - cv.Wait() - globals.nonChunkedConnectionPool.numWaiters-- - } - - globals.nonChunkedConnectionPool.poolInUse++ - - if 0 == globals.nonChunkedConnectionPool.lifoIndex { - connectionToBeCreated = true - } else { - globals.nonChunkedConnectionPool.lifoIndex-- - connection = globals.nonChunkedConnectionPool.lifoOfAvailableConnections[globals.nonChunkedConnectionPool.lifoIndex] - globals.nonChunkedConnectionPool.lifoOfAvailableConnections[globals.nonChunkedConnectionPool.lifoIndex] = nil - } - - globals.nonChunkedConnectionPool.Unlock() - - if connectionToBeCreated { - connection = &connectionStruct{connectionNonce: globals.connectionNonce, reserveForVolumeName: ""} - err = openConnection("swiftclient.acquireNonChunkedConnection()", connection) - if err != nil { - connection = nil - globals.nonChunkedConnectionPool.Lock() - globals.nonChunkedConnectionPool.poolInUse-- - globals.nonChunkedConnectionPool.Unlock() - - // if openConnection() failed then the cached connections are - // likely broken and using them will create further mischief - drainConnections() - } - stats.IncrementOperations(&stats.SwiftNonChunkedConnsCreateOps) - } else { - stats.IncrementOperations(&stats.SwiftNonChunkedConnsReuseOps) - } - - if wasStalled { - stats.IncrementOperations(&stats.SwiftNonChunkedConnectionPoolStallOps) - } else { - stats.IncrementOperations(&stats.SwiftNonChunkedConnectionPoolNonStallOps) - } - return -} - -func releaseNonChunkedConnection(connection *connectionStruct, keepAlive bool) { - var ( - connectionToBeClosed bool - cv *sync.Cond - waiter *list.Element - ) - - connectionToBeClosed = false - - globals.nonChunkedConnectionPool.Lock() - - globals.nonChunkedConnectionPool.poolInUse-- - - if keepAlive && (connection.connectionNonce == globals.connectionNonce) { - globals.nonChunkedConnectionPool.lifoOfAvailableConnections[globals.nonChunkedConnectionPool.lifoIndex] = connection - globals.nonChunkedConnectionPool.lifoIndex++ - } else { - connectionToBeClosed = true - } - - if 0 < globals.nonChunkedConnectionPool.waiters.Len() { - waiter = globals.nonChunkedConnectionPool.waiters.Front() - cv = waiter.Value.(*sync.Cond) - _ = globals.nonChunkedConnectionPool.waiters.Remove(waiter) - cv.Signal() - } - - globals.nonChunkedConnectionPool.Unlock() - - if connectionToBeClosed { - _ = connection.tcpConn.Close() - } -} - -// Grow or shrink the non-chunked connection pool -// -func resizeNonChunkedConnectionPool(newPoolSize uint) { - var ( - index uint16 - newAvailableConnections []*connectionStruct - newSize uint16 - ) - - newSize = uint16(newPoolSize) - - if newSize != globals.nonChunkedConnectionPool.poolCapacity { - // Need to adjust the size of the pool - - globals.nonChunkedConnectionPool.Lock() - - if newSize < globals.nonChunkedConnectionPool.poolCapacity { - // Shrinking the pool... must close any open connections beyond newSize - - for globals.nonChunkedConnectionPool.lifoIndex > newSize { - globals.nonChunkedConnectionPool.lifoIndex-- - index = globals.nonChunkedConnectionPool.lifoIndex - globals.nonChunkedConnectionPool.lifoOfAvailableConnections[index].tcpConn.Close() - globals.nonChunkedConnectionPool.lifoOfAvailableConnections[index] = nil - } - } else { - // Growing the pool... so clone and extend existing pool - - newAvailableConnections = make([]*connectionStruct, newPoolSize) - for i := uint16(0); i < globals.nonChunkedConnectionPool.poolCapacity; i++ { - newAvailableConnections[i] = globals.nonChunkedConnectionPool.lifoOfAvailableConnections[i] - } - for i := globals.nonChunkedConnectionPool.poolCapacity; i < newSize; i++ { - newAvailableConnections[globals.nonChunkedConnectionPool.poolCapacity] = nil - } - - globals.nonChunkedConnectionPool.lifoOfAvailableConnections = newAvailableConnections - } - - globals.nonChunkedConnectionPool.poolCapacity = newSize - - globals.nonChunkedConnectionPool.Unlock() - } -} - -func chunkedConnectionFreeCnt() (freeChunkedConnections int64) { - globals.chunkedConnectionPool.Lock() - freeChunkedConnections = int64(globals.chunkedConnectionPool.poolCapacity) - int64(globals.chunkedConnectionPool.poolInUse) - globals.chunkedConnectionPool.Unlock() - return -} - -func nonChunkedConnectionFreeCnt() (freeNonChunkedConnections int64) { - globals.nonChunkedConnectionPool.Lock() - freeNonChunkedConnections = int64(globals.nonChunkedConnectionPool.poolCapacity) - int64(globals.nonChunkedConnectionPool.poolInUse) - globals.nonChunkedConnectionPool.Unlock() - return -} - -// used during testing for error injection -var openConnectionCallCnt uint32 - -// Open a connection to the Swift NoAuth Proxy. -// -func openConnection(caller string, connection *connectionStruct) (err error) { - if atomic.LoadUint32(&globals.chaosOpenConnectionFailureRate) > 0 { - // atomic add only used when testing - if atomic.AddUint32(&openConnectionCallCnt, 1)%atomic.LoadUint32(&globals.chaosOpenConnectionFailureRate) == 0 { - err = fmt.Errorf("Simulated openConnection() error") - } - } - - if err == nil { - connection.tcpConn, err = net.DialTCP("tcp4", nil, globals.noAuthTCPAddr) - } - if err != nil { - logger.WarnfWithError(err, "%s cannot connect to Swift NoAuth Pipeline at %s", - caller, globals.noAuthStringAddr) - } - return -} - -func pathEscape(pathElements ...string) (pathEscaped string) { - if 0 == len(pathElements) { - pathEscaped = "" - } else { - pathEscaped = url.PathEscape(pathElements[0]) - for i := 1; i < len(pathElements); i++ { - pathEscaped += "/" + url.PathEscape(pathElements[i]) - } - } - return -} - -func writeBytesToTCPConn(tcpConn *net.TCPConn, buf []byte) (err error) { - var ( - bufPos = int(0) - written int - ) - - for bufPos < len(buf) { - written, err = tcpConn.Write(buf[bufPos:]) - if nil != err { - return - } - - bufPos += written - } - - err = nil - return -} - -func writeHTTPRequestLineAndHeaders(tcpConn *net.TCPConn, method string, path string, headers map[string][]string) (err error) { - var ( - bytesBuffer bytes.Buffer - headerName string - headerValue string - headerValueIndex int - headerValues []string - ) - - _, _ = bytesBuffer.WriteString(method + " " + path + " HTTP/1.1\r\n") - - _, _ = bytesBuffer.WriteString("Host: " + globals.noAuthStringAddr + "\r\n") - _, _ = bytesBuffer.WriteString("User-Agent: ProxyFS " + version.ProxyFSVersion + "\r\n") - - for headerName, headerValues = range headers { - _, _ = bytesBuffer.WriteString(headerName + ": ") - for headerValueIndex, headerValue = range headerValues { - if 0 == headerValueIndex { - _, _ = bytesBuffer.WriteString(headerValue) - } else { - _, _ = bytesBuffer.WriteString(", " + headerValue) - } - } - _, _ = bytesBuffer.WriteString("\r\n") - } - - _, _ = bytesBuffer.WriteString("\r\n") - - err = writeBytesToTCPConn(tcpConn, bytesBuffer.Bytes()) - - return -} - -func writeHTTPPutChunk(tcpConn *net.TCPConn, buf []byte) (err error) { - err = writeBytesToTCPConn(tcpConn, []byte(fmt.Sprintf("%X\r\n", len(buf)))) - if nil != err { - return - } - - if 0 < len(buf) { - err = writeBytesToTCPConn(tcpConn, buf) - if nil != err { - return - } - } - - err = writeBytesToTCPConn(tcpConn, []byte(fmt.Sprintf("\r\n"))) - - return -} - -func readByteFromTCPConn(tcpConn *net.TCPConn) (b byte, err error) { - var ( - numBytesRead int - oneByteBuf = []byte{byte(0)} - ) - - for { - numBytesRead, err = tcpConn.Read(oneByteBuf) - if nil != err { - return - } - - if 1 == numBytesRead { - b = oneByteBuf[0] - err = nil - return - } - } -} - -func readBytesFromTCPConn(tcpConn *net.TCPConn, bufLen int) (buf []byte, err error) { - var ( - bufPos = int(0) - numBytesRead int - ) - - buf = make([]byte, bufLen) - - for bufPos < bufLen { - numBytesRead, err = tcpConn.Read(buf[bufPos:]) - if nil != err { - return - } - - bufPos += numBytesRead - } - - err = nil - return -} - -func readBytesFromTCPConnIntoBuf(tcpConn *net.TCPConn, buf []byte) (err error) { - var ( - bufLen = cap(buf) - bufPos = int(0) - numBytesRead int - ) - - for bufPos < bufLen { - numBytesRead, err = tcpConn.Read(buf[bufPos:]) - if nil != err { - return - } - - bufPos += numBytesRead - } - - err = nil - return -} - -func readHTTPEmptyLineCRLF(tcpConn *net.TCPConn) (err error) { - var ( - b byte - ) - - b, err = readByteFromTCPConn(tcpConn) - if nil != err { - return - } - if '\r' != b { - err = fmt.Errorf("readHTTPEmptyLineCRLF() didn't find the expected '\\r'") - return - } - - b, err = readByteFromTCPConn(tcpConn) - if nil != err { - return - } - if '\n' != b { - err = fmt.Errorf("readHTTPEmptyLineCRLF() didn't find the expected '\\n'") - return - } - - err = nil - return -} - -func readHTTPLineCRLF(tcpConn *net.TCPConn) (line string, err error) { - var ( - b byte - bytesBuffer bytes.Buffer - ) - - for { - b, err = readByteFromTCPConn(tcpConn) - if nil != err { - return - } - - if '\r' == b { - b, err = readByteFromTCPConn(tcpConn) - if nil != err { - return - } - - if '\n' != b { - err = fmt.Errorf("readHTTPLine() expected '\\n' after '\\r' to terminate line") - return - } - - line = bytesBuffer.String() - err = nil - return - } - - err = bytesBuffer.WriteByte(b) - if nil != err { - return - } - } -} - -func readHTTPLineLF(tcpConn *net.TCPConn) (line string, err error) { - var ( - b byte - bytesBuffer bytes.Buffer - ) - - for { - b, err = readByteFromTCPConn(tcpConn) - if nil != err { - return - } - - if '\n' == b { - line = bytesBuffer.String() - err = nil - return - } - - err = bytesBuffer.WriteByte(b) - if nil != err { - return - } - } -} - -func readHTTPStatusAndHeaders(tcpConn *net.TCPConn) (httpStatus int, headers map[string][]string, err error) { - var ( - colonSplit []string - commaSplit []string - commaSplitIndex int - commaSplitValue string - line string - ) - - line, err = readHTTPLineCRLF(tcpConn) - if nil != err { - return - } - - if len(line) < len("HTTP/1.1 XXX") { - err = fmt.Errorf("readHTTPStatusAndHeaders() expected StatusLine beginning with \"HTTP/1.1 XXX\"") - return - } - - if !strings.HasPrefix(line, "HTTP/1.1 ") { - err = fmt.Errorf("readHTTPStatusAndHeaders() expected StatusLine beginning with \"HTTP/1.1 XXX\"") - return - } - - httpStatus, err = strconv.Atoi(line[len("HTTP/1.1 ") : len("HTTP/1.1 ")+len("XXX")]) - if nil != err { - return - } - - headers = make(map[string][]string) - - for { - line, err = readHTTPLineCRLF(tcpConn) - if nil != err { - return - } - - if 0 == len(line) { - return - } - - colonSplit = strings.SplitN(line, ":", 2) - if 2 != len(colonSplit) { - err = fmt.Errorf("readHTTPStatusAndHeaders() expected HeaderLine") - return - } - - commaSplit = strings.Split(colonSplit[1], ",") - - for commaSplitIndex, commaSplitValue = range commaSplit { - commaSplit[commaSplitIndex] = strings.TrimSpace(commaSplitValue) - } - - headers[colonSplit[0]] = commaSplit - } -} - -func parseContentRange(headers map[string][]string) (firstByte int64, lastByte int64, objectSize int64, err error) { - // A Content-Range header is of the form a-b/n, where a, b, and n - // are all positive integers - bytesPrefix := "bytes " - - values, ok := headers["Content-Range"] - if !ok { - err = fmt.Errorf("Content-Range header not present") - return - } else if ok && 1 != len(values) { - err = fmt.Errorf("expected only one value for Content-Range header") - return - } - - if !strings.HasPrefix(values[0], bytesPrefix) { - err = fmt.Errorf("malformed Content-Range header (doesn't start with %v)", bytesPrefix) - } - - parts := strings.SplitN(values[0][len(bytesPrefix):], "/", 2) - if len(parts) != 2 { - err = fmt.Errorf("malformed Content-Range header (no slash)") - return - } - - byteIndices := strings.SplitN(parts[0], "-", 2) - if len(byteIndices) != 2 { - err = fmt.Errorf("malformed Content-Range header (no dash)") - return - } - - firstByte, err = strconv.ParseInt(byteIndices[0], 10, 64) - if err != nil { - return - } - - lastByte, err = strconv.ParseInt(byteIndices[1], 10, 64) - if err != nil { - return - } - - objectSize, err = strconv.ParseInt(parts[1], 10, 64) - return -} - -func parseContentLength(headers map[string][]string) (contentLength int, err error) { - var ( - contentLengthAsValues []string - ok bool - ) - - contentLengthAsValues, ok = headers["Content-Length"] - - if ok { - if 1 != len(contentLengthAsValues) { - err = fmt.Errorf("parseContentLength() expected Content-Length HeaderLine with single value") - return - } - - contentLength, err = strconv.Atoi(contentLengthAsValues[0]) - if nil != err { - err = fmt.Errorf("parseContentLength() could not parse Content-Length HeaderLine value: \"%s\"", contentLengthAsValues[0]) - return - } - - if 0 > contentLength { - err = fmt.Errorf("parseContentLength() could not parse Content-Length HeaderLine value: \"%s\"", contentLengthAsValues[0]) - return - } - } else { - contentLength = 0 - } - - return -} - -func parseTransferEncoding(headers map[string][]string) (chunkedTransfer bool) { - var ( - transferEncodingAsValues []string - ok bool - ) - - transferEncodingAsValues, ok = headers["Transfer-Encoding"] - if !ok { - chunkedTransfer = false - return - } - - if 1 != len(transferEncodingAsValues) { - chunkedTransfer = false - return - } - - if "chunked" == transferEncodingAsValues[0] { - chunkedTransfer = true - } else { - chunkedTransfer = false - } - - return -} - -func parseConnection(headers map[string][]string) (connectionStillOpen bool) { - var ( - connectionAsValues []string - ok bool - ) - - connectionAsValues, ok = headers["Connection"] - if !ok { - connectionStillOpen = true - return - } - - if 1 != len(connectionAsValues) { - connectionStillOpen = true - return - } - - if "close" == connectionAsValues[0] { - connectionStillOpen = false - } else { - connectionStillOpen = true - } - - return -} - -func readHTTPPayloadAsByteSlice(tcpConn *net.TCPConn, headers map[string][]string) (payloadAsByteSlice []byte, err error) { - var ( - chunk []byte - contentLength int - ) - - if parseTransferEncoding(headers) { - payloadAsByteSlice = make([]byte, 0) - for { - chunk, err = readHTTPChunk(tcpConn) - if nil != err { - return - } - - if 0 == len(chunk) { - break - } - - payloadAsByteSlice = append(payloadAsByteSlice, chunk...) - } - } else { - contentLength, err = parseContentLength(headers) - if nil != err { - return - } - - if 0 == contentLength { - payloadAsByteSlice = make([]byte, 0) - } else { - payloadAsByteSlice, err = readBytesFromTCPConn(tcpConn, contentLength) - if nil != err { - return - } - } - } - - return -} - -func readHTTPPayloadAsString(tcpConn *net.TCPConn, headers map[string][]string) (payloadString string, err error) { - var ( - payloadByteSlice []byte - ) - - payloadByteSlice, err = readHTTPPayloadAsByteSlice(tcpConn, headers) - if nil != err { - return - } - - payloadString = string(payloadByteSlice[:]) - - return -} - -func readHTTPPayloadLines(tcpConn *net.TCPConn, headers map[string][]string) (lines []string, err error) { - var ( - buf []byte - bufCurrentPosition int - bufLineStartPosition int - contentLength int - ) - - buf, err = readHTTPPayloadAsByteSlice(tcpConn, headers) - if nil != err { - return - } - - contentLength = len(buf) - - lines = make([]string, 0) - - if 0 < len(buf) { - bufLineStartPosition = 0 - bufCurrentPosition = 0 - - for bufCurrentPosition < contentLength { - if '\n' == buf[bufCurrentPosition] { - if bufCurrentPosition == bufLineStartPosition { - err = fmt.Errorf("readHTTPPayloadLines() unexpectedly found an empty line in Payload") - return - } - - lines = append(lines, string(buf[bufLineStartPosition:bufCurrentPosition])) - - bufLineStartPosition = bufCurrentPosition + 1 - } - - bufCurrentPosition++ - } - - if bufLineStartPosition != bufCurrentPosition { - err = fmt.Errorf("readHTTPPayloadLines() unexpectedly found a non-terminated line in Payload") - return - } - } - - err = nil - return -} - -func readHTTPChunk(tcpConn *net.TCPConn) (chunk []byte, err error) { - var ( - chunkLen uint64 - line string - ) - - line, err = readHTTPLineCRLF(tcpConn) - if nil != err { - return - } - - chunkLen, err = strconv.ParseUint(line, 16, 64) - if nil != err { - return - } - - if 0 == chunkLen { - chunk = make([]byte, 0) - } else { - chunk, err = readBytesFromTCPConn(tcpConn, int(chunkLen)) - if nil != err { - return - } - } - - err = readHTTPEmptyLineCRLF(tcpConn) - - return -} - -func readHTTPChunkIntoBuf(tcpConn *net.TCPConn, buf []byte) (chunkLen uint64, err error) { - var ( - line string - ) - - line, err = readHTTPLineCRLF(tcpConn) - if nil != err { - return - } - - chunkLen, err = strconv.ParseUint(line, 16, 64) - if nil != err { - return - } - - if 0 < chunkLen { - err = readBytesFromTCPConnIntoBuf(tcpConn, buf[:chunkLen]) - if nil != err { - return - } - } - - err = readHTTPEmptyLineCRLF(tcpConn) - - return -} - -// mergeHeadersAndList performs a logical merge of headers and lists among successive Account or Container GETs. -// -// Content-Length header values are summed -// Other headers that are single valued and don't change don't change the header value -// Multi-valued headers or headers that change value are appended -func mergeHeadersAndList(masterHeaders map[string][]string, masterList *[]string, toAddHeaders map[string][]string, toAddList *[]string) { - var ( - addedContentLength uint64 - err error - ok bool - prevContentLength uint64 - prevContentLengthAsStringSlice []string - prevValues []string - ) - - for key, values := range toAddHeaders { - if "Content-Length" == key { - prevContentLengthAsStringSlice, ok = masterHeaders["Content-Length"] - if !ok { - prevContentLengthAsStringSlice = []string{"0"} - } - if 1 != len(prevContentLengthAsStringSlice) { - err = fmt.Errorf("mergeHeadersAndList() passed masterHeaders with unexpected Content-Length header: %v", prevContentLengthAsStringSlice) - panic(err) - } - prevContentLength, err = strconv.ParseUint(prevContentLengthAsStringSlice[0], 10, 64) - if nil != err { - err = fmt.Errorf("mergeHeadersAndList() passed masterHeaders with unexpected Content-Length header: %v", prevContentLengthAsStringSlice) - panic(err) - } - if 1 != len(values) { - err = fmt.Errorf("mergeHeadersAndList() passed toAddHeaders with unexpected Content-Length header: %v", values) - } - addedContentLength, err = strconv.ParseUint(values[0], 10, 64) - if nil != err { - err = fmt.Errorf("mergeHeadersAndList() passed toAddHeaders with unexpected Content-Length header: %v", values) - panic(err) - } - masterHeaders["Content-Length"] = []string{strconv.FormatUint(prevContentLength+addedContentLength, 10)} - } else { - prevValues, ok = masterHeaders[key] - if ok { - if (1 != len(prevValues)) || (1 != len(values)) || (prevValues[0] != values[0]) { - masterHeaders[key] = append(prevValues, values...) - } - } else { - masterHeaders[key] = values - } - } - } - - *masterList = append(*masterList, *toAddList...) -} diff --git a/swiftclient/utils_test.go b/swiftclient/utils_test.go deleted file mode 100644 index 6b7c2784..00000000 --- a/swiftclient/utils_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package swiftclient - -import ( - "testing" -) - -func TestUtils(t *testing.T) { - masterHeaders := make(map[string][]string) - masterList := make([]string, 0) - - toAddHeaders1 := make(map[string][]string) - toAddList1 := make([]string, 0) - - toAddHeaders2 := make(map[string][]string) - toAddList2 := make([]string, 0) - - toAddHeaders3 := make(map[string][]string) - toAddList3 := make([]string, 0) - - toAddHeaders1["Content-Length"] = []string{"4"} - toAddHeaders1["Dummy-Header-W"] = []string{"W"} - toAddHeaders1["Dummy-Header-X"] = []string{"XA"} - toAddHeaders1["Dummy-Header-Z"] = []string{} - toAddList1 = []string{"A", "B"} - - toAddHeaders2["Content-Length"] = []string{"6"} - toAddHeaders2["Dummy-Header-W"] = []string{"W"} - toAddHeaders2["Dummy-Header-X"] = []string{"XB", "XC"} - toAddHeaders2["Dummy-Header-Y"] = []string{"Y"} - toAddList2 = []string{"C", "D", "E"} - - toAddHeaders3["Content-Length"] = []string{"8"} - toAddHeaders3["Dummy-Header-W"] = []string{"W"} - toAddHeaders3["Dummy-Header-Z"] = []string{"ZA", "ZB"} - toAddList3 = []string{} - - mergeHeadersAndList(masterHeaders, &masterList, toAddHeaders1, &toAddList1) - mergeHeadersAndList(masterHeaders, &masterList, toAddHeaders2, &toAddList2) - mergeHeadersAndList(masterHeaders, &masterList, toAddHeaders3, &toAddList3) - - if 5 != len(masterHeaders) { - t.Fatalf("masterHeaders had unexpected len (%v)", len(masterHeaders)) - } - - masterContentLength, ok := masterHeaders["Content-Length"] - if !ok { - t.Fatalf("masterHeaders[\"Content-Length\"] returned !ok") - } - if 1 != len(masterContentLength) { - t.Fatalf("masterHeaders[\"Content-Length\"] had unexpected len (%v)", len(masterContentLength)) - } - if "18" != masterContentLength[0] { - t.Fatalf("masterHeaders[\"Content-Length\"] returned unexpected value ([\"%v\"])", masterContentLength[0]) - } - - masterDummyHeaderW, ok := masterHeaders["Dummy-Header-W"] - if !ok { - t.Fatalf("masterHeaders[\"Dummy-Header-W\"] returned !ok") - } - if 1 != len(masterDummyHeaderW) { - t.Fatalf("masterHeaders[\"Dummy-Header-W\"] had unexpected len (%v)", len(masterDummyHeaderW)) - } - if "W" != masterDummyHeaderW[0] { - t.Fatalf("masterHeaders[\"Dummy-Header-W\"] had unexpected value ([\"%v\"])", masterDummyHeaderW[0]) - } - - masterDummyHeaderX, ok := masterHeaders["Dummy-Header-X"] - if !ok { - t.Fatalf("masterHeaders[\"Dummy-Header-X\"] returned !ok") - } - if 3 != len(masterDummyHeaderX) { - t.Fatalf("masterHeaders[\"Dummy-Header-X\"] had unexpected len (%v)", len(masterDummyHeaderX)) - } - if "XA" != masterDummyHeaderX[0] { - t.Fatalf("masterHeaders[\"Dummy-Header-X\"][0] had unexpected value (\"%v\")", masterDummyHeaderX[0]) - } - if "XB" != masterDummyHeaderX[1] { - t.Fatalf("masterHeaders[\"Dummy-Header-X\"][1] had unexpected value (\"%v\")", masterDummyHeaderX[1]) - } - if "XC" != masterDummyHeaderX[2] { - t.Fatalf("masterHeaders[\"Dummy-Header-X\"][2] had unexpected value (\"%v\")", masterDummyHeaderX[2]) - } - - masterDummyHeaderY, ok := masterHeaders["Dummy-Header-Y"] - if !ok { - t.Fatalf("masterHeaders[\"Dummy-Header-Y\"] returned !ok") - } - if 1 != len(masterDummyHeaderY) { - t.Fatalf("masterHeaders[\"Dummy-Header-Y\"] had unexpected len (%v)", len(masterDummyHeaderY)) - } - if "Y" != masterDummyHeaderY[0] { - t.Fatalf("masterHeaders[\"Dummy-Header-Y\"] had unexpected value ([\"%v\"])", masterDummyHeaderY[0]) - } - - masterDummyHeaderZ, ok := masterHeaders["Dummy-Header-Z"] - if !ok { - t.Fatalf("masterHeaders[\"Dummy-Header-Z\"] returned !ok") - } - if 2 != len(masterDummyHeaderZ) { - t.Fatalf("masterHeaders[\"Dummy-Header-Z\"] had unexpected len (%v)", len(masterDummyHeaderZ)) - } - if "ZA" != masterDummyHeaderZ[0] { - t.Fatalf("masterHeaders[\"Dummy-Header-Z\"][0] had unexpected value (\"%v\")", masterDummyHeaderZ[0]) - } - if "ZB" != masterDummyHeaderZ[1] { - t.Fatalf("masterHeaders[\"Dummy-Header-Z\"][1] had unexpected value (\"%v\")", masterDummyHeaderZ[1]) - } - - if 5 != len(masterList) { - t.Fatalf("masterList had unexpected len (%v)", len(masterContentLength)) - } - if "A" != masterList[0] { - t.Fatalf("masterList[0] had unexpected value (\"%v\")", masterList[0]) - } - if "B" != masterList[1] { - t.Fatalf("masterList[1] had unexpected value (\"%v\")", masterList[1]) - } - if "C" != masterList[2] { - t.Fatalf("masterList[2] had unexpected value (\"%v\")", masterList[2]) - } - if "D" != masterList[3] { - t.Fatalf("masterList[3] had unexpected value (\"%v\")", masterList[3]) - } - if "E" != masterList[4] { - t.Fatalf("masterList[4] had unexpected value (\"%v\")", masterList[4]) - } -} diff --git a/utf/utf.go b/utf/utf.go deleted file mode 100644 index 7cae47bf..00000000 --- a/utf/utf.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package utf provides utilities for working with UTF strings, notably including UTF16. -package utf - -import "bytes" -import "encoding/binary" -import "fmt" -import "unicode/utf16" -import "unicode/utf8" - -var LittleEndian binary.ByteOrder = binary.LittleEndian -var BigEndian binary.ByteOrder = binary.BigEndian - -func UTF16ByteSliceToString(u8Buf []byte, byteOrder binary.ByteOrder) (utf8String string, err error) { - // Set default return values - - utf8String = "" - err = nil - - // Ensure []byte can be interpretted as []uint16 - - if 0 != (len(u8Buf) % 2) { - err = fmt.Errorf("UTF-16-LE requires []byte with even number of bytes") - return - } - - // Convert u8Buf ([]byte) to u16Buf ([]uint16) - - numUint16s := len(u8Buf) / 2 - u16Reader := bytes.NewReader(u8Buf) - u16Buf := make([]uint16, numUint16s) - - err = binary.Read(u16Reader, byteOrder, &u16Buf) - if nil != err { - return - } - - // Convert u16Buf ([]uint16) to runeForm ([]rune) - - runeFormSlice := utf16.Decode(u16Buf) - - // Encode runeFormSlice elements into bytes.Buffer - - var runeByteBuffer bytes.Buffer - - for _, runeFormElement := range runeFormSlice { - _, _ = runeByteBuffer.WriteRune(runeFormElement) - } - - // Return resultant string from runeByteBuffer - - utf8String = runeByteBuffer.String() - - return -} - -func StringToUTF16ByteSlice(utf8String string, byteOrder binary.ByteOrder) (u8Buf []byte) { - runeArray := []rune(utf8String) - u16Slice := utf16.Encode(runeArray) - - u8Buf = make([]byte, (2 * len(u16Slice))) - - for i := 0; i < len(u16Slice); i++ { - if binary.LittleEndian == byteOrder { - u8Buf[(2*i)+0] = byte((u16Slice[i] >> 0) & 0xFF) - u8Buf[(2*i)+1] = byte((u16Slice[i] >> 8) & 0xFF) - } else { // binary.BigEndian == byteOrder - u8Buf[(2*i)+1] = byte((u16Slice[i] >> 0) & 0xFF) - u8Buf[(2*i)+0] = byte((u16Slice[i] >> 8) & 0xFF) - } - } - - return -} - -func UTF8ByteSliceToString(u8Buf []byte) (utf8String string, err error) { - if !utf8.Valid(u8Buf) { - utf8String = "" - err = fmt.Errorf("Not valid UTF-8") - return - } - - utf8String = string(u8Buf) - err = nil - - return -} - -func StringToUTF8ByteSlice(utf8String string) (u8Buf []byte) { - u8Buf = []byte(utf8String) - - return -} diff --git a/utf/utf_test.go b/utf/utf_test.go deleted file mode 100644 index c396c6e7..00000000 --- a/utf/utf_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package utf - -import "bytes" -import "strings" -import "testing" - -func TestUTF16ByteSliceToString(t *testing.T) { - var goodUTF16ByteSliceLittleEndian = []byte{0x55, 0x00, 0x0054, 0x00, 0x46, 0x00, 0x2D, 0x00, 0x31, 0x00, 0x36, 0x00} - var goodUTF16ByteSliceBigEndian = []byte{0x00, 0x55, 0x00, 0x0054, 0x00, 0x46, 0x00, 0x2D, 0x00, 0x31, 0x00, 0x36} - var goodUTF16ByteSliceToStringExpected = "UTF-16" - var badUTF16ByteSlice = []byte{0x01, 0x02, 0x03} - - goodUTF16ByteSliceLittleEndianToStringReturned, err := UTF16ByteSliceToString(goodUTF16ByteSliceLittleEndian, LittleEndian) - - if nil != err { - t.Fatalf("UTF16ByteSliceToString(goodUTF16ByteSliceLittleEndian, LittleEndian) failed unexpectedly: %v", err) - } - if 0 != strings.Compare(goodUTF16ByteSliceLittleEndianToStringReturned, goodUTF16ByteSliceToStringExpected) { - t.Fatalf("UTF16ByteSliceToString(goodUTF16ByteSliceLittleEndian, LittleEndian) returned \"%v\" instead of the expected \"%v\"", goodUTF16ByteSliceLittleEndianToStringReturned, goodUTF16ByteSliceToStringExpected) - } - - goodUTF16ByteSliceBigEndianToStringReturned, err := UTF16ByteSliceToString(goodUTF16ByteSliceBigEndian, BigEndian) - - if nil != err { - t.Fatalf("UTF16ByteSliceToString(goodUTF16ByteSliceBigEndian, BigEndian) failed unexpectedly: %v", err) - } - if 0 != strings.Compare(goodUTF16ByteSliceBigEndianToStringReturned, goodUTF16ByteSliceToStringExpected) { - t.Fatalf("UTF16ByteSliceToString(goodUTF16ByteSliceBigEndian, BigEndian) returned \"%v\" instead of the expected \"%v\"", goodUTF16ByteSliceBigEndianToStringReturned, goodUTF16ByteSliceToStringExpected) - } - - _, err = UTF16ByteSliceToString(badUTF16ByteSlice, LittleEndian) - - if nil == err { - t.Fatalf("UTF16ByteSliceToString(badUTF16ByteSlice, LittleEndian) succeeded unexpectedly") - } - - _, err = UTF16ByteSliceToString(badUTF16ByteSlice, BigEndian) - - if nil == err { - t.Fatalf("UTF16ByteSliceToString(badUTF16ByteSlice, BigEndian) succeeded unexpectedly") - } -} - -func TestStringToUTF16ByteSlice(t *testing.T) { - var utf8String = "UTF-16" - var u8BufUTF16LittleEndianExpected = []byte{0x55, 0x00, 0x0054, 0x00, 0x46, 0x00, 0x2D, 0x00, 0x31, 0x00, 0x36, 0x00} - var u8BufUTF16BigEndianExpected = []byte{0x00, 0x55, 0x00, 0x0054, 0x00, 0x46, 0x00, 0x2D, 0x00, 0x31, 0x00, 0x36} - - u8BufUTF16LittleEndianReturned := StringToUTF16ByteSlice(utf8String, LittleEndian) - - if 0 != bytes.Compare(u8BufUTF16LittleEndianReturned, u8BufUTF16LittleEndianExpected) { - t.Fatalf("StringToUTF16ByteSlice(utf8String, LittleEndian) returned unexpected u8Buf") - } - - u8BufUTF16BigEndianReturned := StringToUTF16ByteSlice(utf8String, BigEndian) - - if 0 != bytes.Compare(u8BufUTF16BigEndianReturned, u8BufUTF16BigEndianExpected) { - t.Fatalf("StringToUTF16ByteSlice(utf8String, BigEndian) returned unexpected u8Buf") - } -} - -func TestUTF8ByteSliceToString(t *testing.T) { - var goodUTF8ByteSlice = []byte{0x55, 0x54, 0x46, 0x2D, 0x38} - var goodUTF8ByteSliceToStringExpected = "UTF-8" - var badUTF8ByteSlice = []byte{0xFF} - - goodUTF8ByteSliceToStringReturned, err := UTF8ByteSliceToString(goodUTF8ByteSlice) - - if nil != err { - t.Fatalf("UTF8ByteSliceToString(goodUTF8ByteSlice) failed unexpectedly: %v", err) - } - if 0 != strings.Compare(goodUTF8ByteSliceToStringExpected, goodUTF8ByteSliceToStringReturned) { - t.Fatalf("UTF8ByteSliceToString(goodUTF16leByteSlice) returned \"%v\" instead of the expected \"%v\"", goodUTF8ByteSliceToStringReturned, goodUTF8ByteSliceToStringExpected) - } - - _, err = UTF8ByteSliceToString(badUTF8ByteSlice) - - if nil == err { - t.Fatalf("UTF8ByteSliceToString(badUTF8ByteSlice) succeeded unexpectedly") - } -} - -func TestStringToUTF8ByteSlice(t *testing.T) { - var utf8String = "UTF-8" - var utf8StringToUTF8ByteSliceExpected = []byte{0x55, 0x54, 0x46, 0x2D, 0x38} - - utf8StringToUTF8ByteSliceReturned := StringToUTF8ByteSlice(utf8String) - - if 0 != bytes.Compare(utf8StringToUTF8ByteSliceReturned, utf8StringToUTF8ByteSliceExpected) { - t.Fatalf("StringToUTF8ByteSlice(utf8String) returned unexpected u8Buf") - } -} diff --git a/win10/Vagrantfile b/win10/Vagrantfile deleted file mode 100644 index 9a5fc048..00000000 --- a/win10/Vagrantfile +++ /dev/null @@ -1,45 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Notes: -# -# 0) What follows assumes you have a running SAIO for ProxyFS VM sharing non-DHCP Host-only vboxnet0 -# (wherein your "host" is using 172.28.128.1 and the SAIO for ProxyFS VM is using 172.28.128.2) -# 1) "vagrant up" will likely have some problems connecting to Windows VM, -# so just use VirtualBox Manager after initial provisioning -# 2) You may need to ^C out of the vagrant up command following provisioning -# 3) Subsequent restarts should just be done with VirtualBox Manager -# 4) To get to the console, launch VirtualBox Manager, select EdgeOnWindows10, and hit Show -# 5) You may be asked to Update OneDrive on first boot -# 6) Settings->Network&Internet->Ethernet->Change adapter options -# 7) Right-click to Properties on Ethernet2 -# 8) Select Properties for Internet Protocol Version 4 (TCP/IPv4) -# 9) Select Use the following IP address and enter: -# IP address: 172 . 28 . 128 . 3 -# Subnet mask: 255 . 255 . 255 . 0 -# Default gateway: 172 . 28 . 128 . 1 [as this is a Host-only network, you're not going anywhere else though] -# 10) OK/Close back out to Settings->Network&Internet->Ethernet and select Windows Firewall -# 11) Noting that Windows Firewall state is currently "On", select "Advanced settings" (on left) -# 12) At bottom of "Overview" section, select "Windows Firewall Properties" -# 13) For each of Domain Profile, Private Profile, and Public Profile tabs, -# set Firewall state to "Off" and Apply -# 14) Open File Explorer -# 15) Right-click Network->Map network drive... -# 16) Enter \\172.28.128.2\proxyfs in Folder text box, -# uncheck Reconnect at sign-in, and -# check Connect using different credentials and hit Finish to map Drive Letter Z to your running ProxyFS VM -# 17) User name should be set to "\swift" (the leading backslash indicates a user not in a domain) -# and a password of "swift" and hit "OK" -# 18) When you shutdown (probably after disconnecting Z:), you'll likely want to await the Windows Updates - -Vagrant.configure(2) do |config| - config.vm.box = "Microsoft/EdgeOnWindows10" - config.vm.box_version = "1.0" - config.vm.communicator = "winrm" - config.vm.provider :virtualbox do |vb| - vb.name = "EdgeOnWindows10" - vb.cpus = Integer(ENV['VAGRANT_CPUS'] || 1) - vb.memory = Integer(ENV['VAGRANT_RAM'] || 2048) - end - config.vm.network "private_network", ip: "172.28.128.3", :name => 'vboxnet0', :adapter => 2 -end From 01a123d5636c637686d7c0e6eed4f90354506ea2 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 29 Dec 2021 14:48:39 -0800 Subject: [PATCH 224/258] Now we can re-enable make ci to do the cover step w/out worries ...oddly, somebody "fixed" etcd/grpc/et. al. over the past week... so we could theoretically restore the etcd dependency as of "today"... --- Makefile | 8 +- go.mod | 25 ++ go.sum | 791 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 817 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index d8f746d7..031baba7 100644 --- a/Makefile +++ b/Makefile @@ -36,13 +36,7 @@ generatedfiles := \ all: version fmt pre-generate generate build test -# Due to etcd breaking go modules when running make cover, -# we will, for the time being, disable the make cover step -# in make ci :-( -# -# ci: version fmt pre-generate generate build test cover - -ci: all +ci: version fmt pre-generate generate build test cover minimal: version pre-generate generate build diff --git a/go.mod b/go.mod index 97cb6ed3..72e094da 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,33 @@ require ( require ( github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 // indirect github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/ory/viper v1.7.5 // indirect + github.com/pborman/uuid v1.2.1 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/afero v1.7.1 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cobra v1.3.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 57ceb021..7a44ebfe 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,68 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b h1:Jhps18HvZUrJKJR2E57rF5Kp/7chK6NDo/wZnD0gtF0= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b/go.mod h1:9wVslsyxZaBvW/ollg7JLxJOxKb+Ik2KH1WVs1nicMA= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/ansel1/merry v1.5.0/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= github.com/ansel1/merry v1.5.1/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= github.com/ansel1/merry v1.6.1 h1:AuheQTqQ04yQdBMCDnneVPvpBdWTBBg3LKMvKGMyRxU= @@ -13,22 +70,117 @@ github.com/ansel1/merry v1.6.1/go.mod h1:ioJjPJ/IsjxH+cC0lpf5TmbKnbcGa9qTk0fDbeR github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZtKWYbsCus= github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -37,68 +189,677 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.7.1 h1:F37zV8E8RLstLpZ0RUGK2NGg1X57y6/B0Eg6S8oqdoA= +github.com/spf13/afero v1.7.1/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -109,11 +870,41 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From bf013f9ec394c73739c2259315c6d2500180220b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 29 Dec 2021 15:22:49 -0800 Subject: [PATCH 225/258] Deprecating stale package platform --- go.mod | 2 ++ go.sum | 3 ++- platform/Makefile | 6 ----- platform/constants.go | 24 ------------------- platform/dummy_test.go | 11 --------- platform/is_darwin.go | 9 ------- platform/is_linux.go | 9 ------- platform/memsize_darwin.go | 32 ------------------------- platform/memsize_linux_amd64.go | 28 ---------------------- platform/memsize_linux_arm.go | 28 ---------------------- platform/open_file_sync_darwin.go | 39 ------------------------------- platform/open_file_sync_linux.go | 28 ---------------------- 12 files changed, 4 insertions(+), 215 deletions(-) delete mode 100644 platform/Makefile delete mode 100644 platform/constants.go delete mode 100644 platform/dummy_test.go delete mode 100644 platform/is_darwin.go delete mode 100644 platform/is_linux.go delete mode 100644 platform/memsize_darwin.go delete mode 100644 platform/memsize_linux_amd64.go delete mode 100644 platform/memsize_linux_arm.go delete mode 100644 platform/open_file_sync_darwin.go delete mode 100644 platform/open_file_sync_linux.go diff --git a/go.mod b/go.mod index 72e094da..1ee95066 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/kr/pretty v0.2.0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/ory/go-acc v0.2.6 // indirect @@ -41,6 +42,7 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect diff --git a/go.sum b/go.sum index 7a44ebfe..5b4c7c74 100644 --- a/go.sum +++ b/go.sum @@ -300,8 +300,10 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -874,7 +876,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= diff --git a/platform/Makefile b/platform/Makefile deleted file mode 100644 index 31637cc0..00000000 --- a/platform/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/platform - -include ../GoMakefile diff --git a/platform/constants.go b/platform/constants.go deleted file mode 100644 index 4e3bea8c..00000000 --- a/platform/constants.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -const ( - // GoHeapAllocationMultiplier defines the float64 overhead of memory allocations - // in the Golang runtime. This multiplier is > 1 primarily due to the desire to - // avoid overly fragmenting RAM unless/until the Golang Garbage Collector - // implements any form of compaction. - // - // With experimentation, a GoHeapAllocationMultiplier of 2.0 appears to be just - // about right for nearly all access paths/patterns. This appears true for all - // Linux local access (FUSE, SMB loopback, and NFS loopback) as well as SMB access - // from from a Windows client and NFS access from a Mac. The one outlier is a - // Mac doing SMB access. In this case, a GoHeapAllocationMultiplier of 4.0 appears - // to be appropriate. - // - // Still, with multiple SMB read streams (Windows 7 & Windows 10) doing RoboCopy's, - // one test exceeded even that 4.0 value. Hence, the current setting will be very - // conservative to hopefully avoid all such memory fragmentation concerns at - // steady state until such time that a more definitive solution is available. - GoHeapAllocationMultiplier = float64(10.0) -) diff --git a/platform/dummy_test.go b/platform/dummy_test.go deleted file mode 100644 index 7b76d208..00000000 --- a/platform/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/platform/is_darwin.go b/platform/is_darwin.go deleted file mode 100644 index 2caa5235..00000000 --- a/platform/is_darwin.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -const ( - IsDarwin = true - IsLinux = false -) diff --git a/platform/is_linux.go b/platform/is_linux.go deleted file mode 100644 index 50f2e258..00000000 --- a/platform/is_linux.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -const ( - IsDarwin = false - IsLinux = true -) diff --git a/platform/memsize_darwin.go b/platform/memsize_darwin.go deleted file mode 100644 index 2d3c1c28..00000000 --- a/platform/memsize_darwin.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -import ( - "encoding/binary" - - "golang.org/x/sys/unix" -) - -const ( - swiftAccountCheckpointHeaderName = "X-Account-Meta-Checkpoint" -) - -func MemSize() (memSize uint64) { - var ( - err error - sysctlReturn string - ) - - sysctlReturn, err = unix.Sysctl("hw.memsize") - if nil != err { - panic(err) - } - - sysctlReturn += "\x00" - - memSize = uint64(binary.LittleEndian.Uint64([]byte(sysctlReturn))) - - return -} diff --git a/platform/memsize_linux_amd64.go b/platform/memsize_linux_amd64.go deleted file mode 100644 index 4ffdf7ea..00000000 --- a/platform/memsize_linux_amd64.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -import ( - "syscall" -) - -const ( - swiftAccountCheckpointHeaderName = "X-Account-Meta-Checkpoint" -) - -func MemSize() (memSize uint64) { - var ( - err error - sysinfo syscall.Sysinfo_t - ) - - err = syscall.Sysinfo(&sysinfo) - if nil != err { - panic(err) - } - - memSize = sysinfo.Totalram - - return -} diff --git a/platform/memsize_linux_arm.go b/platform/memsize_linux_arm.go deleted file mode 100644 index ae956768..00000000 --- a/platform/memsize_linux_arm.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -import ( - "syscall" -) - -const ( - swiftAccountCheckpointHeaderName = "X-Account-Meta-Checkpoint" -) - -func MemSize() (memSize uint64) { - var ( - err error - sysinfo syscall.Sysinfo_t - ) - - err = syscall.Sysinfo(&sysinfo) - if nil != err { - panic(err) - } - - memSize = uint64(sysinfo.Totalram) - - return -} diff --git a/platform/open_file_sync_darwin.go b/platform/open_file_sync_darwin.go deleted file mode 100644 index 48983308..00000000 --- a/platform/open_file_sync_darwin.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -import ( - "fmt" - "os" - "syscall" -) - -// OpenFileSync that ensures reads and writes are not cached and also that writes -// are not reported as complete until the data and metadata are persisted. -// -// Note that the request for no caching will only be honored if the file has -// not already entered the cache at the time of the call to OpenFile. -func OpenFileSync(name string, flag int, perm os.FileMode) (file *os.File, err error) { - var ( - errno syscall.Errno - modifiedFlag int - ) - - modifiedFlag = flag - modifiedFlag |= syscall.O_SYNC // writes are not complete until data & metadata is persisted - - file, err = os.OpenFile(name, modifiedFlag, perm) - if nil != err { - return - } - - _, _, errno = syscall.Syscall(syscall.SYS_FCNTL, uintptr(file.Fd()), syscall.F_NOCACHE, 1) - if 0 != errno { - err = fmt.Errorf("fcntl(,F_NOCACHE,1) returned non-zero errno: %v", errno) - _ = file.Close() - file = nil - } - - return -} diff --git a/platform/open_file_sync_linux.go b/platform/open_file_sync_linux.go deleted file mode 100644 index 92156b9b..00000000 --- a/platform/open_file_sync_linux.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package platform - -import ( - "os" - "syscall" -) - -// OpenFileSync that ensures reads and writes are not cached and also that writes -// are not reported as complete until the data and metadata are persisted. -// -// Note that buffers must be aligned on the platform's buffer cache boundary -// in order for the cache bypass mode to function. -func OpenFileSync(name string, flag int, perm os.FileMode) (file *os.File, err error) { - var ( - modifiedFlag int - ) - - modifiedFlag = flag - modifiedFlag |= syscall.O_DIRECT // aligned reads & writes DMA'd directly to/from user memory - modifiedFlag |= syscall.O_SYNC // writes are not complete until data & metadata is persisted - - file, err = os.OpenFile(name, modifiedFlag, perm) - - return -} From 43c827b9d58ad87c9d35f47558ab3b1d81aee465 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 30 Dec 2021 09:53:56 -0800 Subject: [PATCH 226/258] Removed dependency on (and presence of) package blunder --- blunder/.gitignore | 1 - blunder/Makefile | 9 - blunder/api.go | 321 ------------- blunder/api_test.go | 220 --------- go.mod | 28 -- go.sum | 877 ---------------------------------- retryrpc/perfrpc/ping_perf.go | 12 +- retryrpc/ping_test.go | 15 +- 8 files changed, 5 insertions(+), 1478 deletions(-) delete mode 100644 blunder/.gitignore delete mode 100644 blunder/Makefile delete mode 100644 blunder/api.go delete mode 100644 blunder/api_test.go diff --git a/blunder/.gitignore b/blunder/.gitignore deleted file mode 100644 index b6f8aa40..00000000 --- a/blunder/.gitignore +++ /dev/null @@ -1 +0,0 @@ -fserror_string.go diff --git a/blunder/Makefile b/blunder/Makefile deleted file mode 100644 index 9466c317..00000000 --- a/blunder/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/blunder - -generatedfiles := \ - fserror_string.go - -include ../GoMakefile diff --git a/blunder/api.go b/blunder/api.go deleted file mode 100644 index cd78e441..00000000 --- a/blunder/api.go +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -// Package blunder provides error-handling wrappers -// -// These wrappers allow callers to provide additional information in Go errors -// while still conforming to the Go error interface. -// -// This package provides APIs to add errno and HTTP status information to regular Go errors. -// -// This package is currently implemented on top of the ansel1/merry package: -// https://github.com/ansel1/merry -// -// merry comes with built-in support for adding information to errors: -// - stacktraces -// - overriding the error message -// - HTTP status codes -// - end user error messages -// - your own additional information -// -// From merry godoc: -// You can add any context information to an error with `e = merry.WithValue(e, "code", 12345)` -// You can retrieve that value with `v, _ := merry.Value(e, "code").(int)` -// -// FUTURE: Create APIs that add error information without doing a stacktrace. -// We probably want to do this, but not for thin spike. -// -// Currently, the merry package always adds a stack trace. -// However a recent change to the merry package permits disabling stacktrace. - -package blunder - -import ( - "fmt" - - "github.com/ansel1/merry" - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/logger" -) - -// Error constants to be used in the ProxyFS namespace. -// -// There are two groups of constants: -// - constants that correspond to linux/POSIX errnos as defined in errno.h -// - ProxyFS-specific constants for errors not covered in the errno space -// -// The linux/POSIX-related constants should be used in cases where there is a clear -// mapping to these errors. Using these constants makes it easier to map errors for -// use by our JSON RPC functionality. -// -// -// NOTE: unix.Errno is used here because they are errno constants that exist in Go-land. -// This type consists of an unsigned number describing an error condition. It implements -// the error interface; we need to cast it to an int to get the errno value. -// -type FsError int - -// The following line of code is a directive to go generate that tells it to create a -// file called fserror_string.go that implements the .String() method for type FsError. -//go:generate stringer -type=FsError - -const ( - // Errors that map to linux/POSIX errnos as defined in errno.h - // - NotPermError FsError = FsError(int(unix.EPERM)) // Operation not permitted - NotFoundError FsError = FsError(int(unix.ENOENT)) // No such file or directory - IOError FsError = FsError(int(unix.EIO)) // I/O error - ReadOnlyError FsError = FsError(int(unix.EROFS)) // Read-only file system - TooBigError FsError = FsError(int(unix.E2BIG)) // Argument list too long - TooManyArgsError FsError = FsError(int(unix.E2BIG)) // Arg list too long - BadFileError FsError = FsError(int(unix.EBADF)) // Bad file number - TryAgainError FsError = FsError(int(unix.EAGAIN)) // Try again - OutOfMemoryError FsError = FsError(int(unix.ENOMEM)) // Out of memory - PermDeniedError FsError = FsError(int(unix.EACCES)) // Permission denied - BadAddressError FsError = FsError(int(unix.EFAULT)) // Bad address - DevBusyError FsError = FsError(int(unix.EBUSY)) // Device or resource busy - FileExistsError FsError = FsError(int(unix.EEXIST)) // File exists - NoDeviceError FsError = FsError(int(unix.ENODEV)) // No such device - NotDirError FsError = FsError(int(unix.ENOTDIR)) // Not a directory - IsDirError FsError = FsError(int(unix.EISDIR)) // Is a directory - InvalidArgError FsError = FsError(int(unix.EINVAL)) // Invalid argument - TableOverflowError FsError = FsError(int(unix.ENFILE)) // File table overflow - TooManyOpenFilesError FsError = FsError(int(unix.EMFILE)) // Too many open files - FileTooLargeError FsError = FsError(int(unix.EFBIG)) // File too large - NoSpaceError FsError = FsError(int(unix.ENOSPC)) // No space left on device - BadSeekError FsError = FsError(int(unix.ESPIPE)) // Illegal seek - TooManyLinksError FsError = FsError(int(unix.EMLINK)) // Too many links - OutOfRangeError FsError = FsError(int(unix.ERANGE)) // Math result not representable - NameTooLongError FsError = FsError(int(unix.ENAMETOOLONG)) // File name too long - NoLocksError FsError = FsError(int(unix.ENOLCK)) // No record locks available - NotImplementedError FsError = FsError(int(unix.ENOSYS)) // Function not implemented - NotEmptyError FsError = FsError(int(unix.ENOTEMPTY)) // Directory not empty - TooManySymlinksError FsError = FsError(int(unix.ELOOP)) // Too many symbolic links encountered - NotSupportedError FsError = FsError(int(unix.ENOTSUP)) // Operation not supported - NoDataError FsError = FsError(int(unix.ENODATA)) // No data available - TimedOut FsError = FsError(int(unix.ETIMEDOUT)) // Connection Timed Out -) - -// Errors that map to constants already defined above -const ( - NotActiveError FsError = NotFoundError - BadLeaseRequest FsError = InvalidArgError - BadMountIDError FsError = InvalidArgError - BadMountVolumeError FsError = InvalidArgError - NotFileError FsError = IsDirError - SegNumNotIntError FsError = IOError - SegNotFoundError FsError = IOError - SegReadError FsError = IOError - InodeFlushError FsError = IOError - BtreeDeleteError FsError = IOError - BtreePutError FsError = IOError - BtreeLenError FsError = IOError - FileWriteError FsError = IOError - GetMetadataError FsError = IOError - NotSymlinkError FsError = InvalidArgError - IsSymlinkError FsError = InvalidArgError - LinkDirError FsError = NotPermError - BadHTTPDeleteError FsError = IOError - BadHTTPGetError FsError = IOError - BadHTTPHeadError FsError = IOError - BadHTTPPutError FsError = IOError - InvalidInodeTypeError FsError = InvalidArgError - InvalidFileModeError FsError = InvalidArgError - InvalidUserIDError FsError = InvalidArgError - InvalidGroupIDError FsError = InvalidArgError - StreamNotFound FsError = NoDataError - AccountNotModifiable FsError = NotPermError - OldMetaDataDifferent FsError = TryAgainError -) - -// Success error (sounds odd, no? - perhaps this could be renamed "NotAnError"?) -const SuccessError FsError = 0 - -const ( // reset iota to 0 - // Errors that are internal/specific to ProxyFS - UnpackError FsError = 1000 + iota - PackError - CorruptInodeError - NotAnObjectError -) - -// Default errno values for success and failure -const successErrno = 0 -const failureErrno = -1 - -// Value returns the int value for the specified FsError constant -func (err FsError) Value() int { - return int(err) -} - -// NewError creates a new merry/blunder.FsError-annotated error using the given -// format string and arguments. -func NewError(errValue FsError, format string, a ...interface{}) error { - return merry.WrapSkipping(fmt.Errorf(format, a...), 1).WithValue("errno", int(errValue)) -} - -// AddError is used to add FS error detail to a Go error. -// -// NOTE: Checks whether the error value has already been set -// Note that by default merry will replace the old with the new. -// -func AddError(e error, errValue FsError) error { - if e == nil { - // Error hasn't been allocated yet; need to create one - // - // Usually we wouldn't want to mess with a nil error, but the caller of - // this function obviously intends to make this a non-nil error. - // - // It's recommended that the caller create an error with some context - // in the error string first, but we don't want to silently not work - // if they forget to do that. - // - return merry.New("regular error").WithValue("errno", int(errValue)) - } - - // Make the error "merry", adding stack trace as well as errno value. - // This is done all in one line because the merry APIs create a new error each time. - - // For now, check and log if an errno has already been added to - // this error, to help debugging in the cases where this was not intentional. - prevValue := Errno(e) - if prevValue != successErrno && prevValue != failureErrno { - logger.Warnf("replacing error value %v with value %v for error %v.\n", prevValue, int(errValue), e) - } - - return merry.WrapSkipping(e, 1).WithValue("errno", int(errValue)) -} - -func hasErrnoValue(e error) bool { - // If the "errno" key/value was not present, merry.Value returns nil. - tmp := merry.Value(e, "errno") - if tmp != nil { - return true - } - - return false -} - -func AddHTTPCode(e error, statusCode int) error { - if e == nil { - // Error hasn't been allocated yet; need to create one - // - // Usually we wouldn't want to mess with a nil error, but the caller of - // this function obviously intends to make this a non-nil error. - // - // It's recommended that the caller create an error with some context - // in the error string first, but we don't want to silently not work - // if they forget to do that. - // - return merry.New("HTTP error").WithHTTPCode(statusCode) - } - - // Make the error "merry", adding stack trace as well as errno value. - // This is done all in one line because the merry APIs create a new error each time. - return merry.WrapSkipping(e, 1).WithHTTPCode(statusCode) -} - -// Errno extracts errno from the error, if it was previously wrapped. -// Otherwise a default value is returned. -// -func Errno(e error) int { - if e == nil { - // nil error = success - return successErrno - } - - // If the "errno" key/value was not present, merry.Value returns nil. - var errno = failureErrno - tmp := merry.Value(e, "errno") - if tmp != nil { - errno = tmp.(int) - } - - return errno -} - -func ErrorString(e error) string { - if e == nil { - return "" - } - - // Get the regular error string - errPlusVal := e.Error() - - // Add the error value to it, if set - var errno = failureErrno - tmp := merry.Value(e, "errno") - if tmp != nil { - errno = tmp.(int) - errPlusVal = fmt.Sprintf("%s. Error Value: %v\n", errPlusVal, errno) - } - - return errPlusVal -} - -// Check if an error matches a particular FsError -// -// NOTE: Because the value of the underlying errno is used to do this check, one cannot -// use this API to distinguish between FsErrors that use the same errno value. -// IOW, it can't tell the difference between InvalidFileModeError/BadMountIDError/InvalidArgError, -// since they all use unix.EINVAL as their underlying errno value. -// -func Is(e error, theError FsError) bool { - return Errno(e) == theError.Value() -} - -// Check if an error is NOT a particular FsError -func IsNot(e error, theError FsError) bool { - return Errno(e) != theError.Value() -} - -// Check if an error is the success FsError -func IsSuccess(e error) bool { - return Errno(e) == successErrno -} - -// Check if an error is NOT the success FsError -func IsNotSuccess(e error) bool { - return Errno(e) != successErrno -} - -func ErrorUpdate(e error, currentVal FsError, changeToVal FsError) error { - errVal := Errno(e) - - if errVal == int(currentVal) { - fmt.Printf("blunder.ErrorUpdate: errVal was %d, changing to %d.\n", errVal, int(changeToVal)) - // Change to the new value - return merry.Wrap(e).WithValue("errno", int(changeToVal)) - } - - return e -} - -// HTTPCode wraps merry.HTTPCode, which returns the HTTP status code. Default value is 500. -func HTTPCode(e error) int { - return merry.HTTPCode(e) -} - -// Location returns the file and line number of the code that generated the error. -// Returns zero values if e has no stacktrace. -func Location(e error) (file string, line int) { - file, line = merry.Location(e) - return -} - -// SourceLine returns the string representation of Location's result -// Returns empty stringif e has no stacktrace. -func SourceLine(e error) string { - return merry.SourceLine(e) -} - -// Details wraps merry.Details, which returns all error details including stacktrace in a string. -func Details(e error) string { - return merry.Details(e) -} - -// Stacktrace wraps merry.Stacktrace, which returns error stacktrace (if set) in a string. -func Stacktrace(e error) string { - return merry.Stacktrace(e) -} diff --git a/blunder/api_test.go b/blunder/api_test.go deleted file mode 100644 index 9a1d05bf..00000000 --- a/blunder/api_test.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package blunder - -import ( - "fmt" - "testing" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/transitions" -) - -var testConfMap conf.ConfMap - -func testSetup(t *testing.T) { - var ( - err error - testConfStrings []string - ) - - testConfStrings = []string{ - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - "Logging.LogFilePath=/dev/null", - "Cluster.WhoAmI=nobody", - "FSGlobals.VolumeGroupList=", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - err = transitions.Up(testConfMap) - if nil != err { - t.Fatalf("transitions.Up() failed: %v", err) - } -} - -func testTeardown(t *testing.T) { - var ( - err error - ) - - err = transitions.Down(testConfMap) - if nil != err { - t.Fatalf("transitions.Down() failed: %v", err) - } -} - -func TestValues(t *testing.T) { - errConstant := NotPermError - expectedValue := int(unix.EPERM) - if errConstant.Value() != expectedValue { - logger.Fatalf("Error, %s != %d", errConstant.String(), expectedValue) - } - // XXX TODO: add the rest of the values -} - -func checkValue(testInfo string, actualVal int, expectedVal int) bool { - if actualVal != expectedVal { - logger.Fatalf("Error, %s value was %d, expected %d", testInfo, actualVal, expectedVal) - return false - } - return true -} - -func TestDefaultErrno(t *testing.T) { - testSetup(t) - - // Nil error test - var err error - - // Now try to get error val out of err. We should get a default value, since error value hasn't been set. - errno := Errno(err) - - // Since err is nil, the default value should be successErrno - checkValue("nil error", errno, successErrno) - - // IsSuccess should return true and IsNotSuccess should return false - if !IsSuccess(err) { - logger.Fatalf("Error, IsSuccess() returned false for error %v (errno %v)", ErrorString(err), Errno(err)) - } - if IsNotSuccess(err) { - logger.Fatalf("Error, IsNotSuccess() returned true for error %v", ErrorString(err)) - } - - // Non-nil error test - err = fmt.Errorf("This is an ordinary error") - - // Since err is non-nil, the default value should be failureErrno (-1) - errno = Errno(err) - checkValue("non-nil error", errno, failureErrno) - - // IsSuccess should return false and IsNotSuccess should return true - if IsSuccess(err) { - logger.Fatalf("Error, IsSuccess() returned true for error %v (errno %v)", ErrorString(err), Errno(err)) - } - if !IsNotSuccess(err) { - logger.Fatalf("Error, IsNotSuccess() returned false for error %v", ErrorString(err)) - } - - // Specific error test - err = AddError(err, InvalidArgError) - errno = Errno(err) - checkValue("specific error", errno, InvalidArgError.Value()) - - testTeardown(t) -} - -func TestAddValue(t *testing.T) { - testSetup(t) - - // Add value to a nil error (not recommended as a strategy, but it needs to work anyway) - var err error - err = AddError(err, ReadOnlyError) - errno := Errno(err) - checkValue("specific error", errno, ReadOnlyError.Value()) - if !hasErrnoValue(err) { - logger.Fatalf("Error, hasErrnoValue returned false for error %v", ErrorString(err)) - } - // Validate the Is* APIs on what started as a nil error - if !Is(err, ReadOnlyError) { - logger.Fatalf("Error, Is() returned false for error %v is NameTooLongError", ErrorString(err)) - } - if Is(err, NotFoundError) { - logger.Fatalf("Error, Is() returned true for error %v is IsDirError", ErrorString(err)) - } - if !IsNot(err, InvalidArgError) { - logger.Fatalf("Error, IsNot() returned false for error %v is IsDirError", ErrorString(err)) - } - if IsSuccess(err) { - logger.Fatalf("Error, IsSuccess() returned true for error %v", ErrorString(err)) - } - if !IsNotSuccess(err) { - logger.Fatalf("Error, IsNotSuccess() returned false for error %v", ErrorString(err)) - } - - // Add value to a non-nil error - err = fmt.Errorf("This is an ordinary error") - err = AddError(err, NameTooLongError) - errno = Errno(err) - checkValue("specific error", errno, NameTooLongError.Value()) - if !hasErrnoValue(err) { - logger.Fatalf("Error, hasErrnoValue returned false for error %v", ErrorString(err)) - } - // Validate the Is* APIs on what started as a non-nil error - if !Is(err, NameTooLongError) { - logger.Fatalf("Error, Is() returned false for error %v is NameTooLongError", ErrorString(err)) - } - if Is(err, IsDirError) { - logger.Fatalf("Error, Is() returned true for error %v is IsDirError", ErrorString(err)) - } - if !IsNot(err, IsDirError) { - logger.Fatalf("Error, IsNot() returned false for error %v is IsDirError", ErrorString(err)) - } - if IsSuccess(err) { - logger.Fatalf("Error, IsSuccess() returned true for error %v", ErrorString(err)) - } - if !IsNotSuccess(err) { - logger.Fatalf("Error, IsNotSuccess() returned false for error %v", ErrorString(err)) - } - - // Add a different value to a non-nil error - err = AddError(err, ReadOnlyError) - errno = Errno(err) - checkValue("specific error", errno, ReadOnlyError.Value()) - if !hasErrnoValue(err) { - logger.Fatalf("Error, hasErrnoValue returned false for error %v", ErrorString(err)) - } - if !Is(err, ReadOnlyError) { - logger.Fatalf("Error, Is() returned false for error %v is NameTooLongError", ErrorString(err)) - } - - testTeardown(t) -} - -func TestHTTPCode(t *testing.T) { - testSetup(t) - - // Nil error test - // Add http code to a nil error (not recommended as a strategy, but it needs to work anyway) - var err error - - // Now try to get http code out of err. We should get a default value, since error value hasn't been set. - code := HTTPCode(err) - - // Since err is nil, the default value should be 200 OK - checkValue("nil error", code, 200) - - // Non-nil error test - err = fmt.Errorf("This is an ordinary error") - - // Err is non-nil but http code is not set, the default value should be 500 - code = HTTPCode(err) - checkValue("non-nil error", code, 500) - - // Specific error test - err = AddHTTPCode(err, 400) - code = HTTPCode(err) - checkValue("specific error", code, 400) - - testTeardown(t) -} diff --git a/go.mod b/go.mod index 1ee95066..f4543d4c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.17 require ( github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 - github.com/ansel1/merry v1.6.1 github.com/google/btree v1.0.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 @@ -14,36 +13,9 @@ require ( require ( github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 // indirect - github.com/ansel1/merry/v2 v2.0.0-beta.10 // indirect - github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/golang/glog v1.0.0 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/kr/pretty v0.2.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect - github.com/ory/go-acc v0.2.6 // indirect - github.com/ory/viper v1.7.5 // indirect - github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/afero v1.7.1 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/cobra v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.2.0 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 5b4c7c74..558048ee 100644 --- a/go.sum +++ b/go.sum @@ -1,911 +1,34 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b h1:Jhps18HvZUrJKJR2E57rF5Kp/7chK6NDo/wZnD0gtF0= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b/go.mod h1:9wVslsyxZaBvW/ollg7JLxJOxKb+Ik2KH1WVs1nicMA= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/ansel1/merry v1.5.0/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= -github.com/ansel1/merry v1.5.1/go.mod h1:wUy/yW0JX0ix9GYvUbciq+bi3jW/vlKPlbpI7qdZpOw= -github.com/ansel1/merry v1.6.1 h1:AuheQTqQ04yQdBMCDnneVPvpBdWTBBg3LKMvKGMyRxU= -github.com/ansel1/merry v1.6.1/go.mod h1:ioJjPJ/IsjxH+cC0lpf5TmbKnbcGa9qTk0fDbeRfnGQ= -github.com/ansel1/merry/v2 v2.0.0-beta.10 h1:F+TWcEeqJeQrsWBLg/fqf5LirLS1UdDkwZtKWYbsCus= -github.com/ansel1/merry/v2 v2.0.0-beta.10/go.mod h1:OUvUYh4KLVhf3+sR9Hk8QxCukijznkpheEd837b7vLg= -github.com/ansel1/vespucci/v4 v4.1.1/go.mod h1:zzdrO4IgBfgcGMbGTk/qNGL8JPslmW3nPpcBHKReFYY= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= -github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= -github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= -github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.7.1 h1:F37zV8E8RLstLpZ0RUGK2NGg1X57y6/B0Eg6S8oqdoA= -github.com/spf13/afero v1.7.1/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= -github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210224155714-063164c882e6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/retryrpc/perfrpc/ping_perf.go b/retryrpc/perfrpc/ping_perf.go index 21b3714a..d9795c74 100644 --- a/retryrpc/perfrpc/ping_perf.go +++ b/retryrpc/perfrpc/ping_perf.go @@ -7,7 +7,7 @@ import ( "bytes" "fmt" - "github.com/NVIDIA/proxyfs/blunder" + "golang.org/x/sys/unix" ) // PerfPingServer is a struct with pointer receivers implementing RpcPerfPing*() @@ -23,13 +23,6 @@ type PerfPingReply struct { Message string } -// Simple ping for performance testing the RPC layer -func perfEncodeErrno(e *error) { - if *e != nil { - *e = fmt.Errorf("errno: %d", blunder.Errno(*e)) - } -} - // RpcTestPing simply does a len on the message path and returns the result func (s *PerfPingServer) RpcPerfPing(in *PerfPingReq, reply *PerfPingReply) (err error) { reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) @@ -53,8 +46,7 @@ func (s *PerfPingServer) RpcPerfPingLarge(in *PerfPingReq, reply *PerfPingReply) // RpcPerfPingWithError returns an error func (s *PerfPingServer) RpcPerfPingWithError(in *PerfPingReq, reply *PerfPingReply) (err error) { - err = blunder.AddError(err, blunder.NotFoundError) - perfEncodeErrno(&err) + err = fmt.Errorf("errno: %v", unix.EIO) reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) return err } diff --git a/retryrpc/ping_test.go b/retryrpc/ping_test.go index 8108e2de..9daec24d 100644 --- a/retryrpc/ping_test.go +++ b/retryrpc/ping_test.go @@ -7,7 +7,7 @@ import ( "bytes" "fmt" - "github.com/NVIDIA/proxyfs/blunder" + "golang.org/x/sys/unix" ) // TestPingServer is a struct with pointer receivers implementing RpcTestPing*() @@ -23,13 +23,6 @@ type TestPingReply struct { Message string } -// Simple ping for testing the RPC layer -func testEncodeErrno(e *error) { - if *e != nil { - *e = fmt.Errorf("errno: %d", blunder.Errno(*e)) - } -} - // RpcTestPing simply does a len on the message path and returns the result func (s *TestPingServer) RpcTestPing(in *TestPingReq, reply *TestPingReply) (err error) { reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) @@ -53,8 +46,7 @@ func (s *TestPingServer) RpcTestPingLarge(in *TestPingReq, reply *TestPingReply) // RpcTestPingWithError returns an error func (s *TestPingServer) RpcTestPingWithError(in *TestPingReq, reply *TestPingReply) (err error) { - err = blunder.AddError(err, blunder.NotFoundError) - testEncodeErrno(&err) + err = fmt.Errorf("errno: %v", unix.EIO) reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) return err } @@ -68,8 +60,7 @@ func (s *TestPingServer) RpcTestPingWithClientID(clientID uint64, in *TestPingRe // RpcTestPingWithInvalidClientID is not a valid RPC // Note: Currently unused func (s *TestPingServer) RpcTestPingWithInvalidClientID(clientID int, in *TestPingReq, reply *TestPingReply) (err error) { - err = blunder.AddError(err, blunder.NotFoundError) - testEncodeErrno(&err) + err = fmt.Errorf("errno: %v", unix.EIO) reply.Message = fmt.Sprintf("client ID: %v pong %d bytes", clientID, len(in.Message)) return err } From 26f7aabeac7b954de883cbff5f4c394447ca6be2 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 30 Dec 2021 11:08:12 -0800 Subject: [PATCH 227/258] Removed dependency on (and presence of) package trackedlock --- go.mod | 25 ++ go.sum | 862 ++++++++++++++++++++++++++++++++++++ trackedlock/Makefile | 6 - trackedlock/api.go | 143 ------ trackedlock/api_internal.go | 539 ---------------------- trackedlock/api_test.go | 808 --------------------------------- trackedlock/config.go | 221 --------- 7 files changed, 887 insertions(+), 1717 deletions(-) delete mode 100644 trackedlock/Makefile delete mode 100644 trackedlock/api.go delete mode 100644 trackedlock/api_internal.go delete mode 100644 trackedlock/api_test.go delete mode 100644 trackedlock/config.go diff --git a/go.mod b/go.mod index f4543d4c..f9f40a74 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,34 @@ require ( require ( github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/kr/pretty v0.2.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/ory/viper v1.7.5 // indirect + github.com/pborman/uuid v1.2.1 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/afero v1.7.1 // indirect + github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cobra v1.3.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + golang.org/x/text v0.3.7 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 558048ee..3d01b878 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,896 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8 h1:hMAAyAeYB1T1DnxqdDZzjWeTDz/hL0ZGFhz3uQyH1nQ= github.com/NVIDIA/cstruct v0.0.0-20210817223100-441a06a021c8/go.mod h1:GPbuJvLD4QWiHPS6vivLzh+XMAx6va0Aucm6ipa5S0I= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b h1:Jhps18HvZUrJKJR2E57rF5Kp/7chK6NDo/wZnD0gtF0= github.com/NVIDIA/fission v0.0.0-20211223045359-ebc0c4203e7b/go.mod h1:9wVslsyxZaBvW/ollg7JLxJOxKb+Ik2KH1WVs1nicMA= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5 h1:mDx/maO8psu+pHQqEDoL15WTj/BAAnu/sKSeOVR8wZI= github.com/NVIDIA/sortedmap v0.0.0-20210902154213-c8c741ed94c5/go.mod h1:YtiQTabdmrFxECTKRqpuY/sXCKXOvaEc8plI2zYFb+k= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.7.1 h1:F37zV8E8RLstLpZ0RUGK2NGg1X57y6/B0Eg6S8oqdoA= +github.com/spf13/afero v1.7.1/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/trackedlock/Makefile b/trackedlock/Makefile deleted file mode 100644 index d047b0e9..00000000 --- a/trackedlock/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/trackedlock - -include ../GoMakefile diff --git a/trackedlock/api.go b/trackedlock/api.go deleted file mode 100644 index 6e81d6e3..00000000 --- a/trackedlock/api.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package trackedlock - -import ( - "fmt" - "sync" - "sync/atomic" - - "github.com/NVIDIA/proxyfs/logger" -) - -/* - - * The trackedlock packages provides an implementation of sync.Mutex and - * sync.RWMutex interfaces that provides additional functionality in the form of - * lock hold tracking. In addition, the trackedlock.RWMutexTrack routines can - * be used to track other RWMutex-like synchronization primitives, like - * dlm.RWLockStruct and chunked put connections. - * - * Specifically, if lock tracking is enabled, the trackedlock packages checks - * the lock hold time. When a lock unlocked, if it was held longer than - * "LockHoldTimeLimit" then a warning is logged along with the stack trace of - * the Lock() and Unlock() of the lock. In addition, a daemon, the trackedlock - * watcher, periodically checks to see if any lock has been locked too long. - * When a lock is held too long, the daemon logs the goroutine ID and the stack - * trace of the goroutine that acquired the lock. - * - * Subsequent versions of this package may include lock heirarchy checking. - * - * The config variable "TrackedLock.LockHoldTimeLimit" is the hold time that - * triggers warning messages being logged. If it is 0 then locks are not - * tracked and the overhead of this package is minimal. - * - * The config variable "TrackedLock.LockCheckPeriod" is how often the daemon - * checks tracked locks. If it is 0 then no daemon is created and lock hold - * time is checked only when the lock is unlocked (assuming it is unlocked). - * - * trackedlock locks can be locked before this package is initialized, but they - * will not be tracked until the first time they are locked after - * initializaiton. - * - * The API consists of the config based trackedlock.Up() / Down() / - * PauseAndContract() / ExpandAndResume() (which are not defined here) and then - * the Mutex, RWMutex, and RWMutexTrack interfaces. - */ - -// The Mutex type that we export, which wraps sync.Mutex to add tracking of lock -// hold time and the stack trace of the locker. -// -type Mutex struct { - wrappedMutex sync.Mutex // the actual Mutex - tracker MutexTrack // tracking information for the Mutex -} - -// The RWMutex type that we export, which wraps sync.RWMutex to add tracking of -// lock hold time and the stack trace of the locker. -// -type RWMutex struct { - wrappedRWMutex sync.RWMutex // actual Mutex - rwTracker RWMutexTrack // track holds in shared (reader) mode -} - -// -// Tracked Mutex API -// -func (m *Mutex) Lock() { - m.wrappedMutex.Lock() - - m.tracker.lockTrack(m, nil) -} - -func (m *Mutex) Unlock() { - m.tracker.unlockTrack(m) - - m.wrappedMutex.Unlock() -} - -// -// Tracked RWMutex API -// -func (m *RWMutex) Lock() { - m.wrappedRWMutex.Lock() - - m.rwTracker.lockTrack(m) -} - -func (m *RWMutex) Unlock() { - m.rwTracker.unlockTrack(m) - - m.wrappedRWMutex.Unlock() -} - -func (m *RWMutex) RLock() { - m.wrappedRWMutex.RLock() - - m.rwTracker.rLockTrack(m) -} - -func (m *RWMutex) RUnlock() { - m.rwTracker.rUnlockTrack(m) - - m.wrappedRWMutex.RUnlock() -} - -// -// Direct access to trackedlock API for DLM locks -// -func (rwmt *RWMutexTrack) LockTrack(lck interface{}) { - rwmt.lockTrack(lck) -} - -func (rwmt *RWMutexTrack) UnlockTrack(lck interface{}) { - rwmt.unlockTrack(lck) -} - -func (rwmt *RWMutexTrack) RLockTrack(lck interface{}) { - rwmt.rLockTrack(lck) -} - -func (rwmt *RWMutexTrack) RUnlockTrack(lck interface{}) { - rwmt.rUnlockTrack(lck) -} - -func (rwmt *RWMutexTrack) DLMUnlockTrack(lck interface{}) { - // This uses m.tracker.lockCnt without holding the mutex that protects it. - // Because this goroutine holds the lock, it cannot change from -1 to 0 - // or >0 to 0 (unless there's a bug where another goroutine releases the - // lock). It can change from, say, 1 to 2 or 4 to 3, but that's benign - // (let's hope the race detector doesn't complain). - lockCnt := atomic.LoadInt32(&rwmt.tracker.lockCnt) - switch { - case lockCnt == -1: - rwmt.unlockTrack(lck) - case lockCnt > 0: - rwmt.rUnlockTrack(lck) - default: - errstring := fmt.Errorf("tracker for RWMutexTrack has illegal lockCnt %d", lockCnt) - logger.PanicfWithError(errstring, "%T lock at %p: %+v", lck, lck, lck) - } - return -} diff --git a/trackedlock/api_internal.go b/trackedlock/api_internal.go deleted file mode 100644 index 9ccb4563..00000000 --- a/trackedlock/api_internal.go +++ /dev/null @@ -1,539 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package trackedlock - -import ( - "runtime" - "sync" - "sync/atomic" - "time" - - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/utils" -) - -type globalsStruct struct { - mapMutex sync.Mutex // protects mutexMap and rwMutexMap - mutexMap map[*MutexTrack]interface{} // the Mutex like locks being watched - rwMutexMap map[*RWMutexTrack]interface{} // the RWMutex like locks being watched - lockHoldTimeLimit int64 // locks held longer then this get logged (time.Duration) - lockCheckPeriod int64 // check locks once each period (time.Duration) - lockWatcherLocksLogged int // max overlimit locks logged by lockWatcher() - lockCheckChan <-chan time.Time // wait here to check on locks - stopChan chan struct{} // time to shutdown and go home - doneChan chan struct{} // shutdown complete - lockCheckTicker *time.Ticker // ticker for lock check time - dlmRWLockType interface{} // set by a callback from the DLM code -} - -var globals globalsStruct - -// stackTraceSlice holds the stack trace of one thread. stackTraceBuf is the -// storage required to hold one stack trace. We keep a pool of them around. -// -type stackTraceSlice []byte -type stackTraceBuf [4040]byte - -// time.IsZero() will return true if applied to timeZero. It indicates an -// uninitialized time stamp. -// -var timeZero time.Time - -type stackTraceObj struct { - stackTrace stackTraceSlice // stack trace of curent or last locker - stackTraceBuf stackTraceBuf // storage for stack trace slice -} - -var stackTraceObjPool = sync.Pool{ - New: func() interface{} { - return &stackTraceObj{} - }, -} - -// Track a Mutex or RWMutex held in exclusive mode (lockCnt is also used to -// track the number of shared lockers) -// -type MutexTrack struct { - isWatched bool // true if lock is on list of checked mutexes - lockCnt int32 // 0 if unlocked, -1 locked exclusive, > 0 locked shared - lockTime time.Time // time last lock operation completed - lockerGoId uint64 // goroutine ID of the last locker - lockStack *stackTraceObj // stack trace when object was last locked -} - -// Track an RWMutex -// -type RWMutexTrack struct { - tracker MutexTrack // tracking info when the lock is held exclusive - sharedStateLock sync.Mutex // lock the following fields (shared mode state) - rLockTime map[uint64]time.Time // GoId -> lock acquired time - rLockStack map[uint64]*stackTraceObj // GoId -> locker stack trace -} - -// Locking a Mutex or an RWMutex in exclusive (writer) mode. If this an -// Mutex-like lock then rwmt is nil, otherwise it points to the RWMutexTrack. -// -// Note that holding an RWMutex in exclusive mode insures that no goroutine -// holds it in shared mode. -// -func (mt *MutexTrack) lockTrack(wrappedLock interface{}, rwmt *RWMutexTrack) { - - // if lock tracking is disabled, just mark the lock as locked and clear - // the last locked time. (Recording the current time using time.Now() - // is too expensive, but we don't want this lock marked as held too long - // if lock tracking is enabled while its locked.) - if atomic.LoadInt64(&globals.lockHoldTimeLimit) == 0 { - mt.lockTime = timeZero - atomic.StoreInt32(&mt.lockCnt, -1) - return - } - - mt.lockStack = stackTraceObjPool.Get().(*stackTraceObj) - mt.lockStack.stackTrace = mt.lockStack.stackTraceBuf[:] - - cnt := runtime.Stack(mt.lockStack.stackTrace, false) - mt.lockStack.stackTrace = mt.lockStack.stackTrace[0:cnt] - mt.lockerGoId = utils.StackTraceToGoId(mt.lockStack.stackTrace) - mt.lockTime = time.Now() - atomic.StoreInt32(&mt.lockCnt, -1) - - // add to the list of watched mutexes if anybody is watching -- its only - // at this point that we need to know if this a Mutex or RWMutex, and it - // only happens when not currently watched - if !mt.isWatched && atomic.LoadInt64(&globals.lockCheckPeriod) != 0 { - globals.mapMutex.Lock() - - if rwmt != nil { - globals.rwMutexMap[rwmt] = wrappedLock - } else { - globals.mutexMap[mt] = wrappedLock - } - globals.mapMutex.Unlock() - mt.isWatched = true - } - - return -} - -// Unlocking a Mutex or unlocking an RWMutex held in exclusive (writer) mode -// -func (mt *MutexTrack) unlockTrack(wrappedLock interface{}) { - - // if we're checking the lock hold time and the locking time is recorded - // then check it - if atomic.LoadInt64(&globals.lockHoldTimeLimit) != 0 && !mt.lockTime.IsZero() { - now := time.Now() - if now.Sub(mt.lockTime) >= time.Duration(atomic.LoadInt64(&globals.lockHoldTimeLimit)) { - - var buf stackTraceBuf - unlockStackTrace := buf[:] - cnt := runtime.Stack(unlockStackTrace, false) - unlockStr := string(unlockStackTrace[0:cnt]) - - // mt.lockTime is recorded even if globals.lockHoldTimeLimit == 0, - // so its possible mt.lockStack is not allocated - lockStr := "goroutine 9999 [unknown]\nlocked before lock tracking enabled\n" - if mt.lockStack != nil { - lockStr = string(mt.lockStack.stackTrace) - } - logger.Warnf("Unlock(): %T at %p locked for %f sec;"+ - " stack at call to Lock():\n%s\nstack at Unlock():\n%s", - wrappedLock, wrappedLock, - float64(now.Sub(mt.lockTime))/float64(time.Second), lockStr, unlockStr) - } - } - - // release the lock - atomic.StoreInt32(&mt.lockCnt, 0) - if mt.lockStack != nil { - stackTraceObjPool.Put(mt.lockStack) - mt.lockStack = nil - } - return -} - -// Tracking an RWMutex locked exclusive is just like a regular Mutex -// -func (rwmt *RWMutexTrack) lockTrack(wrappedLock interface{}) { - rwmt.tracker.lockTrack(wrappedLock, rwmt) -} - -func (rwmt *RWMutexTrack) unlockTrack(wrappedLock interface{}) { - rwmt.tracker.unlockTrack(wrappedLock) -} - -// Tracking an RWMutex locked shared is more work -// -func (rwmt *RWMutexTrack) rLockTrack(wrappedLock interface{}) { - - // if lock tracking is disabled, just mark the lock as locked and clear - // the last locked time. (Recording the current time using time.Now() - // is too expensive, but we don't want this lock marked as held too long - // if lock tracking is enabled while its locked.) - if atomic.LoadInt64(&globals.lockHoldTimeLimit) == 0 { - rwmt.sharedStateLock.Lock() - atomic.AddInt32(&rwmt.tracker.lockCnt, 1) - rwmt.tracker.lockTime = timeZero - rwmt.sharedStateLock.Unlock() - - return - } - - // get the stack trace and goId before getting the shared state lock to - // cut down on lock contention - lockStack := stackTraceObjPool.Get().(*stackTraceObj) - lockStack.stackTrace = lockStack.stackTraceBuf[:] - - cnt := runtime.Stack(lockStack.stackTrace, false) - lockStack.stackTrace = lockStack.stackTrace[0:cnt] - goId := utils.StackTraceToGoId(lockStack.stackTrace) - - // The lock is held as a reader (shared mode) so no goroutine can have - // it locked exclusive. Holding rwmt.sharedStateLock is sufficient to - // insure that no other goroutine is changing rwmt.tracker.lockCnt. - rwmt.sharedStateLock.Lock() - - if rwmt.rLockStack == nil { - rwmt.rLockStack = make(map[uint64]*stackTraceObj) - rwmt.rLockTime = make(map[uint64]time.Time) - } - rwmt.tracker.lockTime = time.Now() - rwmt.rLockTime[goId] = rwmt.tracker.lockTime - rwmt.rLockStack[goId] = lockStack - atomic.AddInt32(&rwmt.tracker.lockCnt, 1) - - // add to the list of watched mutexes if anybody is watching - if !rwmt.tracker.isWatched && atomic.LoadInt64(&globals.lockCheckPeriod) != 0 { - - globals.mapMutex.Lock() - globals.rwMutexMap[rwmt] = wrappedLock - globals.mapMutex.Unlock() - rwmt.tracker.isWatched = true - } - rwmt.sharedStateLock.Unlock() - - return -} - -func (rwmt *RWMutexTrack) rUnlockTrack(wrappedLock interface{}) { - - if atomic.LoadInt64(&globals.lockHoldTimeLimit) == 0 { - rwmt.sharedStateLock.Lock() - - // since lock hold time is not being checked, discard any info - // left over from an earlier time when it was - if rwmt.rLockStack != nil { - for goId, lockStack := range rwmt.rLockStack { - stackTraceObjPool.Put(lockStack) - delete(rwmt.rLockStack, goId) - delete(rwmt.rLockTime, goId) - } - rwmt.rLockStack = nil - rwmt.rLockTime = nil - } - atomic.AddInt32(&rwmt.tracker.lockCnt, -1) - - rwmt.sharedStateLock.Unlock() - return - } - - // Lock tracking is enabled; track the unlock of this lock. - // - // If the goroutine unlocking a shared lock is not the same as the one - // that locked it then we cannot match up RLock() and RUnlock() - // operations (unless we add an API that allows lock "ownership" to be - // passed between goroutines and the client of the API actually calls - // it). This is complicated by the situation when lock tracking - // (globals.lockHoldTimeLimit > 0) is enabled after some locks have - // already been acquired in reader mode. - // - // To handle the second case this code does not kick up a fuss if it - // can't find an RLock() operation with a goID corresponding to this - // RUnlock(), assuming that the RLock() occurred when locks were not - // being tracked. It does nothing (and does not remove an entry from - // rwmt.rLockTime). As a consequence, if shared locks are been passed - // between go routines rwmt.rLockTime and rwmt.rLockStack can have a - // bunch of stale entries which would trigger false instances of "lock - // held too long" by the lock watcher. To compensate for this, when the - // last RLock() is released, we clear out extra stale entries. - goId := utils.GetGoId() - now := time.Now() - rwmt.sharedStateLock.Lock() - rLockTime, ok := rwmt.rLockTime[goId] - if ok && now.Sub(rLockTime) >= time.Duration(atomic.LoadInt64(&globals.lockHoldTimeLimit)) { - - var buf stackTraceBuf - stackTrace := buf[:] - cnt := runtime.Stack(stackTrace, false) - stackTrace = stackTrace[0:cnt] - - logger.Warnf("RUnlock(): %T at %p locked for %f sec;"+ - " stack at call to RLock():\n%s\nstack at RUnlock():\n%s", - wrappedLock, wrappedLock, float64(now.Sub(rLockTime))/float64(time.Second), - rwmt.rLockStack[goId].stackTrace, string(stackTrace)) - } - - if atomic.LoadInt32(&rwmt.tracker.lockCnt) == 1 && len(rwmt.rLockTime) > 1 { - for goId, stackTraceObj := range rwmt.rLockStack { - stackTraceObjPool.Put(stackTraceObj) - delete(rwmt.rLockStack, goId) - delete(rwmt.rLockTime, goId) - } - } else if ok { - stackTraceObjPool.Put(rwmt.rLockStack[goId]) - delete(rwmt.rLockStack, goId) - delete(rwmt.rLockTime, goId) - } - atomic.AddInt32(&rwmt.tracker.lockCnt, -1) - rwmt.sharedStateLock.Unlock() - - return -} - -// information about a lock that is held too long -type longLockHolder struct { - lockPtr interface{} // pointer to the actual lock - lockTime time.Time // time last lock operation completed - lockerGoId uint64 // goroutine ID of the last locker - lockStackStr string // stack trace when the object was locked - lockOp string // lock operation name ("Lock()" or "RLock()") - mutexTrack *MutexTrack // if lock is a Mutex, MutexTrack for lock - rwMutexTrack *RWMutexTrack // if lock is an RWMutex, RWMutexTrack for lock -} - -// Record a lock that has been held too long. -// -// longLockHolders is a slice containing longLockHolder information for upto -// globals.lockWatcherLocksLogged locks, sorted from longest to shortest held. Add -// newHolder to the slice, potentially discarding the lock that has been held least -// long. -// -func recordLongLockHolder(longLockHolders []*longLockHolder, newHolder *longLockHolder) []*longLockHolder { - - // if there's room append the new entry, else overwrite the youngest lock holder - if len(longLockHolders) < globals.lockWatcherLocksLogged { - longLockHolders = append(longLockHolders, newHolder) - } else { - longLockHolders[len(longLockHolders)-1] = newHolder - } - - // the new entry may not be the shortest lock holder; resort the list - for i := len(longLockHolders) - 2; i >= 0; i -= 1 { - - if longLockHolders[i].lockTime.Before(longLockHolders[i+1].lockTime) { - break - } - tmpHolder := longLockHolders[i] - longLockHolders[i] = longLockHolders[i+1] - longLockHolders[i+1] = tmpHolder - } - - return longLockHolders -} - -// Periodically check for locks that have been held too long. -// -func lockWatcher() { - - for shutdown := false; !shutdown; { - select { - case <-globals.stopChan: - shutdown = true - logger.Infof("trackedlock lock watcher shutting down") - // fall through and perform one last check - - case <-globals.lockCheckChan: - // fall through and perform checks - } - - // LongLockHolders holds information about locks for locks that have - // been held longer then globals.lockHoldTimeLimit, upto a maximum of - // globals.lockWatcherLocksLogged locks. They are sorted longest to - // shortest. - // - // longestDuration is the minimum lock hold time needed to be added to the - // slice, which increases once the slice is full. - // - // Note that this could be running after lock tracking or the lock watcher - // has been disabled (globals.lockHoldTimeLimit and/or - // globals.lockCheckPeriod could be 0). - var ( - longLockHolders = make([]*longLockHolder, 0) - longestDuration = time.Duration(atomic.LoadInt64(&globals.lockHoldTimeLimit)) - ) - if atomic.LoadInt64(&globals.lockHoldTimeLimit) == 0 { - // ignore locks held less than 10 years - longestDuration, _ = time.ParseDuration("24h") - longestDuration *= 365 * 10 - } - - // now does not change during a loop iteration - now := time.Now() - - // See if any Mutex has been held longer then the limit and, if - // so, find the one held the longest. - // - // Go guarantees that looking at mutex.tracker.lockTime is safe even if - // we don't hold the mutex (in practice, looking at tracker.lockCnt - // should be safe as well, though go doesn't guarantee that). - globals.mapMutex.Lock() - for mt, lockPtr := range globals.mutexMap { - - // If the lock is not locked then skip it; if it has been idle - // for the lockCheckPeriod then drop it from the locks being - // watched. - // - // Note that this is the only goroutine that deletes locks from - // globals.mutexMap so any mt pointer we save after this point - // will continue to be valid until the next iteration. - if atomic.LoadInt32(&mt.lockCnt) == 0 { - lastLocked := now.Sub(mt.lockTime) - if lastLocked >= time.Duration(atomic.LoadInt64(&globals.lockCheckPeriod)) { - mt.isWatched = false - delete(globals.mutexMap, mt) - } - continue - } - - lockedDuration := now.Sub(mt.lockTime) - if lockedDuration > longestDuration { - - // We do not hold a lock that prevents mt.lockStack.stackTrace - // from changing if the Mutex were to be unlocked right now, - // although its unlikely since the mutex has been held so long, - // so try to copy the stack robustly. If it does change it - // might be reused and now hold another stack and this stack - // trace would be wrong. - var ( - mtStackTraceObj *stackTraceObj = mt.lockStack - mtStack string - ) - if mt.lockStack != nil { - mtStack = string(mtStackTraceObj.stackTrace) - } - longHolder := &longLockHolder{ - lockPtr: lockPtr, - lockTime: mt.lockTime, - lockerGoId: mt.lockerGoId, - lockStackStr: mtStack, - lockOp: "Lock()", - mutexTrack: mt, - } - longLockHolders = recordLongLockHolder(longLockHolders, longHolder) - - // if we've hit the maximum number of locks then bump - // longestDuration - if len(longLockHolders) == globals.lockWatcherLocksLogged { - longestDuration = lockedDuration - } - } - } - globals.mapMutex.Unlock() - - // give other routines a chance to get the global lock - time.Sleep(10 * time.Millisecond) - - // see if any RWMutex has been held longer then the limit and, if - // so, find the one held the longest. - // - // looking at the rwMutex.tracker.* fields is safe per the - // argument for Mutex, but looking at rTracker maps requires we - // get rtracker.sharedStateLock for each lock. - globals.mapMutex.Lock() - for rwmt, lockPtr := range globals.rwMutexMap { - - // If the lock is not locked then skip it. If it has been - // idle for the lockCheckPeriod (rwmt.tracker.lockTime is - // updated when the lock is locked shared or exclusive) then - // drop it from the locks being watched. - if atomic.LoadInt32(&rwmt.tracker.lockCnt) == 0 { - lastLocked := now.Sub(rwmt.tracker.lockTime) - if lastLocked >= time.Duration(atomic.LoadInt64(&globals.lockCheckPeriod)) { - rwmt.tracker.isWatched = false - delete(globals.rwMutexMap, rwmt) - } - continue - } - - if atomic.LoadInt32(&rwmt.tracker.lockCnt) < 0 { - lockedDuration := now.Sub(rwmt.tracker.lockTime) - if lockedDuration > longestDuration { - - // We do not hold a lock that prevents - // rwmt.lockStack.stackTrace from changing if the - // RWMutex were to be unlocked right now, although its - // unlikely since the rwmutex has been held so long, so - // try to copy the stack robustly. If it does change i - // it might be reused and now hold another stack. - // might be reused and now hold another stack and this - // stack trace would be wrong. - - var ( - rwmtStackTraceObj *stackTraceObj = rwmt.tracker.lockStack - rwmtStack string - ) - if rwmt.tracker.lockStack != nil { - rwmtStack = string(rwmtStackTraceObj.stackTrace) - } - longHolder := &longLockHolder{ - lockPtr: lockPtr, - lockTime: rwmt.tracker.lockTime, - lockerGoId: rwmt.tracker.lockerGoId, - lockStackStr: rwmtStack, - lockOp: "Lock()", - rwMutexTrack: rwmt, - } - longLockHolders = recordLongLockHolder(longLockHolders, longHolder) - if len(longLockHolders) == globals.lockWatcherLocksLogged { - longestDuration = lockedDuration - } - } - - continue - } - - rwmt.sharedStateLock.Lock() - - for goId, lockTime := range rwmt.rLockTime { - lockedDuration := now.Sub(lockTime) - if lockedDuration > longestDuration { - - // we do hold a lock that protects rwmt.rLockStack - longHolder := &longLockHolder{ - lockPtr: lockPtr, - lockTime: lockTime, - lockerGoId: goId, - lockStackStr: string(rwmt.rLockStack[goId].stackTrace), - lockOp: "RLock()", - rwMutexTrack: rwmt, - } - longLockHolders = recordLongLockHolder(longLockHolders, longHolder) - if len(longLockHolders) == globals.lockWatcherLocksLogged { - longestDuration = lockedDuration - } - } - } - rwmt.sharedStateLock.Unlock() - } - globals.mapMutex.Unlock() - - if len(longLockHolders) == 0 { - // nothing to see! move along! - continue - } - - // log a Warning for each lock that has been held too long, - // from longest to shortest - for i := 0; i < len(longLockHolders); i += 1 { - logger.Warnf("trackedlock watcher: %T at %p locked for %f sec rank %d;"+ - " stack at call to %s:\n%s\n", - longLockHolders[i].lockPtr, longLockHolders[i].lockPtr, - float64(now.Sub(longLockHolders[i].lockTime))/float64(time.Second), i, - longLockHolders[i].lockOp, longLockHolders[i].lockStackStr) - - } - } - - globals.doneChan <- struct{}{} -} diff --git a/trackedlock/api_test.go b/trackedlock/api_test.go deleted file mode 100644 index bfc9f882..00000000 --- a/trackedlock/api_test.go +++ /dev/null @@ -1,808 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package trackedlock - -/* - * Test tracked locks. - * - * Most of this file is copied from statslogger/config_test.go because its a - * good place to start. - */ - -import ( - "fmt" - "regexp" - "strconv" - "sync/atomic" - "testing" - "time" - "unsafe" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/logger" -) - -// Common configuration for all tests (unless overridden) -// -var ( - confStrings = []string{ - "TrackedLock.LockHoldTimeLimit=2s", - "TrackedLock.LockCheckPeriod=1s", - - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - } - - // matches: "trackedlock watcher: *trackedlock.Mutex at 0xc420110000 locked for 2.003s sec rank 0; stack at call to Lock():\ngoroutine 19 [running]:..." - // that is it only matches the entry for rank 0 (the longest held lock) - // - watcherRank0LogMatch = `^trackedlock watcher: (?P[*a-zA-Z0-9_.]+) at (?P0x[0-9a-f]+) locked for (?P